44#define DEBUG_TYPE "gi-combiner"
47using namespace MIPatternMatch;
53 cl::desc(
"Force all indexed operations to be "
54 "legal for the GlobalISel combiner"));
60 : Builder(
B),
MRI(Builder.getMF().getRegInfo()), Observer(Observer), KB(KB),
61 MDT(MDT), IsPreLegalize(IsPreLegalize), LI(LI),
62 RBI(Builder.getMF().getSubtarget().getRegBankInfo()),
63 TRI(Builder.getMF().getSubtarget().getRegisterInfo()) {
76 assert(
I < ByteWidth &&
"I must be in [0, ByteWidth)");
95 assert(
I < ByteWidth &&
"I must be in [0, ByteWidth)");
96 return ByteWidth -
I - 1;
116static std::optional<bool>
120 unsigned Width = MemOffset2Idx.
size();
123 bool BigEndian =
true, LittleEndian =
true;
124 for (
unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {
125 auto MemOffsetAndIdx = MemOffset2Idx.
find(MemOffset);
126 if (MemOffsetAndIdx == MemOffset2Idx.
end())
128 const int64_t
Idx = MemOffsetAndIdx->second - LowestIdx;
129 assert(
Idx >= 0 &&
"Expected non-negative byte offset?");
132 if (!BigEndian && !LittleEndian)
136 assert((BigEndian != LittleEndian) &&
137 "Pattern cannot be both big and little endian!");
144 assert(
LI &&
"Must have LegalizerInfo to query isLegal!");
160 return isLegal({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
161 isLegal({TargetOpcode::G_CONSTANT, {EltTy}});
188 unsigned ToOpcode)
const {
213 if (
MI.getOpcode() != TargetOpcode::COPY)
222 MI.eraseFromParent();
228 assert(
MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
229 "Invalid instruction");
239 assert(Def &&
"Operand not defined");
242 switch (Def->getOpcode()) {
243 case TargetOpcode::G_BUILD_VECTOR:
250 case TargetOpcode::G_IMPLICIT_DEF: {
259 "All undefs should have the same type");
263 EltIdx != EltEnd; ++EltIdx)
264 Ops.
push_back(Undef->getOperand(0).getReg());
275 {TargetOpcode::G_BUILD_VECTOR, {DstTy,
MRI.
getType(Ops[0])}})) {
302 MI.eraseFromParent();
317 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
318 "Invalid instruction kind");
343 if (DstNumElts < 2 * SrcNumElts && DstNumElts != 1)
348 if (DstNumElts % SrcNumElts != 0)
354 unsigned NumConcat = DstNumElts / SrcNumElts;
357 for (
unsigned i = 0; i != DstNumElts; ++i) {
364 if ((
Idx % SrcNumElts != (i % SrcNumElts)) ||
365 (ConcatSrcs[i / SrcNumElts] >= 0 &&
366 ConcatSrcs[i / SrcNumElts] != (
int)(
Idx / SrcNumElts)))
369 ConcatSrcs[i / SrcNumElts] =
Idx / SrcNumElts;
376 for (
auto Src : ConcatSrcs) {
402 MI.eraseFromParent();
407 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
408 "Invalid instruction kind");
411 return Mask.size() == 1;
418 int I =
MI.getOperand(3).getShuffleMask()[0];
423 if (
I >= Src1NumElts) {
424 SrcReg =
MI.getOperand(2).getReg();
436 MI.eraseFromParent();
445 const LLT TyForCandidate,
446 unsigned OpcodeForCandidate,
451 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
462 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&
465 else if (CurrentUse.
ExtendOpcode == TargetOpcode::G_ANYEXT &&
466 OpcodeForCandidate != TargetOpcode::G_ANYEXT)
467 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
473 if (!isa<GZExtLoad>(LoadMI) && CurrentUse.
Ty == TyForCandidate) {
475 OpcodeForCandidate == TargetOpcode::G_ZEXT)
477 else if (CurrentUse.
ExtendOpcode == TargetOpcode::G_ZEXT &&
478 OpcodeForCandidate == TargetOpcode::G_SEXT)
479 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
488 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
499static void InsertInsnsWithoutSideEffectsBeforeUse(
511 InsertBB = PredBB->
getMBB();
516 if (InsertBB ==
DefMI.getParent()) {
518 Inserter(InsertBB, std::next(InsertPt), UseMO);
537 unsigned CandidateLoadOpc;
539 case TargetOpcode::G_ANYEXT:
540 CandidateLoadOpc = TargetOpcode::G_LOAD;
542 case TargetOpcode::G_SEXT:
543 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;
545 case TargetOpcode::G_ZEXT:
546 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;
551 return CandidateLoadOpc;
582 if (!llvm::has_single_bit<uint32_t>(LoadValueTy.
getSizeInBits()))
590 unsigned PreferredOpcode =
592 ? TargetOpcode::G_ANYEXT
593 : isa<GSExtLoad>(&
MI) ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
594 Preferred = {
LLT(), PreferredOpcode,
nullptr};
596 if (
UseMI.getOpcode() == TargetOpcode::G_SEXT ||
597 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
598 (
UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
599 const auto &MMO = LoadMI->
getMMO();
609 if (
LI->
getAction({CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})
613 Preferred = ChoosePreferredUse(
MI, Preferred,
624 assert(Preferred.Ty != LoadValueTy &&
"Extending to same type?");
642 if (PreviouslyEmitted) {
652 EmittedInsns[InsertIntoBB] = NewMI;
661 auto &LoadValue =
MI.getOperand(0);
664 Uses.push_back(&UseMO);
666 for (
auto *UseMO :
Uses) {
676 if (UseDstReg != ChosenDstReg) {
677 if (Preferred.
Ty == UseDstTy) {
714 InsertInsnsWithoutSideEffectsBeforeUse(
Builder,
MI, *UseMO,
729 InsertInsnsWithoutSideEffectsBeforeUse(
Builder,
MI, *UseMO, InsertTruncAt);
732 MI.getOperand(0).setReg(ChosenDstReg);
738 assert(
MI.getOpcode() == TargetOpcode::G_AND);
757 APInt MaskVal = MaybeMask->Value;
778 if (MaskSizeBits > LoadSizeBits.
getValue())
798 else if (LoadSizeBits.
getValue() > MaskSizeBits ||
804 {TargetOpcode::G_ZEXTLOAD, {RegTy,
MRI.
getType(PtrReg)}, {MemDesc}}))
808 B.setInstrAndDebugLoc(*LoadMI);
809 auto &MF =
B.getMF();
811 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, MemDesc.
MemoryTy);
812 B.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, Dst, PtrReg, *NewMMO);
821 "shouldn't consider debug uses");
829 if (DefOrUse ==
MBB.
end())
831 return &*DefOrUse == &
DefMI;
837 "shouldn't consider debug uses");
840 else if (
DefMI.getParent() !=
UseMI.getParent())
847 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
858 uint64_t SizeInBits =
MI.getOperand(2).getImm();
861 if (
auto *LoadMI = getOpcodeDef<GSExtLoad>(LoadUser,
MRI)) {
863 auto LoadSizeBits = LoadMI->getMemSizeInBits();
867 if (LoadSizeBits == SizeInBits)
874 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
876 MI.eraseFromParent();
881 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
891 auto *LoadDef = getOpcodeDef<GLoad>(SrcReg,
MRI);
895 uint64_t MemBits = LoadDef->getMemSizeInBits().getValue();
900 unsigned NewSizeBits = std::min((
uint64_t)
MI.getOperand(2).getImm(), MemBits);
915 if (LoadDef->isSimple())
917 else if (MemBits > NewSizeBits || MemBits == RegTy.
getSizeInBits())
927 MatchInfo = std::make_tuple(LoadDef->getDstReg(), NewSizeBits);
933 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
935 unsigned ScalarSizeBits;
936 std::tie(LoadReg, ScalarSizeBits) = MatchInfo;
945 auto &MMO = LoadDef->
getMMO();
948 auto PtrInfo = MMO.getPointerInfo();
952 MI.eraseFromParent();
967 auto *MF =
MI->getMF();
968 auto *
Addr = getOpcodeDef<GPtrAdd>(
MI->getPointerReg(),
MRI);
974 AM.
BaseOffs = CstOff->getSExtValue();
979 MF->getDataLayout(), AM,
981 MF->getFunction().getContext()),
982 MI->getMMO().getAddrSpace());
987 case TargetOpcode::G_LOAD:
988 return TargetOpcode::G_INDEXED_LOAD;
989 case TargetOpcode::G_STORE:
990 return TargetOpcode::G_INDEXED_STORE;
991 case TargetOpcode::G_ZEXTLOAD:
992 return TargetOpcode::G_INDEXED_ZEXTLOAD;
993 case TargetOpcode::G_SEXTLOAD:
994 return TargetOpcode::G_INDEXED_SEXTLOAD;
1000bool CombinerHelper::isIndexedLoadStoreLegal(
GLoadStore &LdSt)
const {
1009 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)
1010 OpTys = {PtrTy, Ty, Ty};
1012 OpTys = {Ty, PtrTy};
1020 cl::desc(
"Number of uses of a base pointer to check before it is no longer "
1021 "considered for post-indexing."));
1025 bool &RematOffset) {
1038 if (!isIndexedLoadStoreLegal(LdSt))
1047 unsigned NumUsesChecked = 0;
1052 auto *PtrAdd = dyn_cast<GPtrAdd>(&
Use);
1060 if (StoredValDef == &
Use)
1063 Offset = PtrAdd->getOffsetReg();
1065 !TLI.isIndexingLegal(LdSt, PtrAdd->getBaseReg(),
Offset,
1071 RematOffset =
false;
1075 if (OffsetDef->
getOpcode() != TargetOpcode::G_CONSTANT)
1081 if (&BasePtrUse == PtrDef)
1086 auto *BasePtrLdSt = dyn_cast<GLoadStore>(&BasePtrUse);
1087 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&
1089 isIndexedLoadStoreLegal(*BasePtrLdSt))
1094 if (
auto *BasePtrUseDef = dyn_cast<GPtrAdd>(&BasePtrUse)) {
1095 Register PtrAddDefReg = BasePtrUseDef->getReg(0);
1099 if (BaseUseUse.getParent() != LdSt.
getParent())
1102 if (
auto *UseUseLdSt = dyn_cast<GLoadStore>(&BaseUseUse))
1111 Addr = PtrAdd->getReg(0);
1112 Base = PtrAdd->getBaseReg();
1133 if (!isIndexedLoadStoreLegal(LdSt))
1137 if (BaseDef->
getOpcode() == TargetOpcode::G_FRAME_INDEX)
1140 if (
auto *St = dyn_cast<GStore>(&LdSt)) {
1142 if (
Base == St->getValueReg())
1147 if (St->getValueReg() ==
Addr)
1153 if (AddrUse.getParent() != LdSt.
getParent())
1158 bool RealUse =
false;
1165 if (
auto *UseLdSt = dyn_cast<GLoadStore>(&AddrUse)) {
1177 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1180 auto *LoadMI = getOpcodeDef<GLoad>(
MI.getOperand(1).getReg(),
MRI);
1194 if (!LoadMI->isSimple())
1206 const unsigned MaxIter = 20;
1208 for (
auto II = LoadMI->
getIterator(), IE =
MI.getIterator(); II != IE; ++II) {
1209 if (II->isLoadFoldBarrier())
1211 if (Iter++ == MaxIter)
1227 int Elt = CVal->getZExtValue();
1240 Register VecPtr = LoadMI->getPointerReg();
1248 LegalityQuery Q = {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}};
1273 B.buildLoad(Result, finalPtr, PtrInfo, Alignment);
1283 auto &LdSt = cast<GLoadStore>(
MI);
1288 MatchInfo.
IsPre = findPreIndexCandidate(LdSt, MatchInfo.
Addr, MatchInfo.
Base,
1290 if (!MatchInfo.
IsPre &&
1291 !findPostIndexCandidate(LdSt, MatchInfo.
Addr, MatchInfo.
Base,
1301 unsigned Opcode =
MI.getOpcode();
1302 bool IsStore = Opcode == TargetOpcode::G_STORE;
1310 *OldCst->getOperand(1).getCImm());
1311 MatchInfo.
Offset = NewCst.getReg(0);
1317 MIB.
addUse(
MI.getOperand(0).getReg());
1319 MIB.
addDef(
MI.getOperand(0).getReg());
1327 MI.eraseFromParent();
1335 unsigned Opcode =
MI.getOpcode();
1336 bool IsDiv, IsSigned;
1341 case TargetOpcode::G_SDIV:
1342 case TargetOpcode::G_UDIV: {
1344 IsSigned = Opcode == TargetOpcode::G_SDIV;
1347 case TargetOpcode::G_SREM:
1348 case TargetOpcode::G_UREM: {
1350 IsSigned = Opcode == TargetOpcode::G_SREM;
1356 unsigned DivOpcode, RemOpcode, DivremOpcode;
1358 DivOpcode = TargetOpcode::G_SDIV;
1359 RemOpcode = TargetOpcode::G_SREM;
1360 DivremOpcode = TargetOpcode::G_SDIVREM;
1362 DivOpcode = TargetOpcode::G_UDIV;
1363 RemOpcode = TargetOpcode::G_UREM;
1364 DivremOpcode = TargetOpcode::G_UDIVREM;
1383 if (
MI.getParent() ==
UseMI.getParent() &&
1384 ((IsDiv &&
UseMI.getOpcode() == RemOpcode) ||
1385 (!IsDiv &&
UseMI.getOpcode() == DivOpcode)) &&
1398 unsigned Opcode =
MI.getOpcode();
1399 assert(OtherMI &&
"OtherMI shouldn't be empty.");
1402 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {
1403 DestDivReg =
MI.getOperand(0).getReg();
1407 DestRemReg =
MI.getOperand(0).getReg();
1411 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;
1421 : TargetOpcode::G_UDIVREM,
1422 {DestDivReg, DestRemReg},
1424 MI.eraseFromParent();
1430 assert(
MI.getOpcode() == TargetOpcode::G_BR);
1449 assert(std::next(BrIt) ==
MBB->
end() &&
"expected G_BR to be a terminator");
1451 BrCond = &*std::prev(BrIt);
1452 if (BrCond->
getOpcode() != TargetOpcode::G_BRCOND)
1458 return BrCondTarget !=
MI.getOperand(0).getMBB() &&
1476 MI.getOperand(0).setMBB(FallthroughBB);
1492 return Helper.lowerMemcpyInline(
MI) ==
1508 switch (
MI.getOpcode()) {
1511 case TargetOpcode::G_FNEG: {
1512 Result.changeSign();
1515 case TargetOpcode::G_FABS: {
1519 case TargetOpcode::G_FPTRUNC: {
1521 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
1526 case TargetOpcode::G_FSQRT: {
1530 Result =
APFloat(sqrt(Result.convertToDouble()));
1533 case TargetOpcode::G_FLOG2: {
1554 MI.eraseFromParent();
1565 if (
MI.getOpcode() != TargetOpcode::G_PTR_ADD)
1575 if (!Add2Def || Add2Def->
getOpcode() != TargetOpcode::G_PTR_ADD)
1588 Type *AccessTy =
nullptr;
1589 auto &MF = *
MI.getMF();
1591 if (
auto *LdSt = dyn_cast<GLoadStore>(&
UseMI)) {
1593 MF.getFunction().getContext());
1598 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;
1603 AMOld.
BaseOffs = MaybeImmVal->Value.getSExtValue();
1606 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1607 if (TLI.isLegalAddressingMode(MF.getDataLayout(), AMOld, AccessTy, AS) &&
1608 !TLI.isLegalAddressingMode(MF.getDataLayout(), AMNew, AccessTy, AS))
1621 assert(
MI.getOpcode() == TargetOpcode::G_PTR_ADD &&
"Expected G_PTR_ADD");
1627 MI.getOperand(1).setReg(MatchInfo.
Base);
1628 MI.getOperand(2).setReg(NewOffset.getReg(0));
1641 unsigned Opcode =
MI.getOpcode();
1642 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1643 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1644 Opcode == TargetOpcode::G_USHLSAT) &&
1645 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1665 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();
1670 if (Opcode == TargetOpcode::G_USHLSAT &&
1679 unsigned Opcode =
MI.getOpcode();
1680 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1681 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1682 Opcode == TargetOpcode::G_USHLSAT) &&
1683 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1687 auto Imm = MatchInfo.
Imm;
1689 if (Imm >= ScalarSizeInBits) {
1691 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {
1693 MI.eraseFromParent();
1698 Imm = ScalarSizeInBits - 1;
1704 MI.getOperand(1).setReg(MatchInfo.
Reg);
1705 MI.getOperand(2).setReg(NewImm);
1721 unsigned ShiftOpcode =
MI.getOpcode();
1722 assert((ShiftOpcode == TargetOpcode::G_SHL ||
1723 ShiftOpcode == TargetOpcode::G_ASHR ||
1724 ShiftOpcode == TargetOpcode::G_LSHR ||
1725 ShiftOpcode == TargetOpcode::G_USHLSAT ||
1726 ShiftOpcode == TargetOpcode::G_SSHLSAT) &&
1727 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
1730 Register LogicDest =
MI.getOperand(1).getReg();
1735 unsigned LogicOpcode = LogicMI->
getOpcode();
1736 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&
1737 LogicOpcode != TargetOpcode::G_XOR)
1741 const Register C1 =
MI.getOperand(2).getReg();
1743 if (!MaybeImmVal || MaybeImmVal->Value == 0)
1746 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();
1750 if (
MI->getOpcode() != ShiftOpcode ||
1760 ShiftVal = MaybeImmVal->Value.getSExtValue();
1771 if (matchFirstShift(LogicMIOp1, C0Val)) {
1773 MatchInfo.
Shift2 = LogicMIOp1;
1774 }
else if (matchFirstShift(LogicMIOp2, C0Val)) {
1776 MatchInfo.
Shift2 = LogicMIOp2;
1780 MatchInfo.
ValSum = C0Val + C1Val;
1786 MatchInfo.
Logic = LogicMI;
1792 unsigned Opcode =
MI.getOpcode();
1793 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1794 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_USHLSAT ||
1795 Opcode == TargetOpcode::G_SSHLSAT) &&
1796 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
1814 Register Shift2Const =
MI.getOperand(2).getReg();
1826 MI.eraseFromParent();
1830 assert(
MI.getOpcode() == TargetOpcode::G_SHL &&
"Expected G_SHL");
1833 auto &Shl = cast<GenericMachineInstr>(
MI);
1853 assert((SrcDef->getOpcode() == TargetOpcode::G_ADD ||
1854 SrcDef->getOpcode() == TargetOpcode::G_OR) &&
"Unexpected op");
1857 auto S1 =
B.buildShl(SrcTy,
X, ShiftReg);
1858 auto S2 =
B.buildShl(SrcTy, C1, ShiftReg);
1859 B.buildInstr(SrcDef->getOpcode(), {DstReg}, {
S1, S2});
1865 unsigned &ShiftVal) {
1866 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
"Expected a G_MUL");
1872 ShiftVal = MaybeImmVal->Value.exactLogBase2();
1873 return (
static_cast<int32_t
>(ShiftVal) != -1);
1877 unsigned &ShiftVal) {
1878 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
"Expected a G_MUL");
1883 MI.setDesc(MIB.
getTII().
get(TargetOpcode::G_SHL));
1884 MI.getOperand(2).setReg(ShiftCst.getReg(0));
1891 assert(
MI.getOpcode() == TargetOpcode::G_SHL &&
KB);
1906 if (!MaybeShiftAmtVal)
1920 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();
1921 MatchData.
Reg = ExtSrc;
1922 MatchData.
Imm = ShiftAmt;
1926 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;
1932 int64_t ShiftAmtVal = MatchData.
Imm;
1939 MI.eraseFromParent();
1946 for (
unsigned I = 0;
I <
Merge.getNumSources(); ++
I)
1949 auto *Unmerge = getOpcodeDef<GUnmerge>(MergedValues[0],
MRI);
1950 if (!Unmerge || Unmerge->getNumDefs() !=
Merge.getNumSources())
1953 for (
unsigned I = 0;
I < MergedValues.
size(); ++
I)
1954 if (MergedValues[
I] != Unmerge->getReg(
I))
1957 MatchInfo = Unmerge->getSourceReg();
1971 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
1972 "Expected an unmerge");
1973 auto &Unmerge = cast<GUnmerge>(
MI);
1976 auto *SrcInstr = getOpcodeDef<GMergeLikeInstr>(SrcReg,
MRI);
1984 if (SrcMergeTy != Dst0Ty && !SameSize)
1988 for (
unsigned Idx = 0;
Idx < SrcInstr->getNumSources(); ++
Idx)
1995 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
1996 "Expected an unmerge");
1998 "Not enough operands to replace all defs");
1999 unsigned NumElems =
MI.getNumOperands() - 1;
2003 bool CanReuseInputDirectly = DstTy == SrcTy;
2004 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2016 if (CanReuseInputDirectly)
2021 MI.eraseFromParent();
2026 unsigned SrcIdx =
MI.getNumOperands() - 1;
2027 Register SrcReg =
MI.getOperand(SrcIdx).getReg();
2029 if (SrcInstr->
getOpcode() != TargetOpcode::G_CONSTANT &&
2030 SrcInstr->
getOpcode() != TargetOpcode::G_FCONSTANT)
2041 for (
unsigned Idx = 0;
Idx != SrcIdx; ++
Idx) {
2043 Val = Val.
lshr(ShiftAmt);
2051 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2052 "Expected an unmerge");
2054 "Not enough operands to replace all defs");
2055 unsigned NumElems =
MI.getNumOperands() - 1;
2056 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2061 MI.eraseFromParent();
2066 unsigned SrcIdx =
MI.getNumOperands() - 1;
2067 Register SrcReg =
MI.getOperand(SrcIdx).getReg();
2069 unsigned NumElems =
MI.getNumOperands() - 1;
2070 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2072 B.buildUndef(DstReg);
2079 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2080 "Expected an unmerge");
2085 for (
unsigned Idx = 1, EndIdx =
MI.getNumDefs();
Idx != EndIdx; ++
Idx) {
2093 Register SrcReg =
MI.getOperand(
MI.getNumDefs()).getReg();
2094 Register Dst0Reg =
MI.getOperand(0).getReg();
2096 MI.eraseFromParent();
2100 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2101 "Expected an unmerge");
2102 Register Dst0Reg =
MI.getOperand(0).getReg();
2109 Register SrcReg =
MI.getOperand(
MI.getNumDefs()).getReg();
2126 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2127 "Expected an unmerge");
2129 Register Dst0Reg =
MI.getOperand(0).getReg();
2134 "Expecting a G_ZEXT");
2144 "ZExt src doesn't fit in destination");
2149 for (
unsigned Idx = 1, EndIdx =
MI.getNumDefs();
Idx != EndIdx; ++
Idx) {
2154 MI.eraseFromParent();
2158 unsigned TargetShiftSize,
2159 unsigned &ShiftVal) {
2160 assert((
MI.getOpcode() == TargetOpcode::G_SHL ||
2161 MI.getOpcode() == TargetOpcode::G_LSHR ||
2162 MI.getOpcode() == TargetOpcode::G_ASHR) &&
"Expected a shift");
2170 if (
Size <= TargetShiftSize)
2178 ShiftVal = MaybeImmVal->Value.getSExtValue();
2179 return ShiftVal >=
Size / 2 && ShiftVal <
Size;
2183 const unsigned &ShiftVal) {
2188 unsigned HalfSize =
Size / 2;
2189 assert(ShiftVal >= HalfSize);
2194 unsigned NarrowShiftAmt = ShiftVal - HalfSize;
2196 if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
2197 Register Narrowed = Unmerge.getReg(1);
2204 if (NarrowShiftAmt != 0) {
2211 }
else if (
MI.getOpcode() == TargetOpcode::G_SHL) {
2212 Register Narrowed = Unmerge.getReg(0);
2217 if (NarrowShiftAmt != 0) {
2225 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
2227 HalfTy, Unmerge.getReg(1),
2230 if (ShiftVal == HalfSize) {
2234 }
else if (ShiftVal ==
Size - 1) {
2242 HalfTy, Unmerge.getReg(1),
2251 MI.eraseFromParent();
2255 unsigned TargetShiftAmount) {
2266 assert(
MI.getOpcode() == TargetOpcode::G_INTTOPTR &&
"Expected a G_INTTOPTR");
2275 assert(
MI.getOpcode() == TargetOpcode::G_INTTOPTR &&
"Expected a G_INTTOPTR");
2278 MI.eraseFromParent();
2282 assert(
MI.getOpcode() == TargetOpcode::G_PTRTOINT &&
"Expected a G_PTRTOINT");
2285 MI.eraseFromParent();
2290 assert(
MI.getOpcode() == TargetOpcode::G_ADD);
2297 PtrReg.second =
false;
2307 PtrReg.second =
true;
2319 const bool DoCommute = PtrReg.second;
2328 MI.eraseFromParent();
2333 auto &PtrAdd = cast<GPtrAdd>(
MI);
2344 NewCst += RHSCst->
sextOrTrunc(DstTy.getSizeInBits());
2354 auto &PtrAdd = cast<GPtrAdd>(
MI);
2358 PtrAdd.eraseFromParent();
2362 assert(
MI.getOpcode() == TargetOpcode::G_ANYEXT &&
"Expected a G_ANYEXT");
2367 SrcReg = OriginalSrcReg;
2374 assert(
MI.getOpcode() == TargetOpcode::G_ZEXT &&
"Expected a G_ZEXT");
2389 assert((
MI.getOpcode() == TargetOpcode::G_ANYEXT ||
2390 MI.getOpcode() == TargetOpcode::G_SEXT ||
2391 MI.getOpcode() == TargetOpcode::G_ZEXT) &&
2392 "Expected a G_[ASZ]EXT");
2396 SrcReg = OriginalSrcReg;
2399 unsigned Opc =
MI.getOpcode();
2401 if (Opc == SrcOpc ||
2402 (Opc == TargetOpcode::G_ANYEXT &&
2403 (SrcOpc == TargetOpcode::G_SEXT || SrcOpc == TargetOpcode::G_ZEXT)) ||
2404 (Opc == TargetOpcode::G_SEXT && SrcOpc == TargetOpcode::G_ZEXT)) {
2413 assert((
MI.getOpcode() == TargetOpcode::G_ANYEXT ||
2414 MI.getOpcode() == TargetOpcode::G_SEXT ||
2415 MI.getOpcode() == TargetOpcode::G_ZEXT) &&
2416 "Expected a G_[ASZ]EXT");
2418 Register Reg = std::get<0>(MatchInfo);
2419 unsigned SrcExtOp = std::get<1>(MatchInfo);
2422 if (
MI.getOpcode() == SrcExtOp) {
2424 MI.getOperand(1).setReg(Reg);
2432 if (
MI.getOpcode() == TargetOpcode::G_ANYEXT ||
2433 (
MI.getOpcode() == TargetOpcode::G_SEXT &&
2434 SrcExtOp == TargetOpcode::G_ZEXT)) {
2437 MI.eraseFromParent();
2443 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2447 if (SrcOpc == TargetOpcode::G_ANYEXT || SrcOpc == TargetOpcode::G_SEXT ||
2448 SrcOpc == TargetOpcode::G_ZEXT) {
2457 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2459 unsigned SrcExtOp = MatchInfo.second;
2463 if (SrcTy == DstTy) {
2464 MI.eraseFromParent();
2472 MI.eraseFromParent();
2480 if (ShiftSize > 32 && TruncSize < 32)
2494 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2511 case TargetOpcode::G_SHL: {
2520 case TargetOpcode::G_LSHR:
2521 case TargetOpcode::G_ASHR: {
2528 if (
User.getOpcode() == TargetOpcode::G_STORE)
2532 if (NewShiftTy == SrcTy)
2546 {NewShiftTy, TL.getPreferredShiftAmountTy(NewShiftTy)}}))
2549 MatchInfo = std::make_pair(SrcMI, NewShiftTy);
2556 LLT NewShiftTy = MatchInfo.second;
2570 if (NewShiftTy == DstTy)
2580 return MO.isReg() &&
2581 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
2587 return !MO.isReg() ||
2588 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
2593 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
2595 return all_of(Mask, [](
int Elt) {
return Elt < 0; });
2599 assert(
MI.getOpcode() == TargetOpcode::G_STORE);
2600 return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
MI.getOperand(0).getReg(),
2605 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
2606 return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
MI.getOperand(1).getReg(),
2611 assert((
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
2612 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&
2613 "Expected an insert/extract element op");
2616 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
2629 OpIdx = Cst->isZero() ? 3 : 2;
2674 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())
2701 return MO.isReg() && MO.getReg().isPhysical();
2711 return I1->isIdenticalTo(*I2);
2726 return I1->findRegisterDefOperandIdx(InstAndDef1->Reg) ==
2737 return MaybeCst && MaybeCst->getBitWidth() <= 64 &&
2738 MaybeCst->getSExtValue() ==
C;
2744 std::optional<FPValueAndVReg> MaybeCst;
2748 return MaybeCst->Value.isExactlyValue(
C);
2753 assert(
MI.getNumExplicitDefs() == 1 &&
"Expected one explicit def?");
2755 Register Replacement =
MI.getOperand(OpIdx).getReg();
2757 MI.eraseFromParent();
2763 assert(
MI.getNumExplicitDefs() == 1 &&
"Expected one explicit def?");
2766 MI.eraseFromParent();
2771 unsigned ConstIdx) {
2772 Register ConstReg =
MI.getOperand(ConstIdx).getReg();
2785 assert((
MI.getOpcode() == TargetOpcode::G_FSHL ||
2786 MI.getOpcode() == TargetOpcode::G_FSHR) &&
2787 "This is not a funnel shift operation");
2789 Register ConstReg =
MI.getOperand(3).getReg();
2794 assert((VRegAndVal) &&
"Value is not a constant");
2797 APInt NewConst = VRegAndVal->Value.
urem(
2802 MI.getOpcode(), {MI.getOperand(0)},
2803 {MI.getOperand(1), MI.getOperand(2), NewConstInstr.getReg(0)});
2805 MI.eraseFromParent();
2809 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
2830 return MO.
isReg() &&
2841 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2843 MI.eraseFromParent();
2847 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2849 MI.eraseFromParent();
2853 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2855 MI.eraseFromParent();
2860 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2862 MI.eraseFromParent();
2866 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2868 MI.eraseFromParent();
2875 Register &NewLHS = std::get<0>(MatchInfo);
2876 Register &NewRHS = std::get<1>(MatchInfo);
2884 NewLHS = MaybeNewLHS;
2893 assert(
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
2902 TargetOpcode::G_INSERT_VECTOR_ELT)
2908 MatchInfo.
resize(NumElts);
2912 if (IntImm >= NumElts || IntImm < 0)
2914 if (!MatchInfo[IntImm])
2915 MatchInfo[IntImm] = TmpReg;
2919 if (CurrInst->
getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
2921 if (TmpInst->
getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
2930 return TmpInst->
getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
2937 auto GetUndef = [&]() {
2944 for (
unsigned I = 0;
I < MatchInfo.
size(); ++
I) {
2946 MatchInfo[
I] = GetUndef();
2949 MI.eraseFromParent();
2955 std::tie(SubLHS, SubRHS) = MatchInfo;
2957 MI.eraseFromParent();
2968 unsigned LogicOpcode =
MI.getOpcode();
2969 assert(LogicOpcode == TargetOpcode::G_AND ||
2970 LogicOpcode == TargetOpcode::G_OR ||
2971 LogicOpcode == TargetOpcode::G_XOR);
2984 if (!LeftHandInst || !RightHandInst)
2986 unsigned HandOpcode = LeftHandInst->
getOpcode();
2987 if (HandOpcode != RightHandInst->
getOpcode())
2999 if (!XTy.
isValid() || XTy != YTy)
3004 switch (HandOpcode) {
3007 case TargetOpcode::G_ANYEXT:
3008 case TargetOpcode::G_SEXT:
3009 case TargetOpcode::G_ZEXT: {
3013 case TargetOpcode::G_AND:
3014 case TargetOpcode::G_ASHR:
3015 case TargetOpcode::G_LSHR:
3016 case TargetOpcode::G_SHL: {
3021 ExtraHandOpSrcReg = ZOp.
getReg();
3043 if (ExtraHandOpSrcReg.
isValid())
3055 "Expected at least one instr to build?");
3057 assert(InstrToBuild.Opcode &&
"Expected a valid opcode?");
3058 assert(InstrToBuild.OperandFns.size() &&
"Expected at least one operand?");
3060 for (
auto &OperandFn : InstrToBuild.OperandFns)
3063 MI.eraseFromParent();
3068 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
3069 int64_t ShlCst, AshrCst;
3075 if (ShlCst != AshrCst)
3078 {TargetOpcode::G_SEXT_INREG, {
MRI.
getType(Src)}}))
3080 MatchInfo = std::make_tuple(Src, ShlCst);
3086 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
3089 std::tie(Src, ShiftAmt) = MatchInfo;
3092 MI.eraseFromParent();
3098 assert(
MI.getOpcode() == TargetOpcode::G_AND);
3113 B.buildAnd(Dst, R,
B.buildConstant(Ty, C1 & C2));
3116 auto Zero =
B.buildConstant(Ty, 0);
3139 assert(
MI.getOpcode() == TargetOpcode::G_AND);
3156 (LHSBits.
Zero | RHSBits.
One).isAllOnes()) {
3163 (LHSBits.
One | RHSBits.
Zero).isAllOnes()) {
3179 assert(
MI.getOpcode() == TargetOpcode::G_OR);
3196 (LHSBits.
One | RHSBits.
Zero).isAllOnes()) {
3203 (LHSBits.
Zero | RHSBits.
One).isAllOnes()) {
3214 unsigned ExtBits =
MI.getOperand(2).getImm();
3220 int64_t Cst,
bool IsVector,
bool IsFP) {
3222 return (ScalarSizeBits == 1 && Cst == -1) ||
3228 assert(
MI.getOpcode() == TargetOpcode::G_XOR);
3248 for (
unsigned I = 0;
I < RegsToNegate.
size(); ++
I) {
3253 switch (Def->getOpcode()) {
3258 case TargetOpcode::G_ICMP:
3264 case TargetOpcode::G_FCMP:
3270 case TargetOpcode::G_AND:
3271 case TargetOpcode::G_OR:
3277 RegsToNegate.
push_back(Def->getOperand(1).getReg());
3278 RegsToNegate.
push_back(Def->getOperand(2).getReg());
3305 for (
Register Reg : RegsToNegate) {
3310 switch (Def->getOpcode()) {
3313 case TargetOpcode::G_ICMP:
3314 case TargetOpcode::G_FCMP: {
3321 case TargetOpcode::G_AND:
3324 case TargetOpcode::G_OR:
3332 MI.eraseFromParent();
3338 assert(
MI.getOpcode() == TargetOpcode::G_XOR);
3342 Register SharedReg =
MI.getOperand(2).getReg();
3363 return Y == SharedReg;
3370 std::tie(
X,
Y) = MatchInfo;
3374 MI.getOperand(1).setReg(Not->getOperand(0).getReg());
3375 MI.getOperand(2).setReg(
Y);
3380 auto &PtrAdd = cast<GPtrAdd>(
MI);
3381 Register DstReg = PtrAdd.getReg(0);
3390 return ConstVal && *ConstVal == 0;
3399 auto &PtrAdd = cast<GPtrAdd>(
MI);
3401 PtrAdd.eraseFromParent();
3408 Register Pow2Src1 =
MI.getOperand(2).getReg();
3415 MI.eraseFromParent();
3419 unsigned &SelectOpNo) {
3429 if (
Select->getOpcode() != TargetOpcode::G_SELECT ||
3431 OtherOperandReg =
LHS;
3434 if (
Select->getOpcode() != TargetOpcode::G_SELECT ||
3451 unsigned BinOpcode =
MI.getOpcode();
3456 bool CanFoldNonConst =
3457 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&
3462 if (CanFoldNonConst)
3473 const unsigned &SelectOperand) {
3484 unsigned BinOpcode =
MI.getOpcode();
3491 if (SelectOperand == 1) {
3505 MI.eraseFromParent();
3508std::optional<SmallVector<Register, 8>>
3509CombinerHelper::findCandidatesForLoadOrCombine(
const MachineInstr *Root)
const {
3510 assert(Root->
getOpcode() == TargetOpcode::G_OR &&
"Expected G_OR only!");
3539 const unsigned MaxIter =
3541 for (
unsigned Iter = 0; Iter < MaxIter; ++Iter) {
3550 return std::nullopt;
3566 if (RegsToVisit.
empty() || RegsToVisit.
size() % 2 != 0)
3567 return std::nullopt;
3579static std::optional<std::pair<GZExtLoad *, int64_t>>
3583 "Expected Reg to only have one non-debug use?");
3592 if (Shift % MemSizeInBits != 0)
3593 return std::nullopt;
3596 auto *Load = getOpcodeDef<GZExtLoad>(MaybeLoad,
MRI);
3598 return std::nullopt;
3600 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits)
3601 return std::nullopt;
3603 return std::make_pair(Load, Shift / MemSizeInBits);
3606std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
3607CombinerHelper::findLoadOffsetsForLoadOrCombine(
3641 for (
auto Reg : RegsToVisit) {
3646 return std::nullopt;
3649 std::tie(Load, DstPos) = *LoadAndPos;
3657 return std::nullopt;
3660 auto &LoadMMO =
Load->getMMO();
3664 return std::nullopt;
3671 LoadPtr =
Load->getOperand(1).getReg();
3677 return std::nullopt;
3684 if (BasePtr != LoadPtr)
3685 return std::nullopt;
3687 if (
Idx < LowestIdx) {
3689 LowestIdxLoad =
Load;
3697 return std::nullopt;
3705 if (!EarliestLoad ||
dominates(*Load, *EarliestLoad))
3706 EarliestLoad =
Load;
3707 if (!LatestLoad ||
dominates(*LatestLoad, *Load))
3714 "Expected to find a load for each register?");
3715 assert(EarliestLoad != LatestLoad && EarliestLoad &&
3716 LatestLoad &&
"Expected at least two loads?");
3725 const unsigned MaxIter = 20;
3731 if (
MI.isLoadFoldBarrier())
3732 return std::nullopt;
3733 if (Iter++ == MaxIter)
3734 return std::nullopt;
3737 return std::make_tuple(LowestIdxLoad, LowestIdx, LatestLoad);
3742 assert(
MI.getOpcode() == TargetOpcode::G_OR);
3762 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)
3766 auto RegsToVisit = findCandidatesForLoadOrCombine(&
MI);
3773 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();
3774 if (NarrowMemSizeInBits % 8 != 0)
3787 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(
3788 MemOffset2Idx, *RegsToVisit, NarrowMemSizeInBits);
3791 std::tie(LowestIdxLoad, LowestIdx, LatestLoad) = *MaybeLoadInfo;
3798 std::optional<bool> IsBigEndian =
isBigEndian(MemOffset2Idx, LowestIdx);
3801 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;
3813 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;
3814 const unsigned ZeroByteOffset =
3818 auto ZeroOffsetIdx = MemOffset2Idx.
find(ZeroByteOffset);
3819 if (ZeroOffsetIdx == MemOffset2Idx.
end() ||
3820 ZeroOffsetIdx->second != LowestIdx)
3844 MIB.setInstrAndDebugLoc(*LatestLoad);
3846 MIB.buildLoad(LoadDst,
Ptr, *NewMMO);
3848 MIB.buildBSwap(Dst, LoadDst);
3855 auto &
PHI = cast<GPhi>(
MI);
3868 case TargetOpcode::G_ANYEXT:
3870 case TargetOpcode::G_ZEXT:
3871 case TargetOpcode::G_SEXT:
3885 for (
unsigned I = 0;
I <
PHI.getNumIncomingValues(); ++
I) {
3888 case TargetOpcode::G_LOAD:
3889 case TargetOpcode::G_TRUNC:
3890 case TargetOpcode::G_SEXT:
3891 case TargetOpcode::G_ZEXT:
3892 case TargetOpcode::G_ANYEXT:
3893 case TargetOpcode::G_CONSTANT:
3897 if (InSrcs.
size() > 2)
3909 auto &
PHI = cast<GPhi>(
MI);
3918 for (
unsigned I = 0;
I <
PHI.getNumIncomingValues(); ++
I) {
3919 auto SrcReg =
PHI.getIncomingValue(
I);
3921 if (!SrcMIs.
insert(SrcMI))
3927 if (InsertPt !=
MBB->
end() && InsertPt->isPHI())
3933 OldToNewSrcMap[SrcMI] = NewExt;
3942 NewPhi.
addMBB(MO.getMBB());
3946 NewPhi.addUse(NewSrc->getOperand(0).getReg());
3954 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
3964 unsigned VecIdx = Cst->Value.getZExtValue();
3969 if (SrcVecMI->
getOpcode() == TargetOpcode::G_TRUNC) {
3973 if (SrcVecMI->
getOpcode() != TargetOpcode::G_BUILD_VECTOR &&
3974 SrcVecMI->
getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)
3994 if (ScalarTy != DstTy) {
3997 MI.eraseFromParent();
4006 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4029 if (II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
4034 unsigned Idx = Cst->getZExtValue();
4038 SrcDstPairs.emplace_back(
4039 std::make_pair(
MI.getOperand(
Idx + 1).getReg(), &II));
4042 return ExtractedElts.
all();
4048 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4049 for (
auto &Pair : SrcDstPairs) {
4050 auto *ExtMI = Pair.second;
4052 ExtMI->eraseFromParent();
4054 MI.eraseFromParent();
4060 MI.eraseFromParent();
4078 assert(
MI.getOpcode() == TargetOpcode::G_OR);
4084 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;
4085 unsigned FshOpc = 0;
4096 int64_t CstShlAmt, CstLShrAmt;
4099 CstShlAmt + CstLShrAmt ==
BitWidth) {
4100 FshOpc = TargetOpcode::G_FSHR;
4107 FshOpc = TargetOpcode::G_FSHL;
4113 FshOpc = TargetOpcode::G_FSHR;
4124 B.buildInstr(FshOpc, {Dst}, {ShlSrc, LShrSrc, Amt});
4131 unsigned Opc =
MI.getOpcode();
4132 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4137 unsigned RotateOpc =
4138 Opc == TargetOpcode::G_FSHL ? TargetOpcode::G_ROTL : TargetOpcode::G_ROTR;
4143 unsigned Opc =
MI.getOpcode();
4144 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4145 bool IsFSHL = Opc == TargetOpcode::G_FSHL;
4148 : TargetOpcode::G_ROTR));
4149 MI.removeOperand(2);
4155 assert(
MI.getOpcode() == TargetOpcode::G_ROTL ||
4156 MI.getOpcode() == TargetOpcode::G_ROTR);
4160 bool OutOfRange =
false;
4161 auto MatchOutOfRange = [Bitsize, &OutOfRange](
const Constant *
C) {
4162 if (
auto *CI = dyn_cast<ConstantInt>(
C))
4163 OutOfRange |= CI->getValue().uge(Bitsize);
4170 assert(
MI.getOpcode() == TargetOpcode::G_ROTL ||
4171 MI.getOpcode() == TargetOpcode::G_ROTR);
4179 MI.getOperand(2).setReg(Amt);
4184 int64_t &MatchInfo) {
4185 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
4189 std::optional<bool> KnownVal;
4238 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
4263 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)
4269 unsigned Op = TargetOpcode::COPY;
4270 if (DstSize != LHSSize)
4271 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;
4281 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4291 int64_t AndMaskBits;
4299 if (AndMaskBits & OrMaskBits)
4305 if (
MI.getOperand(1).getReg() == AndMaskReg)
4306 MI.getOperand(2).setReg(AndMaskReg);
4307 MI.getOperand(1).setReg(Src);
4316 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
4323 int64_t Width =
MI.getOperand(2).getImm();
4335 auto Cst1 =
B.buildConstant(ExtractTy, ShiftImm);
4336 auto Cst2 =
B.buildConstant(ExtractTy, Width);
4337 B.buildSbfx(Dst, ShiftSrc, Cst1, Cst2);
4345 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4352 int64_t AndImm, LSBImm;
4361 auto MaybeMask =
static_cast<uint64_t>(AndImm);
4362 if (MaybeMask & (MaybeMask + 1))
4371 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4372 auto LSBCst =
B.buildConstant(ExtractTy, LSBImm);
4373 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {ShiftSrc, LSBCst, WidthCst});
4380 const unsigned Opcode =
MI.getOpcode();
4381 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);
4383 const Register Dst =
MI.getOperand(0).getReg();
4385 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR
4386 ? TargetOpcode::G_SBFX
4387 : TargetOpcode::G_UBFX;
4408 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >=
Size)
4412 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)
4416 const int64_t Pos = ShrAmt - ShlAmt;
4417 const int64_t Width =
Size - ShrAmt;
4420 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4421 auto PosCst =
B.buildConstant(ExtractTy, Pos);
4422 B.buildInstr(ExtrOpcode, {Dst}, {ShlSrc, PosCst, WidthCst});
4429 const unsigned Opcode =
MI.getOpcode();
4430 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);
4432 const Register Dst =
MI.getOperand(0).getReg();
4449 if (ShrAmt < 0 || ShrAmt >=
Size)
4453 if (0 == (SMask >> ShrAmt)) {
4455 B.buildConstant(Dst, 0);
4462 UMask |= maskTrailingOnes<uint64_t>(ShrAmt);
4463 UMask &= maskTrailingOnes<uint64_t>(
Size);
4468 const int64_t Pos = ShrAmt;
4473 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt ==
Size)
4477 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4478 auto PosCst =
B.buildConstant(ExtractTy, Pos);
4479 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {AndSrc, PosCst, WidthCst});
4484bool CombinerHelper::reassociationCanBreakAddressingModePattern(
4486 auto &PtrAdd = cast<GPtrAdd>(
MI);
4488 Register Src1Reg = PtrAdd.getBaseReg();
4489 auto *Src1Def = getOpcodeDef<GPtrAdd>(Src1Reg,
MRI);
4493 Register Src2Reg = PtrAdd.getOffsetReg();
4505 const APInt &C1APIntVal = *C1;
4506 const APInt &C2APIntVal = *C2;
4507 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();
4513 unsigned ConvUseOpc = ConvUseMI->
getOpcode();
4514 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||
4515 ConvUseOpc == TargetOpcode::G_PTRTOINT) {
4522 auto *LdStMI = dyn_cast<GLoadStore>(ConvUseMI);
4533 PtrAdd.getMF()->getFunction().getContext());
4534 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();
4535 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,
4541 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,
4553 Register Src1Reg =
MI.getOperand(1).getReg();
4554 if (
RHS->getOpcode() != TargetOpcode::G_ADD)
4566 MI.getOperand(1).setReg(NewBase.getReg(0));
4567 MI.getOperand(2).setReg(
RHS->getOperand(2).getReg());
4570 return !reassociationCanBreakAddressingModePattern(
MI);
4580 std::optional<ValueAndVReg> LHSCstOff;
4585 auto *LHSPtrAdd = cast<GPtrAdd>(
LHS);
4590 LHSPtrAdd->moveBefore(&
MI);
4593 auto NewCst =
B.buildConstant(
MRI.
getType(RHSReg), LHSCstOff->Value);
4595 MI.getOperand(2).setReg(NewCst.getReg(0));
4598 LHSPtrAdd->getOperand(2).setReg(RHSReg);
4601 return !reassociationCanBreakAddressingModePattern(
MI);
4609 auto *LHSPtrAdd = dyn_cast<GPtrAdd>(
LHS);
4613 Register Src2Reg =
MI.getOperand(2).getReg();
4614 Register LHSSrc1 = LHSPtrAdd->getBaseReg();
4615 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();
4624 auto NewCst =
B.buildConstant(
MRI.
getType(Src2Reg), *C1 + *C2);
4626 MI.getOperand(1).setReg(LHSSrc1);
4627 MI.getOperand(2).setReg(NewCst.getReg(0));
4630 return !reassociationCanBreakAddressingModePattern(
MI);
4635 auto &PtrAdd = cast<GPtrAdd>(
MI);
4687 auto NewCst =
B.buildInstr(Opc, {OpRHSTy}, {OpLHSRHS, OpRHS});
4688 B.buildInstr(Opc, {DstReg}, {OpLHSLHS, NewCst});
4696 auto NewLHSLHS =
B.buildInstr(Opc, {OpRHSTy}, {OpLHSLHS, OpRHS});
4697 B.buildInstr(Opc, {DstReg}, {NewLHSLHS, OpLHSRHS});
4710 unsigned Opc =
MI.getOpcode();
4727 MatchInfo = *MaybeCst;
4740 MatchInfo = *MaybeCst;
4751 ConstantFP::get(
MI.getMF()->getFunction().getContext(), *MaybeCst);
4757 assert(
MI.getOpcode() == TargetOpcode::G_FMA ||
4758 MI.getOpcode() == TargetOpcode::G_FMAD);
4759 auto [
_, Op1, Op2, Op3] =
MI.getFirst4Regs();
4776 MatchInfo = ConstantFP::get(
MI.getMF()->getFunction().getContext(), Op1F);
4798 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4822 case TargetOpcode::G_ADD:
4823 case TargetOpcode::G_SUB:
4824 case TargetOpcode::G_MUL:
4825 case TargetOpcode::G_AND:
4826 case TargetOpcode::G_OR:
4827 case TargetOpcode::G_XOR:
4835 auto Mask = Cst->Value;
4840 unsigned NarrowWidth = Mask.countr_one();
4846 auto &MF = *
MI.getMF();
4849 auto &
DL = MF.getDataLayout();
4850 if (!TLI.isTruncateFree(WideTy, NarrowTy,
DL, Ctx) ||
4851 !TLI.isZExtFree(NarrowTy, WideTy,
DL, Ctx))
4865 MI.getOperand(1).setReg(Ext.getReg(0));
4872 unsigned Opc =
MI.getOpcode();
4873 assert(Opc == TargetOpcode::G_UMULO || Opc == TargetOpcode::G_SMULO);
4880 unsigned NewOpc = Opc == TargetOpcode::G_UMULO ? TargetOpcode::G_UADDO
4881 : TargetOpcode::G_SADDO;
4883 MI.getOperand(3).setReg(
MI.getOperand(2).getReg());
4891 assert(
MI.getOpcode() == TargetOpcode::G_UMULO ||
4892 MI.getOpcode() == TargetOpcode::G_SMULO);
4901 B.buildConstant(Dst, 0);
4902 B.buildConstant(Carry, 0);
4910 assert(
MI.getOpcode() == TargetOpcode::G_UADDE ||
4911 MI.getOpcode() == TargetOpcode::G_SADDE ||
4912 MI.getOpcode() == TargetOpcode::G_USUBE ||
4913 MI.getOpcode() == TargetOpcode::G_SSUBE);
4918 switch (
MI.getOpcode()) {
4919 case TargetOpcode::G_UADDE:
4920 NewOpcode = TargetOpcode::G_UADDO;
4922 case TargetOpcode::G_SADDE:
4923 NewOpcode = TargetOpcode::G_SADDO;
4925 case TargetOpcode::G_USUBE:
4926 NewOpcode = TargetOpcode::G_USUBO;
4928 case TargetOpcode::G_SSUBE:
4929 NewOpcode = TargetOpcode::G_SSUBO;
4933 MI.setDesc(
B.getTII().get(NewOpcode));
4934 MI.removeOperand(4);
4942 assert(
MI.getOpcode() == TargetOpcode::G_SUB);
4976 B.buildSub(Dst, Zero, ReplaceReg);
4985 assert(
MI.getOpcode() == TargetOpcode::G_UDIV);
4986 auto &UDiv = cast<GenericMachineInstr>(
MI);
4997 bool UseNPQ =
false;
5000 auto BuildUDIVPattern = [&](
const Constant *
C) {
5001 auto *CI = cast<ConstantInt>(
C);
5002 const APInt &Divisor = CI->getValue();
5004 bool SelNPQ =
false;
5006 unsigned PreShift = 0, PostShift = 0;
5011 if (!Divisor.
isOne()) {
5015 Magic = std::move(magics.
Magic);
5018 "We shouldn't generate an undefined shift!");
5020 "We shouldn't generate an undefined shift!");
5024 SelNPQ = magics.
IsAdd;
5028 MIB.buildConstant(ScalarShiftAmtTy, PreShift).getReg(0));
5029 MagicFactors.
push_back(MIB.buildConstant(ScalarTy, Magic).getReg(0));
5031 MIB.buildConstant(ScalarTy,
5036 MIB.buildConstant(ScalarShiftAmtTy, PostShift).getReg(0));
5044 assert(Matched &&
"Expected unary predicate match to succeed");
5046 Register PreShift, PostShift, MagicFactor, NPQFactor;
5047 auto *RHSDef = getOpcodeDef<GBuildVector>(
RHS,
MRI);
5049 PreShift = MIB.buildBuildVector(ShiftAmtTy, PreShifts).getReg(0);
5050 MagicFactor = MIB.buildBuildVector(Ty, MagicFactors).getReg(0);
5051 NPQFactor = MIB.buildBuildVector(Ty, NPQFactors).getReg(0);
5052 PostShift = MIB.buildBuildVector(ShiftAmtTy, PostShifts).getReg(0);
5055 "Non-build_vector operation should have been a scalar");
5056 PreShift = PreShifts[0];
5057 MagicFactor = MagicFactors[0];
5058 PostShift = PostShifts[0];
5062 Q = MIB.buildLShr(Ty, Q, PreShift).getReg(0);
5065 Q = MIB.buildUMulH(Ty, Q, MagicFactor).getReg(0);
5068 Register NPQ = MIB.buildSub(Ty,
LHS, Q).getReg(0);
5073 NPQ = MIB.buildUMulH(Ty, NPQ, NPQFactor).getReg(0);
5075 NPQ = MIB.buildLShr(Ty, NPQ, MIB.buildConstant(ShiftAmtTy, 1)).getReg(0);
5077 Q = MIB.buildAdd(Ty, NPQ, Q).getReg(0);
5080 Q = MIB.buildLShr(Ty, Q, PostShift).getReg(0);
5081 auto One = MIB.buildConstant(Ty, 1);
5082 auto IsOne = MIB.buildICmp(
5085 return MIB.buildSelect(Ty, IsOne,
LHS, Q);
5089 assert(
MI.getOpcode() == TargetOpcode::G_UDIV);
5097 auto &MF = *
MI.getMF();
5101 auto &
DL = MF.getDataLayout();
5107 if (MF.getFunction().hasMinSize())
5117 {TargetOpcode::G_ICMP,
5123 auto CheckEltValue = [&](
const Constant *
C) {
5124 if (
auto *CI = dyn_cast_or_null<ConstantInt>(
C))
5125 return !CI->isZero();
5137 assert(
MI.getOpcode() == TargetOpcode::G_SDIV &&
"Expected SDIV");
5142 auto &MF = *
MI.getMF();
5146 auto &
DL = MF.getDataLayout();
5152 if (MF.getFunction().hasMinSize())
5171 assert(
MI.getOpcode() == TargetOpcode::G_SDIV &&
"Expected SDIV");
5172 auto &SDiv = cast<GenericMachineInstr>(
MI);
5182 bool UseSRA =
false;
5188 auto BuildSDIVPattern = [&](
const Constant *
C) {
5190 if (IsSplat && !Factors.
empty()) {
5196 auto *CI = cast<ConstantInt>(
C);
5197 APInt Divisor = CI->getValue();
5207 Shifts.
push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));
5208 Factors.
push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));
5215 assert(Matched &&
"Expected unary predicate match to succeed");
5219 Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);
5220 Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);
5223 Factor = Factors[0];
5231 return MIB.buildMul(Ty, Res, Factor);
5235 assert((
MI.getOpcode() == TargetOpcode::G_SDIV ||
5236 MI.getOpcode() == TargetOpcode::G_UDIV) &&
5237 "Expected SDIV or UDIV");
5238 auto &Div = cast<GenericMachineInstr>(
MI);
5240 auto MatchPow2 = [&](
const Constant *
C) {
5241 auto *CI = dyn_cast<ConstantInt>(
C);
5242 return CI && (CI->getValue().isPowerOf2() ||
5243 (IsSigned && CI->getValue().isNegatedPowerOf2()));
5249 assert(
MI.getOpcode() == TargetOpcode::G_SDIV &&
"Expected SDIV");
5250 auto &SDiv = cast<GenericMachineInstr>(
MI);
5304 MI.eraseFromParent();
5308 assert(
MI.getOpcode() == TargetOpcode::G_UDIV &&
"Expected UDIV");
5309 auto &UDiv = cast<GenericMachineInstr>(
MI);
5318 MI.eraseFromParent();
5322 assert(
MI.getOpcode() == TargetOpcode::G_UMULH);
5327 auto MatchPow2ExceptOne = [&](
const Constant *
C) {
5328 if (
auto *CI = dyn_cast<ConstantInt>(
C))
5329 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();
5350 MI.eraseFromParent();
5355 unsigned Opc =
MI.getOpcode();
5356 assert(Opc == TargetOpcode::G_FADD || Opc == TargetOpcode::G_FSUB ||
5357 Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
5358 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA);
5370 Opc = TargetOpcode::G_FSUB;
5375 Opc = TargetOpcode::G_FADD;
5381 else if ((Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
5382 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA) &&
5391 MI.setDesc(
B.getTII().get(Opc));
5392 MI.getOperand(1).setReg(
X);
5393 MI.getOperand(2).setReg(
Y);
5400 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5403 MatchInfo =
MI.getOperand(2).getReg();
5413 if (LHSCst->Value.isNegZero())
5417 if (LHSCst->Value.isPosZero())
5433 if (
MI.getOpcode() != TargetOpcode::G_FMUL)
5441 MRI.use_instr_nodbg_end()) >
5443 MRI.use_instr_nodbg_end());
5447 bool &AllowFusionGlobally,
5449 bool CanReassociate) {
5451 auto *MF =
MI.getMF();
5452 const auto &TLI = *MF->getSubtarget().getTargetLowering();
5456 if (CanReassociate &&
5463 bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(*MF, DstType) &&
5466 if (!HasFMAD && !HasFMA)
5470 Options.UnsafeFPMath || HasFMAD;
5475 Aggressive = TLI.enableAggressiveFMAFusion(DstType);
5481 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5483 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5491 unsigned PreferredFusedOpcode =
5492 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5506 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5507 {
LHS.MI->getOperand(1).getReg(),
5508 LHS.MI->getOperand(2).getReg(),
RHS.Reg});
5517 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5518 {
RHS.MI->getOperand(1).getReg(),
5519 RHS.MI->getOperand(2).getReg(),
LHS.Reg});
5529 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5531 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5535 const auto &TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
5542 unsigned PreferredFusedOpcode =
5543 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5557 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5562 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5563 {FpExtX.getReg(0), FpExtY.getReg(0),
RHS.Reg});
5572 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5577 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5578 {FpExtX.getReg(0), FpExtY.getReg(0),
LHS.Reg});
5588 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5590 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5600 unsigned PreferredFusedOpcode =
5601 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5614 if (
LHS.MI->getOpcode() == PreferredFusedOpcode &&
5616 TargetOpcode::G_FMUL) &&
5623 else if (
RHS.MI->getOpcode() == PreferredFusedOpcode &&
5625 TargetOpcode::G_FMUL) &&
5634 Register X = FMA->getOperand(1).getReg();
5635 Register Y = FMA->getOperand(2).getReg();
5641 B.buildInstr(PreferredFusedOpcode, {InnerFMA}, {U, V, Z});
5642 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5653 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5655 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5662 const auto &TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
5669 unsigned PreferredFusedOpcode =
5670 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5683 Register FpExtU =
B.buildFPExt(DstType, U).getReg(0);
5684 Register FpExtV =
B.buildFPExt(DstType, V).getReg(0);
5686 B.buildInstr(PreferredFusedOpcode, {DstType}, {FpExtU, FpExtV, Z})
5688 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5695 if (
LHS.MI->getOpcode() == PreferredFusedOpcode &&
5699 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5704 LHS.MI->getOperand(1).getReg(),
5705 LHS.MI->getOperand(2).getReg(),
B);
5716 FMAMI->
getOpcode() == PreferredFusedOpcode) {
5719 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5724 X =
B.buildFPExt(DstType,
X).getReg(0);
5725 Y =
B.buildFPExt(DstType,
Y).getReg(0);
5736 if (
RHS.MI->getOpcode() == PreferredFusedOpcode &&
5740 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5745 RHS.MI->getOperand(1).getReg(),
5746 RHS.MI->getOperand(2).getReg(),
B);
5757 FMAMI->
getOpcode() == PreferredFusedOpcode) {
5760 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5765 X =
B.buildFPExt(DstType,
X).getReg(0);
5766 Y =
B.buildFPExt(DstType,
Y).getReg(0);
5779 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5781 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5793 int FirstMulHasFewerUses =
true;
5797 FirstMulHasFewerUses =
false;
5799 unsigned PreferredFusedOpcode =
5800 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5803 if (FirstMulHasFewerUses &&
5808 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5809 {
LHS.MI->getOperand(1).getReg(),
5810 LHS.MI->getOperand(2).getReg(), NegZ});
5819 B.buildFNeg(DstTy,
RHS.MI->getOperand(1).getReg()).
getReg(0);
5820 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5821 {NegY,
RHS.MI->getOperand(2).getReg(),
LHS.Reg});
5831 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5833 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5841 unsigned PreferredFusedOpcode =
5842 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5853 Register NegZ =
B.buildFNeg(DstTy, RHSReg).getReg(0);
5854 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5866 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5878 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5880 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5888 unsigned PreferredFusedOpcode =
5889 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5901 Register NegZ =
B.buildFNeg(DstTy, RHSReg).getReg(0);
5902 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5903 {FpExtX, FpExtY, NegZ});
5915 Register NegY =
B.buildFNeg(DstTy, FpExtY).getReg(0);
5918 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5919 {NegY, FpExtZ, LHSReg});
5929 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5931 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5935 const auto &TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
5940 unsigned PreferredFusedOpcode =
5941 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5945 Register FpExtX =
B.buildFPExt(DstTy,
X).getReg(0);
5946 Register FpExtY =
B.buildFPExt(DstTy,
Y).getReg(0);
5947 B.buildInstr(PreferredFusedOpcode, {Dst}, {FpExtX, FpExtY, Z});
5958 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstTy,
5964 B.buildFNeg(
MI.getOperand(0).getReg(), FMAReg);
5974 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstTy,
5987 unsigned &IdxToPropagate) {
5989 switch (
MI.getOpcode()) {
5992 case TargetOpcode::G_FMINNUM:
5993 case TargetOpcode::G_FMAXNUM:
5994 PropagateNaN =
false;
5996 case TargetOpcode::G_FMINIMUM:
5997 case TargetOpcode::G_FMAXIMUM:
5998 PropagateNaN =
true;
6002 auto MatchNaN = [&](
unsigned Idx) {
6007 IdxToPropagate = PropagateNaN ?
Idx : (
Idx == 1 ? 2 : 1);
6011 return MatchNaN(1) || MatchNaN(2);
6015 assert(
MI.getOpcode() == TargetOpcode::G_ADD &&
"Expected a G_ADD");
6025 Reg == MaybeSameReg;
6060 std::optional<ValueAndVReg> ShiftAmount;
6091 std::optional<ValueAndVReg> ShiftAmt;
6098 return ShiftAmt->Value.getZExtValue() == MatchTy.
getSizeInBits() &&
6102unsigned CombinerHelper::getFPMinMaxOpcForSelect(
6104 SelectPatternNaNBehaviour VsNaNRetVal)
const {
6105 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&
6106 "Expected a NaN behaviour?");
6116 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
6117 return TargetOpcode::G_FMAXNUM;
6118 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
6119 return TargetOpcode::G_FMAXIMUM;
6120 if (
isLegal({TargetOpcode::G_FMAXNUM, {DstTy}}))
6121 return TargetOpcode::G_FMAXNUM;
6122 if (
isLegal({TargetOpcode::G_FMAXIMUM, {DstTy}}))
6123 return TargetOpcode::G_FMAXIMUM;
6129 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
6130 return TargetOpcode::G_FMINNUM;
6131 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
6132 return TargetOpcode::G_FMINIMUM;
6133 if (
isLegal({TargetOpcode::G_FMINNUM, {DstTy}}))
6134 return TargetOpcode::G_FMINNUM;
6135 if (!
isLegal({TargetOpcode::G_FMINIMUM, {DstTy}}))
6137 return TargetOpcode::G_FMINIMUM;
6141CombinerHelper::SelectPatternNaNBehaviour
6143 bool IsOrderedComparison)
const {
6147 if (!LHSSafe && !RHSSafe)
6148 return SelectPatternNaNBehaviour::NOT_APPLICABLE;
6149 if (LHSSafe && RHSSafe)
6150 return SelectPatternNaNBehaviour::RETURNS_ANY;
6153 if (IsOrderedComparison)
6154 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN
6155 : SelectPatternNaNBehaviour::RETURNS_OTHER;
6158 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER
6159 : SelectPatternNaNBehaviour::RETURNS_NAN;
6181 SelectPatternNaNBehaviour ResWithKnownNaNInfo =
6183 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)
6185 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
6188 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)
6189 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;
6190 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)
6191 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;
6193 if (TrueVal != CmpLHS || FalseVal != CmpRHS)
6196 unsigned Opc = getFPMinMaxOpcForSelect(Pred, DstTy, ResWithKnownNaNInfo);
6197 if (!Opc || !
isLegal({Opc, {DstTy}}))
6201 if (Opc != TargetOpcode::G_FMAXIMUM && Opc != TargetOpcode::G_FMINIMUM) {
6206 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {
6208 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())
6213 B.buildInstr(Opc, {Dst}, {CmpLHS, CmpRHS});
6221 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
6228 Register TrueVal =
MI.getOperand(2).getReg();
6229 Register FalseVal =
MI.getOperand(3).getReg();
6230 return matchFPSelectToMinMax(Dst,
Cond, TrueVal, FalseVal, MatchInfo);
6235 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
6248 if (MatchedSub &&
X != OpLHS)
6256 Y =
X == OpLHS ? OpRHS :
X == OpRHS ? OpLHS :
Register();
6260 B.buildICmp(Pred, Dst,
Y, Zero);
6266 Register ShiftReg =
MI.getOperand(2).getReg();
6268 auto IsShiftTooBig = [&](
const Constant *
C) {
6269 auto *CI = dyn_cast<ConstantInt>(
C);
6276 unsigned LHSOpndIdx = 1;
6277 unsigned RHSOpndIdx = 2;
6278 switch (
MI.getOpcode()) {
6279 case TargetOpcode::G_UADDO:
6280 case TargetOpcode::G_SADDO:
6281 case TargetOpcode::G_UMULO:
6282 case TargetOpcode::G_SMULO:
6296 TargetOpcode::G_CONSTANT_FOLD_BARRIER)
6301 TargetOpcode::G_CONSTANT_FOLD_BARRIER &&
6308 std::optional<FPValueAndVReg> ValAndVReg;
6316 unsigned LHSOpndIdx = 1;
6317 unsigned RHSOpndIdx = 2;
6318 switch (
MI.getOpcode()) {
6319 case TargetOpcode::G_UADDO:
6320 case TargetOpcode::G_SADDO:
6321 case TargetOpcode::G_UMULO:
6322 case TargetOpcode::G_SMULO:
6329 Register LHSReg =
MI.getOperand(LHSOpndIdx).getReg();
6330 Register RHSReg =
MI.getOperand(RHSOpndIdx).getReg();
6331 MI.getOperand(LHSOpndIdx).setReg(RHSReg);
6332 MI.getOperand(RHSOpndIdx).setReg(LHSReg);
6336bool CombinerHelper::isOneOrOneSplat(
Register Src,
bool AllowUndefs) {
6339 return isConstantSplatVector(Src, 1, AllowUndefs);
6341 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src,
MRI) !=
nullptr)
6344 return IConstant && IConstant->Value == 1;
6349bool CombinerHelper::isZeroOrZeroSplat(
Register Src,
bool AllowUndefs) {
6352 return isConstantSplatVector(Src, 0, AllowUndefs);
6354 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src,
MRI) !=
nullptr)
6357 return IConstant && IConstant->Value == 0;
6364bool CombinerHelper::isConstantSplatVector(
Register Src, int64_t SplatValue,
6371 for (
unsigned I = 0;
I < NumSources; ++
I) {
6374 if (ImplicitDef && AllowUndefs)
6376 if (ImplicitDef && !AllowUndefs)
6378 std::optional<ValueAndVReg> IConstant =
6380 if (IConstant && IConstant->Value == SplatValue)
6390CombinerHelper::getConstantOrConstantSplatVector(
Register Src) {
6393 return IConstant->Value;
6397 return std::nullopt;
6400 std::optional<APInt>
Value = std::nullopt;
6401 for (
unsigned I = 0;
I < NumSources; ++
I) {
6402 std::optional<ValueAndVReg> IConstant =
6405 return std::nullopt;
6409 return std::nullopt;
6415bool CombinerHelper::isConstantOrConstantVectorI(
Register Src)
const {
6425 for (
unsigned I = 0;
I < NumSources; ++
I) {
6426 std::optional<ValueAndVReg> IConstant =
6435bool CombinerHelper::tryFoldSelectOfConstants(
GSelect *
Select,
6453 std::optional<ValueAndVReg> TrueOpt =
6455 std::optional<ValueAndVReg> FalseOpt =
6458 if (!TrueOpt || !FalseOpt)
6461 APInt TrueValue = TrueOpt->Value;
6462 APInt FalseValue = FalseOpt->Value;
6467 B.setInstrAndDebugLoc(*
Select);
6468 B.buildZExtOrTrunc(Dest,
Cond);
6476 B.setInstrAndDebugLoc(*
Select);
6477 B.buildSExtOrTrunc(Dest,
Cond);
6485 B.setInstrAndDebugLoc(*
Select);
6487 B.buildNot(Inner,
Cond);
6488 B.buildZExtOrTrunc(Dest, Inner);
6496 B.setInstrAndDebugLoc(*
Select);
6498 B.buildNot(Inner,
Cond);
6499 B.buildSExtOrTrunc(Dest, Inner);
6505 if (TrueValue - 1 == FalseValue) {
6507 B.setInstrAndDebugLoc(*
Select);
6509 B.buildZExtOrTrunc(Inner,
Cond);
6510 B.buildAdd(Dest, Inner, False);
6516 if (TrueValue + 1 == FalseValue) {
6518 B.setInstrAndDebugLoc(*
Select);
6520 B.buildSExtOrTrunc(Inner,
Cond);
6521 B.buildAdd(Dest, Inner, False);
6529 B.setInstrAndDebugLoc(*
Select);
6531 B.buildZExtOrTrunc(Inner,
Cond);
6534 auto ShAmtC =
B.buildConstant(ShiftTy, TrueValue.
exactLogBase2());
6535 B.buildShl(Dest, Inner, ShAmtC, Flags);
6542 B.setInstrAndDebugLoc(*
Select);
6544 B.buildSExtOrTrunc(Inner,
Cond);
6545 B.buildOr(Dest, Inner, False, Flags);
6553 B.setInstrAndDebugLoc(*
Select);
6555 B.buildNot(Not,
Cond);
6557 B.buildSExtOrTrunc(Inner, Not);
6558 B.buildOr(Dest, Inner, True, Flags);
6567bool CombinerHelper::tryFoldBoolSelectToLogic(
GSelect *
Select,
6584 if (CondTy != TrueTy)
6589 if ((
Cond == True) || isOneOrOneSplat(True,
true)) {
6591 B.setInstrAndDebugLoc(*
Select);
6593 B.buildZExtOrTrunc(Ext,
Cond);
6594 auto FreezeFalse =
B.buildFreeze(TrueTy, False);
6595 B.buildOr(DstReg, Ext, FreezeFalse, Flags);
6602 if ((
Cond == False) || isZeroOrZeroSplat(False,
true)) {
6604 B.setInstrAndDebugLoc(*
Select);
6606 B.buildZExtOrTrunc(Ext,
Cond);
6607 auto FreezeTrue =
B.buildFreeze(TrueTy, True);
6608 B.buildAnd(DstReg, Ext, FreezeTrue);
6614 if (isOneOrOneSplat(False,
true)) {
6616 B.setInstrAndDebugLoc(*
Select);
6619 B.buildNot(Inner,
Cond);
6622 B.buildZExtOrTrunc(Ext, Inner);
6623 auto FreezeTrue =
B.buildFreeze(TrueTy, True);
6624 B.buildOr(DstReg, Ext, FreezeTrue, Flags);
6630 if (isZeroOrZeroSplat(True,
true)) {
6632 B.setInstrAndDebugLoc(*
Select);
6635 B.buildNot(Inner,
Cond);
6638 B.buildZExtOrTrunc(Ext, Inner);
6639 auto FreezeFalse =
B.buildFreeze(TrueTy, False);
6640 B.buildAnd(DstReg, Ext, FreezeFalse);
6648bool CombinerHelper::tryFoldSelectToIntMinMax(
GSelect *
Select,
6678 if (True == CmpRHS && False == CmpLHS) {
6686 if (True == CmpLHS && False == CmpRHS) {
6693 B.buildUMax(DstReg, True, False);
6702 B.buildSMax(DstReg, True, False);
6711 B.buildUMin(DstReg, True, False);
6720 B.buildSMin(DstReg, True, False);
6735 if (tryFoldSelectOfConstants(
Select, MatchInfo))
6738 if (tryFoldBoolSelectToLogic(
Select, MatchInfo))
6741 if (tryFoldSelectToIntMinMax(
Select, MatchInfo))
6751bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(
GLogicalBinOp *Logic,
6753 assert(Logic->
getOpcode() != TargetOpcode::G_XOR &&
"unexpected xor");
6754 bool IsAnd = Logic->
getOpcode() == TargetOpcode::G_AND;
6758 unsigned Flags = Logic->
getFlags();
6761 GICmp *Cmp1 = getOpcodeDef<GICmp>(LHS,
MRI);
6766 GICmp *Cmp2 = getOpcodeDef<GICmp>(RHS,
MRI);
6777 std::optional<ValueAndVReg> MaybeC1 =
6781 C1 = MaybeC1->Value;
6783 std::optional<ValueAndVReg> MaybeC2 =
6787 C2 = MaybeC2->Value;
6808 std::optional<APInt> Offset1;
6809 std::optional<APInt> Offset2;
6811 if (
GAdd *
Add = getOpcodeDef<GAdd>(R1,
MRI)) {
6812 std::optional<ValueAndVReg> MaybeOffset1 =
6815 R1 =
Add->getLHSReg();
6816 Offset1 = MaybeOffset1->Value;
6820 std::optional<ValueAndVReg> MaybeOffset2 =
6823 R2 =
Add->getLHSReg();
6824 Offset2 = MaybeOffset2->Value;
6843 bool CreateMask =
false;
6856 if (!LowerDiff.
isPowerOf2() || LowerDiff != UpperDiff ||
6869 CR->getEquivalentICmp(NewPred, NewC,
Offset);
6879 if (CreateMask &&
Offset != 0) {
6880 auto TildeLowerDiff =
B.buildConstant(CmpOperandTy, ~LowerDiff);
6881 auto And =
B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);
6882 auto OffsetC =
B.buildConstant(CmpOperandTy,
Offset);
6883 auto Add =
B.buildAdd(CmpOperandTy,
And, OffsetC, Flags);
6884 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6885 auto ICmp =
B.buildICmp(NewPred, CmpTy,
Add, NewCon);
6886 B.buildZExtOrTrunc(DstReg, ICmp);
6887 }
else if (CreateMask &&
Offset == 0) {
6888 auto TildeLowerDiff =
B.buildConstant(CmpOperandTy, ~LowerDiff);
6889 auto And =
B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);
6890 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6891 auto ICmp =
B.buildICmp(NewPred, CmpTy,
And, NewCon);
6892 B.buildZExtOrTrunc(DstReg, ICmp);
6893 }
else if (!CreateMask &&
Offset != 0) {
6894 auto OffsetC =
B.buildConstant(CmpOperandTy,
Offset);
6895 auto Add =
B.buildAdd(CmpOperandTy, R1, OffsetC, Flags);
6896 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6897 auto ICmp =
B.buildICmp(NewPred, CmpTy,
Add, NewCon);
6898 B.buildZExtOrTrunc(DstReg, ICmp);
6899 }
else if (!CreateMask &&
Offset == 0) {
6900 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6901 auto ICmp =
B.buildICmp(NewPred, CmpTy, R1, NewCon);
6902 B.buildZExtOrTrunc(DstReg, ICmp);
6910bool CombinerHelper::tryFoldLogicOfFCmps(
GLogicalBinOp *Logic,
6916 bool IsAnd = Logic->
getOpcode() == TargetOpcode::G_AND;
6919 GFCmp *Cmp1 = getOpcodeDef<GFCmp>(LHS,
MRI);
6924 GFCmp *Cmp2 = getOpcodeDef<GFCmp>(RHS,
MRI);
6934 {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||
6948 if (LHS0 == RHS1 && LHS1 == RHS0) {
6954 if (LHS0 == RHS0 && LHS1 == RHS1) {
6958 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;
6965 auto False =
B.buildConstant(CmpTy, 0);
6966 B.buildZExtOrTrunc(DestReg, False);
6973 B.buildZExtOrTrunc(DestReg, True);
6975 auto Cmp =
B.buildFCmp(Pred, CmpTy, LHS0, LHS1, Flags);
6976 B.buildZExtOrTrunc(DestReg, Cmp);
6988 if (tryFoldAndOrOrICmpsUsingRanges(
And, MatchInfo))
6991 if (tryFoldLogicOfFCmps(
And, MatchInfo))
7000 if (tryFoldAndOrOrICmpsUsingRanges(
Or, MatchInfo))
7003 if (tryFoldLogicOfFCmps(
Or, MatchInfo))
7017 bool IsSigned =
Add->isSigned();
7026 B.buildUndef(Carry);
7032 if (isConstantOrConstantVectorI(
LHS) && !isConstantOrConstantVectorI(
RHS)) {
7035 B.buildSAddo(Dst, Carry,
RHS,
LHS);
7041 B.buildUAddo(Dst, Carry,
RHS,
LHS);
7046 std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(
LHS);
7047 std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(
RHS);
7053 APInt Result = IsSigned ? MaybeLHS->sadd_ov(*MaybeRHS, Overflow)
7054 : MaybeLHS->uadd_ov(*MaybeRHS, Overflow);
7056 B.buildConstant(Dst, Result);
7057 B.buildConstant(Carry, Overflow);
7065 B.buildCopy(Dst,
LHS);
7066 B.buildConstant(Carry, 0);
7078 std::optional<APInt> MaybeAddRHS =
7079 getConstantOrConstantSplatVector(AddLHS->
getRHSReg());
7082 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(*MaybeRHS, Overflow)
7083 : MaybeAddRHS->uadd_ov(*MaybeRHS, Overflow);
7087 auto ConstRHS =
B.buildConstant(DstTy, NewC);
7088 B.buildSAddo(Dst, Carry, AddLHS->
getLHSReg(), ConstRHS);
7094 auto ConstRHS =
B.buildConstant(DstTy, NewC);
7095 B.buildUAddo(Dst, Carry, AddLHS->
getLHSReg(), ConstRHS);
7120 B.buildConstant(Carry, 0);
7128 B.buildConstant(Carry, 1);
7143 B.buildConstant(Carry, 0);
7159 B.buildConstant(Carry, 0);
7167 B.buildConstant(Carry, 1);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1, const MachineRegisterInfo &MRI)
static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally)
Checks if MI is TargetOpcode::G_FMUL and contractable either due to global flags or MachineInstr flag...
static unsigned getIndexedOpc(unsigned LdStOpc)
static APFloat constantFoldFpUnary(const MachineInstr &MI, const MachineRegisterInfo &MRI, const APFloat &Val)
static std::optional< std::pair< GZExtLoad *, int64_t > > matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits, const MachineRegisterInfo &MRI)
Helper function for findLoadOffsetsForLoadOrCombine.
static Register peekThroughBitcast(Register Reg, const MachineRegisterInfo &MRI)
static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I)
static cl::opt< bool > ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(false), cl::desc("Force all indexed operations to be " "legal for the GlobalISel combiner"))
static cl::opt< unsigned > PostIndexUseThreshold("post-index-use-threshold", cl::Hidden, cl::init(32), cl::desc("Number of uses of a base pointer to check before it is no longer " "considered for post-indexing."))
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static unsigned getExtLoadOpcForExtend(unsigned ExtOpc)
static Type * getTypeForLLT(LLT Ty, LLVMContext &C)
static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits, int64_t Cst, bool IsVector, bool IsFP)
static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy)
static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI, MachineRegisterInfo &MRI)
Return true if 'MI' is a load or a store that may be fold it's address operand into the load / store ...
static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I)
static Register buildLogBase2(Register V, MachineIRBuilder &MIB)
Determines the LogBase2 value for a non-null input value using the transform: LogBase2(V) = (EltBits ...
This contains common combine transformations that may be used in a combine pass,or by the target else...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Rewrite Partial Register Uses
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Interface for Targets to specify which operations they can successfully select and how the others sho...
Implement a low-level type suitable for MachineInstr level instruction selection.
Contains matchers for matching SSA Machine Instructions.
mir Rename Register Operands
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const SmallVectorImpl< MachineOperand > & Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file describes how to lower LLVM code to machine code.
const fltSemantics & getSemantics() const
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
int32_t exactLogBase2() const
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
unsigned countr_zero() const
Count the number of trailing zero bits.
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
unsigned countl_one() const
Count the number of leading one bits.
APInt multiplicativeInverse() const
bool isMask(unsigned numBits) const
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
bool isOne() const
Determine if this is a value of 1.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
AttributeSet getAttributes(unsigned Index) const
The attributes for the specified index are returned.
bool isEquality() const
Determine if this is an equals/not equals predicate.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ ICMP_SGE
signed greater or equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
static bool isOrdered(Predicate predicate)
Determine if the predicate is an ordered operation.
void applyUDivByConst(MachineInstr &MI)
void applyCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal)
bool matchCombineShuffleVector(MachineInstr &MI, SmallVectorImpl< Register > &Ops)
Check if the G_SHUFFLE_VECTOR MI can be replaced by a concat_vectors.
bool matchPtrAddZero(MachineInstr &MI)
}
bool matchAllExplicitUsesAreUndef(MachineInstr &MI)
Return true if all register explicit use operands on MI are defined by a G_IMPLICIT_DEF.
void replaceSingleDefInstWithOperand(MachineInstr &MI, unsigned OpIdx)
Delete MI and replace all of its uses with its OpIdx-th operand.
const RegisterBank * getRegBank(Register Reg) const
Get the register bank of Reg.
bool matchReassocPtrAdd(MachineInstr &MI, BuildFnTy &MatchInfo)
Reassociate pointer calculations with G_ADD involved, to allow better addressing mode usage.
bool matchUDivByConst(MachineInstr &MI)
Combine G_UDIV by constant into a multiply by magic constant.
void applyExtractVecEltBuildVec(MachineInstr &MI, Register &Reg)
bool matchInsertExtractVecEltOutOfBounds(MachineInstr &MI)
Return true if a G_{EXTRACT,INSERT}_VECTOR_ELT has an out of range index.
bool matchShiftsTooBig(MachineInstr &MI)
Match shifts greater or equal to the bitwidth of the operation.
bool tryCombineCopy(MachineInstr &MI)
If MI is COPY, try to combine it.
bool matchTruncLshrBuildVectorFold(MachineInstr &MI, Register &MatchInfo)
bool matchUndefStore(MachineInstr &MI)
Return true if a G_STORE instruction MI is storing an undef value.
bool matchRedundantBinOpInEquality(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform: (X + Y) == X -> Y == 0 (X - Y) == X -> Y == 0 (X ^ Y) == X -> Y == 0 (X + Y) !...
bool matchRedundantSExtInReg(MachineInstr &MI)
bool matchCombineFAddFpExtFMulToFMadOrFMAAggressive(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchReassocConstantInnerRHS(GPtrAdd &MI, MachineInstr *RHS, BuildFnTy &MatchInfo)
bool matchSubAddSameReg(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform: (x + y) - y -> x (x + y) - x -> y x - (y + x) -> 0 - y x - (x + z) -> 0 - z.
bool matchConstantFoldFPBinOp(MachineInstr &MI, ConstantFP *&MatchInfo)
Do constant FP folding when opportunities are exposed after MIR building.
void applyCombineShiftToUnmerge(MachineInstr &MI, const unsigned &ShiftVal)
void applyCombineUnmergeZExtToZExt(MachineInstr &MI)
void applyCommuteBinOpOperands(MachineInstr &MI)
bool matchBinOpSameVal(MachineInstr &MI)
Optimize (x op x) -> x.
void applyCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts)
bool matchCombineFSubFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z)) (fsub (fneg (fmul,...
bool matchCombineCopy(MachineInstr &MI)
bool matchConstantSelectCmp(MachineInstr &MI, unsigned &OpIdx)
Return true if a G_SELECT instruction MI has a constant comparison.
void eraseInst(MachineInstr &MI)
Erase MI.
void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const
MachineRegisterInfo::replaceRegWith() and inform the observer of the changes.
void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, Register ToReg) const
Replace a single register operand with a new register and inform the observer of the changes.
bool matchCombineFAddFMAFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z)) (fadd (fmad x,...
void applySimplifyAddToSub(MachineInstr &MI, std::tuple< Register, Register > &MatchInfo)
bool matchSimplifySelectToMinMax(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops)
If MI is G_CONCAT_VECTORS, try to combine it.
bool matchAddSubSameReg(MachineInstr &MI, Register &Src)
Transform G_ADD(x, G_SUB(y, x)) to y.
void applyRotateOutOfRange(MachineInstr &MI)
bool matchMulOBy2(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: (G_UMULO x, 2) -> (G_UADDO x, x) (G_SMULO x, 2) -> (G_SADDO x, x)
bool matchRotateOutOfRange(MachineInstr &MI)
void applyCombineConstPtrAddToI2P(MachineInstr &MI, APInt &NewCst)
void applyCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo)
void applyCombineShuffleVector(MachineInstr &MI, const ArrayRef< Register > Ops)
Replace MI with a concat_vectors with Ops.
const TargetLowering & getTargetLowering() const
void applyBuildFnNoErase(MachineInstr &MI, BuildFnTy &MatchInfo)
Use a function which takes in a MachineIRBuilder to perform a combine.
void applyPtrAddZero(MachineInstr &MI)
bool matchTruncBuildVectorFold(MachineInstr &MI, Register &MatchInfo)
void setRegBank(Register Reg, const RegisterBank *RegBank)
Set the register bank of Reg.
bool matchRedundantAnd(MachineInstr &MI, Register &Replacement)
void replaceInstWithConstant(MachineInstr &MI, int64_t C)
Replace an instruction with a G_CONSTANT with value C.
bool matchAshrShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo)
Match ashr (shl x, C), C -> sext_inreg (C)
bool tryCombineExtendingLoads(MachineInstr &MI)
If MI is extend that consumes the result of a load, try to combine it.
bool tryCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftAmount)
bool matchCombineUnmergeUndef(MachineInstr &MI, std::function< void(MachineIRBuilder &)> &MatchInfo)
Transform G_UNMERGE G_IMPLICIT_DEF -> G_IMPLICIT_DEF, G_IMPLICIT_DEF, ...
void applySDivByConst(MachineInstr &MI)
bool matchUndefSelectCmp(MachineInstr &MI)
Return true if a G_SELECT instruction MI has an undef comparison.
void replaceInstWithUndef(MachineInstr &MI)
Replace an instruction with a G_IMPLICIT_DEF.
bool matchRedundantOr(MachineInstr &MI, Register &Replacement)
bool matchOperandIsUndef(MachineInstr &MI, unsigned OpIdx)
Check if operand OpIdx is undef.
void applyBuildFn(MachineInstr &MI, BuildFnTy &MatchInfo)
Use a function which takes in a MachineIRBuilder to perform a combine.
bool matchCombineConstPtrAddToI2P(MachineInstr &MI, APInt &NewCst)
void replaceInstWithFConstant(MachineInstr &MI, double C)
Replace an instruction with a G_FCONSTANT with value C.
bool matchBitfieldExtractFromSExtInReg(MachineInstr &MI, BuildFnTy &MatchInfo)
Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
bool matchEqualDefs(const MachineOperand &MOP1, const MachineOperand &MOP2)
Return true if MOP1 and MOP2 are register operands are defined by equivalent instructions.
bool matchShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo)
Fold (shift (shift base, x), y) -> (shift base (x+y))
bool matchPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo)
void applyShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo)
void applyOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond)
bool matchMulOBy0(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: (G_*MULO x, 0) -> 0 + no carry out.
void replaceSingleDefInstWithReg(MachineInstr &MI, Register Replacement)
Delete MI and replace all of its uses with Replacement.
bool matchFunnelShiftToRotate(MachineInstr &MI)
Match an FSHL or FSHR that can be combined to a ROTR or ROTL rotate.
bool matchNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate)
Combine inverting a result of a compare into the opposite cond code.
void applyCombineExtOfExt(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
void replaceOpcodeWith(MachineInstr &FromMI, unsigned ToOpcode) const
Replace the opcode in instruction with a new opcode and inform the observer of the changes.
bool matchOperandIsKnownToBeAPowerOfTwo(MachineInstr &MI, unsigned OpIdx)
Check if operand OpIdx is known to be a power of 2.
void applyCombineCopy(MachineInstr &MI)
void applyCombineTruncOfExt(MachineInstr &MI, std::pair< Register, unsigned > &MatchInfo)
bool matchAnyExplicitUseIsUndef(MachineInstr &MI)
Return true if any explicit use operand on MI is defined by a G_IMPLICIT_DEF.
bool matchFsubToFneg(MachineInstr &MI, Register &MatchInfo)
void applyCombineAddP2IToPtrAdd(MachineInstr &MI, std::pair< Register, bool > &PtrRegAndCommute)
bool matchNarrowBinopFeedingAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
void applyCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops)
Replace MI with a flattened build_vector with Ops or an implicit_def if Ops is empty.
bool matchSextTruncSextLoad(MachineInstr &MI)
bool matchShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo)
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
bool matchExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo)
void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo)
MachineInstr * buildSDivUsingMul(MachineInstr &MI)
Given an G_SDIV MI expressing a signed divide by constant, return an expression that implements it by...
void applySDivByPow2(MachineInstr &MI)
void applyFunnelShiftConstantModulo(MachineInstr &MI)
Replaces the shift amount in MI with ShiftAmt % BW.
bool matchConstantFoldBinOp(MachineInstr &MI, APInt &MatchInfo)
Do constant folding when opportunities are exposed after MIR building.
bool isPreLegalize() const
bool matchCombineLoadWithAndMask(MachineInstr &MI, BuildFnTy &MatchInfo)
Match (and (load x), mask) -> zextload x.
bool matchConstantOp(const MachineOperand &MOP, int64_t C)
Return true if MOP is defined by a G_CONSTANT or splat with a value equal to C.
bool matchCombineFSubFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fmul x, y), z) -> (fma x, y, -z) (fsub (fmul x, y), z) -> (fmad x,...
bool matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine ands.
void applyCombineI2PToP2I(MachineInstr &MI, Register &Reg)
void applyNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate)
void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo)
bool matchConstantFPOp(const MachineOperand &MOP, double C)
Return true if MOP is defined by a G_FCONSTANT or splat with a value exactly equal to C.
bool matchSimplifyAddToSub(MachineInstr &MI, std::tuple< Register, Register > &MatchInfo)
Return true if MI is a G_ADD which can be simplified to a G_SUB.
bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0)
Optimize memcpy intrinsics et al, e.g.
bool matchSelectSameVal(MachineInstr &MI)
Optimize (cond ? x : x) -> x.
void applyCombineConstantFoldFpUnary(MachineInstr &MI, const ConstantFP *Cst)
Transform fp_instr(cst) to constant result of the fp operation.
bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo)
bool tryReassocBinOp(unsigned Opc, Register DstReg, Register Op0, Register Op1, BuildFnTy &MatchInfo)
Try to reassociate to reassociate operands of a commutative binop.
bool isConstantLegalOrBeforeLegalizer(const LLT Ty) const
bool tryEmitMemcpyInline(MachineInstr &MI)
Emit loads and stores that perform the given memcpy.
void applyXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo)
bool matchXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo)
Fold (xor (and x, y), y) -> (and (not x), y) {.
bool matchCombineFSubFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z)) (fsub (fpext (fmul x,...
bool matchCombineFMinMaxNaN(MachineInstr &MI, unsigned &Info)
bool matchCombineShlOfExtend(MachineInstr &MI, RegisterImmPair &MatchData)
bool matchConstantFoldFMA(MachineInstr &MI, ConstantFP *&MatchInfo)
Constant fold G_FMA/G_FMAD.
bool matchBitfieldExtractFromAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: and (lshr x, cst), mask -> ubfx x, cst, width.
void applyShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo)
bool isLegal(const LegalityQuery &Query) const
bool matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine selects.
bool matchCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts)
Transform G_UNMERGE Constant -> Constant1, Constant2, ...
bool matchICmpToTrueFalseKnownBits(MachineInstr &MI, int64_t &MatchInfo)
bool matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg)
Transform anyext(trunc(x)) to x.
void applySimplifyURemByPow2(MachineInstr &MI)
Combine G_UREM x, (known power of 2) to an add and bitmasking.
bool matchReassocFoldConstantsInSubTree(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo)
MachineRegisterInfo & MRI
void applyUMulHToLShr(MachineInstr &MI)
bool matchLoadOrCombine(MachineInstr &MI, BuildFnTy &MatchInfo)
Match expression trees of the form.
bool matchShuffleToExtract(MachineInstr &MI)
bool matchUndefShuffleVectorMask(MachineInstr &MI)
Return true if a G_SHUFFLE_VECTOR instruction MI has an undef mask.
bool isLegalOrBeforeLegalizer(const LegalityQuery &Query) const
bool matchExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI)
bool matchAndOrDisjointMask(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchCombineExtractedVectorLoad(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine a G_EXTRACT_VECTOR_ELT of a load into a narrowed load.
bool matchCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal)
Transform a multiply by a power-of-2 value to a left shift.
bool matchBitfieldExtractFromShr(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: shr (shl x, n), k -> sbfx/ubfx x, pos, width.
void applyFoldBinOpIntoSelect(MachineInstr &MI, const unsigned &SelectOpNo)
SelectOperand is the operand in binary operator MI that is the select to fold.
bool matchBuildVectorIdentityFold(MachineInstr &MI, Register &MatchInfo)
bool matchCombineFAddFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd (fmul x, y), z) -> (fma x, y, z) (fadd (fmul x, y), z) -> (fmad x,...
bool matchRedundantNegOperands(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd x, fneg(y)) -> (fsub x, y) (fadd fneg(x), y) -> (fsub y, x) (fsub x,...
bool matchCombineMergeUnmerge(MachineInstr &MI, Register &MatchInfo)
Fold away a merge of an unmerge of the corresponding values.
void applyCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo)
bool matchCombineUnmergeZExtToZExt(MachineInstr &MI)
Transform X, Y = G_UNMERGE(G_ZEXT(Z)) -> X = G_ZEXT(Z); Y = G_CONSTANT 0.
bool matchCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI)
Transform X, Y<dead> = G_UNMERGE Z -> X = G_TRUNC Z.
bool matchConstantLargerBitWidth(MachineInstr &MI, unsigned ConstIdx)
Checks if constant at ConstIdx is larger than MI 's bitwidth.
CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize, GISelKnownBits *KB=nullptr, MachineDominatorTree *MDT=nullptr, const LegalizerInfo *LI=nullptr)
bool matchCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI)
Try to combine G_[SU]DIV and G_[SU]REM into a single G_[SU]DIVREM when their source operands are iden...
bool matchCombineTruncOfExt(MachineInstr &MI, std::pair< Register, unsigned > &MatchInfo)
Transform trunc ([asz]ext x) to x or ([asz]ext x) or (trunc x).
bool isPredecessor(const MachineInstr &DefMI, const MachineInstr &UseMI)
Returns true if DefMI precedes UseMI or they are the same instruction.
bool matchDivByPow2(MachineInstr &MI, bool IsSigned)
Given an G_SDIV MI expressing a signed divided by a pow2 constant, return expressions that implements...
bool matchExtractVecEltBuildVec(MachineInstr &MI, Register &Reg)
bool matchUMulHToLShr(MachineInstr &MI)
bool dominates(const MachineInstr &DefMI, const MachineInstr &UseMI)
Returns true if DefMI dominates UseMI.
MachineInstr * buildUDivUsingMul(MachineInstr &MI)
Given an G_UDIV MI expressing a divide by constant, return an expression that implements it by multip...
bool matchCombineZextTrunc(MachineInstr &MI, Register &Reg)
Transform zext(trunc(x)) to x.
void applyCombineShlOfExtend(MachineInstr &MI, const RegisterImmPair &MatchData)
bool canCombineFMadOrFMA(MachineInstr &MI, bool &AllowFusionGlobally, bool &HasFMAD, bool &Aggressive, bool CanReassociate=false)
void applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI)
void applyShuffleToExtract(MachineInstr &MI)
MachineDominatorTree * MDT
bool matchSDivByConst(MachineInstr &MI)
void applySextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
bool matchCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands)
Transform <ty,...> G_UNMERGE(G_MERGE ty X, Y, Z) -> ty X, Y, Z.
void applyExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo)
void applyBuildFnMO(const MachineOperand &MO, BuildFnTy &MatchInfo)
Use a function which takes in a MachineIRBuilder to perform a combine.
bool matchCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo)
Transform trunc (shl x, K) to shl (trunc x), K if K < VT.getScalarSizeInBits().
const RegisterBankInfo * RBI
bool matchCommuteShift(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchCombineFAddFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z) (fadd (fpext (fmul x,...
bool matchReassocConstantInnerLHS(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo)
void applyExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI)
const TargetRegisterInfo * TRI
bool tryCombineShuffleVector(MachineInstr &MI)
Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS.
bool matchCombineI2PToP2I(MachineInstr &MI, Register &Reg)
Transform IntToPtr(PtrToInt(x)) to x if cast is in the same address space.
bool matchICmpToLHSKnownBits(MachineInstr &MI, BuildFnTy &MatchInfo)
GISelChangeObserver & Observer
bool matchCombineExtOfExt(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
Transform [asz]ext([asz]ext(x)) to [asz]ext x.
bool matchOverlappingAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Fold and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0.
bool matchSextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
Match sext_inreg(load p), imm -> sextload p.
bool matchCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo)
bool matchCombineAddP2IToPtrAdd(MachineInstr &MI, std::pair< Register, bool > &PtrRegAndCommute)
Transform G_ADD (G_PTRTOINT x), y -> G_PTRTOINT (G_PTR_ADD x, y) Transform G_ADD y,...
bool matchOr(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine ors.
void applyFunnelShiftToRotate(MachineInstr &MI)
void applyCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands)
bool matchOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond)
If a brcond's true block is not the fallthrough, make it so by inverting the condition and swapping o...
bool matchAddEToAddO(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: (G_*ADDE x, y, 0) -> (G_*ADDO x, y) (G_*SUBE x, y, 0) -> (G_*SUBO x, y)
bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine addos.
void applyCombineP2IToI2P(MachineInstr &MI, Register &Reg)
Transform PtrToInt(IntToPtr(x)) to x.
bool matchCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftSize, unsigned &ShiftVal)
Reduce a shift by a constant to an unmerge and a shift on a half sized type.
bool matchCommuteConstantToRHS(MachineInstr &MI)
Match constant LHS ops that should be commuted.
void applyPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo)
void applyCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI)
void applyFsubToFneg(MachineInstr &MI, Register &MatchInfo)
void applyBuildInstructionSteps(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo)
Replace MI with a series of instructions described in MatchInfo.
bool matchCombineFSubFpExtFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fpext (fneg (fmul x, y))), z) -> (fneg (fma (fpext x), (fpext y),...
MachineIRBuilder & Builder
bool matchBitfieldExtractFromShrAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: shr (and x, n), k -> ubfx x, pos, width.
bool matchReassocCommBinOp(MachineInstr &MI, BuildFnTy &MatchInfo)
Reassociate commutative binary operations like G_ADD.
bool matchFoldBinOpIntoSelect(MachineInstr &MI, unsigned &SelectOpNo)
Push a binary operator through a select on constants.
bool matchConstantFoldCastOp(MachineInstr &MI, APInt &MatchInfo)
Do constant folding when opportunities are exposed after MIR building.
bool matchOperandIsZero(MachineInstr &MI, unsigned OpIdx)
Check if operand OpIdx is zero.
bool matchOrShiftToFunnelShift(MachineInstr &MI, BuildFnTy &MatchInfo)
void applyUDivByPow2(MachineInstr &MI)
Given an G_UDIV MI expressing an unsigned divided by a pow2 constant, return expressions that impleme...
bool matchHoistLogicOpWithSameOpcodeHands(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo)
Match (logic_op (op x...), (op y...)) -> (op (logic_op x, y))
void applyAshShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo)
void applySextTruncSextLoad(MachineInstr &MI)
bool matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo)
bool matchCommuteFPConstantToRHS(MachineInstr &MI)
Match constant LHS FP ops that should be commuted.
ConstantFP - Floating Point Values [float, double].
const APFloat & getValue() const
const APFloat & getValueAPF() const
const APInt & getValue() const
Return the constant as an APInt value reference.
This class represents a range of values.
std::optional< ConstantRange > exactUnionWith(const ConstantRange &CR) const
Union the two ranges and return the result if it can be represented exactly, otherwise return std::nu...
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
const APInt & getLower() const
Return the lower value for this range.
OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
bool isWrappedSet() const
Return true if this set wraps around the unsigned domain.
const APInt & getUpper() const
Return the upper value for this range.
static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
OverflowResult signedAddMayOverflow(const ConstantRange &Other) const
Return whether signed add of the two ranges always/never overflows.
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Represents overflowing add operations.
Represents an integer addition.
Represents a logical and.
CmpInst::Predicate getCond() const
Register getLHSReg() const
Register getRHSReg() const
Represents any generic load, including sign/zero extending variants.
Register getDstReg() const
Get the definition register of the loaded value.
Register getLHSReg() const
Register getRHSReg() const
Represents a G_BUILD_VECTOR.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
void finishedChangingAllUsesOfReg()
All instructions reported as changing by changingAllUsesOfReg() have finished being changed.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
virtual void erasingInstr(MachineInstr &MI)=0
An instruction is about to be erased.
void changingAllUsesOfReg(const MachineRegisterInfo &MRI, Register Reg)
All the instructions using the given register are being changed.
unsigned computeNumSignBits(Register R, const APInt &DemandedElts, unsigned Depth=0)
KnownBits getKnownBits(Register R)
APInt getKnownZeroes(Register R)
Simple wrapper observer that takes several observers, and calls each one for each event.
Represents a G_IMPLICIT_DEF.
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
Represents a logical binary operation.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
bool isAtomic() const
Returns true if the attached MachineMemOperand has the atomic flag set.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
Register getSourceReg(unsigned I) const
Returns the I'th source register.
unsigned getNumSources() const
Returns the number of source registers.
Represents a G_MERGE_VALUES.
Register getCondReg() const
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
constexpr bool isScalableVector() const
Returns true if the LLT is a scalable vector.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
constexpr bool isByteSized() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
constexpr unsigned getAddressSpace() const
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
constexpr LLT getScalarType() const
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
This is an important class for using LLVM in a threaded context.
@ Legalized
Instruction has been legalized and the MachineFunction changed.
LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0)
Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index)
Get a pointer to vector element Index located in memory for a vector of type VecTy starting at a base...
bool isLegalOrCustom(const LegalityQuery &Query) const
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
TypeSize getValue() const
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
MachineInstrBuilder insertInstr(MachineInstrBuilder MIB)
Insert an existing instruction at the insertion point.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
LLVMContext & getContext() const
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildCTTZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTTZ Op0, Src0.
MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
const TargetInstrInfo & getTII()
MachineInstrBuilder buildURem(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_UREM Op0, Op1.
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntToPtr(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_INTTOPTR instruction.
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildNeg(const DstOp &Dst, const SrcOp &Src0)
Build and insert integer negation Zero = G_CONSTANT 0 Res = G_SUB Zero, Op0.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
virtual MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val)
Build and insert Res = G_FCONSTANT Val.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
MachineFunction & getMF()
Getter for the function we currently build.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op)
Build and insert Res = ExtOpc, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes of...
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FNEG Op0.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = <opcode> Addr, MMO.
MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_XOR Op0, Op1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
MachineInstrBuilder buildFCanonicalize(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_FCANONICALIZE Src0.
MachineInstrBuilder buildSExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and insert Res = G_SEXT_INREG Op, ImmOp.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
int findRegisterDefOperandIdx(Register Reg, bool isDead=false, bool Overlap=false, const TargetRegisterInfo *TRI=nullptr) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
bool isDereferenceableInvariantLoad() const
Return true if this load instruction never traps and points to a memory location whose value doesn't ...
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI)
Clone another MachineInstr's memory reference descriptor list and replace ours with it.
unsigned getNumOperands() const
Retuns the total number of operands.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
A description of a memory reference used in the backend.
LLT getMemoryType() const
Return the memory type of the memory reference.
unsigned getAddrSpace() const
const MachinePointerInfo & getPointerInfo() const
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setReg(Register Reg)
Change the register this operand corresponds to.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
void setMBB(MachineBasicBlock *MBB)
void setPredicate(unsigned Predicate)
Register getReg() const
getReg - Returns the register number.
const ConstantFP * getFPImm() const
unsigned getPredicate() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
use_instr_iterator use_instr_begin(Register RegNo) const
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
const RegClassOrRegBank & getRegClassOrRegBank(Register Reg) const
Return the register bank or register class of Reg.
void setRegClassOrRegBank(Register Reg, const RegClassOrRegBank &RCOrRB)
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
bool hasOneUse(Register RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
use_instr_nodbg_iterator use_instr_nodbg_begin(Register RegNo) const
void setRegBank(Register Reg, const RegisterBank &RegBank)
Set the register bank to RegBank for Reg.
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
Register cloneVirtualRegister(Register VReg, StringRef Name="")
Create and return a new virtual register in the function with the same attributes as the given regist...
bool constrainRegAttrs(Register Reg, Register ConstrainingReg, unsigned MinNumRegs=0)
Constrain the register class or the register bank of the virtual register Reg (and low-level type) to...
iterator_range< use_iterator > use_operands(Register Reg) const
void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
size_type size() const
Determine the number of elements in the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool all() const
Returns true if all bits are set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
virtual bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, MachineRegisterInfo &MRI) const
Given the generic extension instruction ExtMI, returns true if this extension is a likely candidate f...
virtual bool produceSameValue(const MachineInstr &MI0, const MachineInstr &MI1, const MachineRegisterInfo *MRI=nullptr) const
Return true if two machine instructions would produce identical values.
virtual LLVM_READONLY LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const
virtual const TargetLowering * getTargetLowering() const
The instances of the Type class are immutable: once they are created, they are never changed.
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Value(Type *Ty, unsigned scid)
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ Legal
The operation is expected to be selectable directly by the target, and no transformation is necessary...
operand_type_match m_Reg()
SpecificConstantOrSplatMatch m_SpecificICstOrSplat(int64_t RequestedValue)
Matches a RequestedValue constant or a constant splat of RequestedValue.
BinaryOp_match< LHS, RHS, TargetOpcode::G_BUILD_VECTOR, false > m_GBuildVector(const LHS &L, const RHS &R)
GCstAndRegMatch m_GCst(std::optional< ValueAndVReg > &ValReg)
SpecificConstantMatch m_SpecificICst(int64_t RequestedValue)
Matches a constant equal to RequestedValue.
operand_type_match m_Pred()
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_XOR, true > m_GXor(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_SEXT > m_GSExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_FPEXT > m_GFPExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
UnaryOp_match< SrcTy, TargetOpcode::G_INTTOPTR > m_GIntToPtr(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
ICstOrSplatMatch< APInt > m_ICstOrSplat(APInt &Cst)
ImplicitDefMatch m_GImplicitDef()
OneNonDBGUse_match< SubPat > m_OneNonDBGUse(const SubPat &SP)
CheckType m_SpecificType(LLT Ty)
BinaryOp_match< LHS, RHS, TargetOpcode::G_FADD, true > m_GFAdd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_PTRTOINT > m_GPtrToInt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_FSUB, false > m_GFSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ASHR, false > m_GAShr(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_BITCAST > m_GBitcast(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_BUILD_VECTOR_TRUNC, false > m_GBuildVectorTrunc(const LHS &L, const RHS &R)
bind_ty< MachineInstr * > m_MInstr(MachineInstr *&MI)
UnaryOp_match< SrcTy, TargetOpcode::G_FNEG > m_GFNeg(const SrcTy &Src)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP, true > m_c_GICmp(const Pred &P, const LHS &L, const RHS &R)
G_ICMP matcher that also matches commuted compares.
TernaryOp_match< Src0Ty, Src1Ty, Src2Ty, TargetOpcode::G_INSERT_VECTOR_ELT > m_GInsertVecElt(const Src0Ty &Src0, const Src1Ty &Src1, const Src2Ty &Src2)
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
And< Preds... > m_all_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_ANYEXT > m_GAnyExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_FCMP > m_GFCmp(const Pred &P, const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool isBuildVectorAllZeros(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
static double log2(double V)
const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
EVT getApproximateEVTForLLT(LLT Ty, const DataLayout &DL, LLVMContext &Ctx)
std::optional< APInt > getIConstantVRegVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT, return the corresponding value.
std::optional< APInt > getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI)
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
std::optional< APFloat > ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to have exactly one bit set when defined.
std::optional< APInt > isConstantOrConstantSplatVector(MachineInstr &MI, const MachineRegisterInfo &MRI)
Determines if MI defines a constant integer or a splat vector of constant integers.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg, std::function< bool(const Constant *ConstVal)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant G_B...
bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Returns true if given the TargetLowering's boolean contents information, the value Val contains a tru...
std::function< void(MachineIRBuilder &)> BuildFnTy
std::optional< APInt > ConstantFoldBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool isConstantOrConstantVector(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowFP=true, bool AllowOpaqueConstants=true)
Return true if the specified instruction is known to be a constant, or a vector of constants.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
bool canReplaceReg(Register DstReg, Register SrcReg, MachineRegisterInfo &MRI)
Check if DstReg can be replaced with SrcReg depending on the register constraints.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
std::optional< FPValueAndVReg > getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI, bool AllowUndef=true)
Returns a floating point scalar constant of a build vector splat if it exists.
std::optional< APInt > ConstantFoldCastOp(unsigned Opcode, LLT DstTy, const Register Op0, const MachineRegisterInfo &MRI)
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
std::optional< FPValueAndVReg > getFConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_FCONSTANT returns it...
constexpr unsigned BitWidth
int64_t getICmpTrueVal(const TargetLowering &TLI, bool IsVector, bool IsFP)
Returns an integer representing true, as defined by the TargetBooleanContents.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isKnownNeverNaN(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
unsigned getFCmpCode(CmpInst::Predicate CC)
Similar to getICmpCode but for FCmpInst.
std::optional< int64_t > getIConstantSplatSExtVal(const Register Reg, const MachineRegisterInfo &MRI)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
Simple struct used to hold a Register value and the instruction which defines it.
SmallVector< InstructionBuildSteps, 2 > InstrsToBuild
Describes instructions to be built during a combine.
static std::optional< bool > eq(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_EQ result.
static std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
static std::optional< bool > sge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGE result.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
static std::optional< bool > ugt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGT result.
static std::optional< bool > slt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLT result.
static std::optional< bool > ult(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULT result.
static std::optional< bool > ule(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULE result.
static std::optional< bool > sle(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLE result.
static std::optional< bool > sgt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGT result.
static std::optional< bool > uge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGE result.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
LegalizeAction Action
The action to take or the final answer.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
MachinePointerInfo getWithOffset(int64_t O) const
const RegisterBank * Bank
Register LogicNonShiftReg
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
Magic data for optimising unsigned division by a constant.
unsigned PreShift
pre-shift amount
static UnsignedDivisionByConstantInfo get(const APInt &D, unsigned LeadingZeros=0, bool AllowEvenDivisorOptimization=true)
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...
unsigned PostShift
post-shift amount