49#define DEBUG_TYPE "gi-combiner"
58 cl::desc(
"Force all indexed operations to be "
59 "legal for the GlobalISel combiner"));
68 TII(
Builder.getMF().getSubtarget().getInstrInfo()),
69 RBI(
Builder.getMF().getSubtarget().getRegBankInfo()),
70 TRI(
Builder.getMF().getSubtarget().getRegisterInfo()) {
75 return *
Builder.getMF().getSubtarget().getTargetLowering();
89 assert(
I < ByteWidth &&
"I must be in [0, ByteWidth)");
108 assert(
I < ByteWidth &&
"I must be in [0, ByteWidth)");
109 return ByteWidth -
I - 1;
129static std::optional<bool>
133 unsigned Width = MemOffset2Idx.
size();
136 bool BigEndian =
true, LittleEndian =
true;
137 for (
unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {
138 auto MemOffsetAndIdx = MemOffset2Idx.
find(MemOffset);
139 if (MemOffsetAndIdx == MemOffset2Idx.
end())
141 const int64_t Idx = MemOffsetAndIdx->second - LowestIdx;
142 assert(Idx >= 0 &&
"Expected non-negative byte offset?");
145 if (!BigEndian && !LittleEndian)
149 assert((BigEndian != LittleEndian) &&
150 "Pattern cannot be both big and little endian!");
157 assert(
LI &&
"Must have LegalizerInfo to query isLegal!");
178bool CombinerHelper::isImmConstantLegalOrBeforeLegalizer(
unsigned Opc,
179 const LLT Ty)
const {
188 return isLegal({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
193 return isImmConstantLegalOrBeforeLegalizer(TargetOpcode::G_CONSTANT, Ty);
197 return isImmConstantLegalOrBeforeLegalizer(TargetOpcode::G_FCONSTANT, Ty);
204 if (
MRI.constrainRegAttrs(ToReg, FromReg))
205 MRI.replaceRegWith(FromReg, ToReg);
207 Builder.buildCopy(FromReg, ToReg);
209 Observer.finishedChangingAllUsesOfReg();
224 unsigned ToOpcode)
const {
239 MRI.setRegBank(Reg, *RegBank);
243 if (
MI.getOpcode() != TargetOpcode::COPY)
253 MI.eraseFromParent();
258 assert(
MI.getOpcode() == TargetOpcode::G_FREEZE &&
"Invalid instruction");
264 if (!
MRI.hasOneNonDBGUse(OrigOp))
285 std::optional<MachineOperand> MaybePoisonOperand;
287 if (!Operand.isReg())
293 if (!MaybePoisonOperand)
294 MaybePoisonOperand = Operand;
303 if (!MaybePoisonOperand) {
308 B.buildCopy(
DstOp, OrigOp);
313 Register MaybePoisonOperandReg = MaybePoisonOperand->getReg();
314 LLT MaybePoisonOperandRegTy =
MRI.getType(MaybePoisonOperandReg);
317 {TargetOpcode::G_FREEZE, {MaybePoisonOperandRegTy}}))
325 auto Freeze =
B.buildFreeze(MaybePoisonOperandRegTy, MaybePoisonOperandReg);
336 assert(
MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
337 "Invalid instruction");
349 if (!
MRI.hasOneNonDBGUse(Reg))
351 switch (Def->getOpcode()) {
352 case TargetOpcode::G_BUILD_VECTOR:
357 Ops.push_back(BuildVecMO.getReg());
359 case TargetOpcode::G_IMPLICIT_DEF: {
360 LLT OpType =
MRI.getType(Reg);
367 OpType.getScalarType() &&
368 "All undefs should have the same type");
371 for (
unsigned EltIdx = 0, EltEnd = OpType.getNumElements();
372 EltIdx != EltEnd; ++EltIdx)
373 Ops.push_back(
Undef->getOperand(0).getReg());
382 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
384 {TargetOpcode::G_BUILD_VECTOR, {DstTy,
MRI.getType(
Ops[0])}})) {
399 Register NewDstReg =
MRI.cloneVirtualRegister(DstReg);
412 MI.eraseFromParent();
421 if (!Unmerge || Unmerge->
getReg(0) != BV.getSourceReg(0))
426 LLT InputTy =
MRI.getType(BCSrc);
428 if (!InputTy.
isScalar() || BV.getNumSources() % Factor != 0)
433 if (!
isLegal({TargetOpcode::G_BUILD_VECTOR, {BVDstTy, InputTy}}))
437 for (
unsigned Idx = 0; Idx < BV.getNumSources(); Idx += Factor) {
453 Ops.push_back(BCSrc);
474 auto BV =
Builder.buildBuildVector(BVDstTy,
Ops);
475 Builder.buildBitcast(
MI.getOperand(0).getReg(), BV);
476 MI.eraseFromParent();
482 Register SrcVec1 = Shuffle.getSrc1Reg();
483 Register SrcVec2 = Shuffle.getSrc2Reg();
484 LLT EltTy =
MRI.getType(SrcVec1).getElementType();
485 int Width =
MRI.getType(SrcVec1).getNumElements();
487 auto Unmerge1 =
Builder.buildUnmerge(EltTy, SrcVec1);
488 auto Unmerge2 =
Builder.buildUnmerge(EltTy, SrcVec2);
492 for (
int Val : Shuffle.getMask()) {
495 else if (Val < Width)
496 Extracts.
push_back(Unmerge1.getReg(Val));
498 Extracts.
push_back(Unmerge2.getReg(Val - Width));
500 assert(Extracts.
size() > 0 &&
"Expected at least one element in the shuffle");
501 if (Extracts.
size() == 1)
502 Builder.buildCopy(
MI.getOperand(0).getReg(), Extracts[0]);
504 Builder.buildBuildVector(
MI.getOperand(0).getReg(), Extracts);
505 MI.eraseFromParent();
510 Register SrcVec1 = Shuffle.getSrc1Reg();
511 Register SrcVec2 = Shuffle.getSrc2Reg();
514 assert((BuildMI1 && (BuildMI2 ||
MRI.getVRegDef(SrcVec2)->getOpcode() ==
515 TargetOpcode::G_IMPLICIT_DEF)) &&
516 "Unexpected input to applyCombineShuffleOfBuildVector");
517 LLT EltTy =
MRI.getType(SrcVec1).getElementType();
518 int NumElts =
MRI.getType(SrcVec1).getNumElements();
521 for (
int Val : Shuffle.getMask()) {
524 else if (Val < NumElts)
525 Extracts.
push_back(BuildMI1->getSourceReg(Val));
527 Extracts.
push_back(BuildMI2->getSourceReg(Val - NumElts));
532 if (Extracts.
size() == 1)
533 Builder.buildCopy(
MI.getOperand(0).getReg(), Extracts[0]);
535 Builder.buildBuildVector(
MI.getOperand(0).getReg(), Extracts);
536 MI.eraseFromParent();
553 LLT ShuffleSrcTy1 =
MRI.getType(
MI.getOperand(1).getReg());
555 for (
unsigned i = 0; i < Mask.size(); i += ConcatSrcNumElt) {
559 for (
unsigned j = 1; j < ConcatSrcNumElt; j++) {
560 if (i + j >= Mask.size())
562 if (Mask[i + j] != -1)
566 {TargetOpcode::G_IMPLICIT_DEF, {ConcatSrcTy}}))
569 }
else if (Mask[i] % ConcatSrcNumElt == 0) {
570 for (
unsigned j = 1; j < ConcatSrcNumElt; j++) {
571 if (i + j >= Mask.size())
573 if (Mask[i + j] != Mask[i] +
static_cast<int>(j))
590 {TargetOpcode::G_CONCAT_VECTORS,
591 {
MRI.getType(
MI.getOperand(0).getReg()), ConcatSrcTy}}))
602 SrcTy =
MRI.getType(Reg);
604 assert(SrcTy.isValid() &&
"Unexpected full undef vector in concat combine");
611 UndefReg =
Builder.buildUndef(SrcTy).getReg(0);
617 Builder.buildConcatVectors(
MI.getOperand(0).getReg(),
Ops);
620 MI.eraseFromParent();
625 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
626 "Invalid instruction kind");
627 LLT DstType =
MRI.getType(
MI.getOperand(0).getReg());
629 LLT SrcType =
MRI.getType(Src1);
631 unsigned DstNumElts = DstType.getNumElements();
632 unsigned SrcNumElts = SrcType.getNumElements();
649 if (DstNumElts < 2 * SrcNumElts)
654 if (DstNumElts % SrcNumElts != 0)
660 unsigned NumConcat = DstNumElts / SrcNumElts;
663 for (
unsigned i = 0; i != DstNumElts; ++i) {
670 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
671 (ConcatSrcs[i / SrcNumElts] >= 0 &&
672 ConcatSrcs[i / SrcNumElts] != (
int)(Idx / SrcNumElts)))
675 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
682 for (
auto Src : ConcatSrcs) {
686 UndefReg =
Builder.buildUndef(SrcType).getReg(0);
688 Ops.push_back(UndefReg);
701 Register NewDstReg =
MRI.cloneVirtualRegister(DstReg);
709 MI.eraseFromParent();
718 const LLT TyForCandidate,
719 unsigned OpcodeForCandidate,
724 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
735 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&
738 else if (CurrentUse.
ExtendOpcode == TargetOpcode::G_ANYEXT &&
739 OpcodeForCandidate != TargetOpcode::G_ANYEXT)
740 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
748 OpcodeForCandidate == TargetOpcode::G_ZEXT)
750 else if (CurrentUse.
ExtendOpcode == TargetOpcode::G_ZEXT &&
751 OpcodeForCandidate == TargetOpcode::G_SEXT)
752 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
761 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
772static void InsertInsnsWithoutSideEffectsBeforeUse(
784 InsertBB = PredBB->
getMBB();
789 if (InsertBB ==
DefMI.getParent()) {
791 Inserter(InsertBB, std::next(InsertPt), UseMO);
810 unsigned CandidateLoadOpc;
812 case TargetOpcode::G_ANYEXT:
813 CandidateLoadOpc = TargetOpcode::G_LOAD;
815 case TargetOpcode::G_SEXT:
816 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;
818 case TargetOpcode::G_ZEXT:
819 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;
824 return CandidateLoadOpc;
841 LLT LoadValueTy =
MRI.getType(LoadReg);
863 unsigned PreferredOpcode =
865 ? TargetOpcode::G_ANYEXT
867 Preferred = {
LLT(), PreferredOpcode,
nullptr};
868 for (
auto &
UseMI :
MRI.use_nodbg_instructions(LoadReg)) {
869 if (
UseMI.getOpcode() == TargetOpcode::G_SEXT ||
870 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
871 (
UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
872 const auto &MMO = LoadMI->
getMMO();
880 LLT UseTy =
MRI.getType(
UseMI.getOperand(0).getReg());
882 if (
LI->getAction({CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})
886 Preferred = ChoosePreferredUse(
MI, Preferred,
887 MRI.getType(
UseMI.getOperand(0).getReg()),
897 assert(Preferred.Ty != LoadValueTy &&
"Extending to same type?");
915 if (PreviouslyEmitted) {
922 Builder.setInsertPt(*InsertIntoBB, InsertBefore);
923 Register NewDstReg =
MRI.cloneVirtualRegister(
MI.getOperand(0).getReg());
925 EmittedInsns[InsertIntoBB] = NewMI;
931 MI.setDesc(
Builder.getTII().get(LoadOpc));
938 for (
auto *UseMO :
Uses) {
944 UseMI->getOpcode() == TargetOpcode::G_ANYEXT) {
947 const LLT UseDstTy =
MRI.getType(UseDstReg);
948 if (UseDstReg != ChosenDstReg) {
949 if (Preferred.
Ty == UseDstTy) {
986 InsertInsnsWithoutSideEffectsBeforeUse(
Builder,
MI, *UseMO,
1001 InsertInsnsWithoutSideEffectsBeforeUse(
Builder,
MI, *UseMO, InsertTruncAt);
1004 MI.getOperand(0).setReg(ChosenDstReg);
1010 assert(
MI.getOpcode() == TargetOpcode::G_AND);
1021 if (
MRI.getType(Dst).isVector())
1029 APInt MaskVal = MaybeMask->Value;
1044 LLT RegTy =
MRI.getType(LoadReg);
1047 unsigned MaskSizeBits = MaskVal.
countr_one();
1050 !
MRI.hasOneNonDBGUse(LoadReg))
1055 if (MaskSizeBits > LoadSizeBits)
1074 else if (LoadSizeBits > MaskSizeBits || LoadSizeBits ==
RegSize)
1079 {TargetOpcode::G_ZEXTLOAD, {RegTy,
MRI.getType(PtrReg)}, {MemDesc}}))
1083 B.setInstrAndDebugLoc(*LoadMI);
1084 auto &MF =
B.getMF();
1086 auto *NewMMO = MF.getMachineMemOperand(MMO, PtrInfo, MemDesc.
MemoryTy);
1087 B.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, Dst, PtrReg, *NewMMO);
1097 "shouldn't consider debug uses");
1105 if (DefOrUse ==
MBB.end())
1107 return &*DefOrUse == &
DefMI;
1113 "shouldn't consider debug uses");
1116 else if (
DefMI.getParent() !=
UseMI.getParent())
1123 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1127 if (
MRI.getType(SrcReg).isVector())
1132 LoadUser = TruncSrc;
1134 uint64_t SizeInBits =
MI.getOperand(2).getImm();
1139 auto LoadSizeBits = LoadMI->getMemSizeInBits();
1141 MRI.getType(TruncSrc).getSizeInBits() < LoadSizeBits.getValue())
1143 if (LoadSizeBits == SizeInBits)
1150 MachineInstr &
MI, std::tuple<Register, unsigned> &MatchInfo)
const {
1151 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1154 LLT RegTy =
MRI.getType(DstReg);
1167 uint64_t ExtFrom =
MI.getOperand(2).getImm();
1169 if (MemBits > ExtFrom && !
MRI.hasOneNonDBGUse(SrcReg))
1175 unsigned NewSizeBits = std::min(ExtFrom, MemBits);
1178 if (NewSizeBits < 8)
1191 else if (MemBits > NewSizeBits || MemBits == RegTy.
getSizeInBits())
1196 {TargetOpcode::G_SEXTLOAD, {RegTy,
MRI.getType(PtrReg)}, {MMDesc}}))
1199 MatchInfo = std::make_tuple(SrcReg, NewSizeBits);
1204 MachineInstr &
MI, std::tuple<Register, unsigned> &MatchInfo)
const {
1205 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1207 unsigned ScalarSizeBits;
1208 std::tie(LoadReg, ScalarSizeBits) = MatchInfo;
1217 auto &MMO = LoadDef->
getMMO();
1218 Builder.setInstrAndDebugLoc(*LoadDef);
1220 auto PtrInfo = MMO.getPointerInfo();
1221 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, ScalarSizeBits / 8);
1222 Builder.buildLoadInstr(TargetOpcode::G_SEXTLOAD,
MI.getOperand(0).getReg(),
1225 MI.eraseFromParent();
1236 auto *MF =
MI->getMF();
1243 AM.
BaseOffs = CstOff->getSExtValue();
1247 return TLI.isLegalAddressingMode(
1248 MF->getDataLayout(), AM,
1250 MF->getFunction().getContext()),
1251 MI->getMMO().getAddrSpace());
1256 case TargetOpcode::G_LOAD:
1257 return TargetOpcode::G_INDEXED_LOAD;
1258 case TargetOpcode::G_STORE:
1259 return TargetOpcode::G_INDEXED_STORE;
1260 case TargetOpcode::G_ZEXTLOAD:
1261 return TargetOpcode::G_INDEXED_ZEXTLOAD;
1262 case TargetOpcode::G_SEXTLOAD:
1263 return TargetOpcode::G_INDEXED_SEXTLOAD;
1269bool CombinerHelper::isIndexedLoadStoreLegal(
GLoadStore &LdSt)
const {
1279 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)
1280 OpTys = {PtrTy, Ty, Ty};
1282 OpTys = {Ty, PtrTy};
1284 LegalityQuery Q(IndexedOpc, OpTys, MemDescrs);
1290 cl::desc(
"Number of uses of a base pointer to check before it is no longer "
1291 "considered for post-indexing."));
1295 bool &RematOffset)
const {
1305 if (
MRI.hasOneNonDBGUse(Ptr))
1308 if (!isIndexedLoadStoreLegal(LdSt))
1319 unsigned NumUsesChecked = 0;
1320 for (
auto &
Use :
MRI.use_nodbg_instructions(Ptr)) {
1327 if (!PtrAdd ||
MRI.use_nodbg_empty(PtrAdd->getReg(0)))
1332 if (StoredValDef == &
Use)
1335 Offset = PtrAdd->getOffsetReg();
1337 !
TLI.isIndexingLegal(LdSt, PtrAdd->getBaseReg(),
Offset,
1345 RematOffset =
false;
1349 if (OffsetDef->
getOpcode() != TargetOpcode::G_CONSTANT)
1354 for (
auto &BasePtrUse :
MRI.use_nodbg_instructions(PtrAdd->getBaseReg())) {
1355 if (&BasePtrUse == PtrDef)
1361 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&
1363 isIndexedLoadStoreLegal(*BasePtrLdSt))
1369 Register PtrAddDefReg = BasePtrUseDef->getReg(0);
1370 for (
auto &BaseUseUse :
MRI.use_nodbg_instructions(PtrAddDefReg)) {
1373 if (BaseUseUse.getParent() != LdSt.
getParent())
1385 Addr = PtrAdd->getReg(0);
1386 Base = PtrAdd->getBaseReg();
1397 const auto &
TLI = *MF.getSubtarget().getTargetLowering();
1401 MRI.hasOneNonDBGUse(Addr))
1408 if (!isIndexedLoadStoreLegal(LdSt))
1412 if (BaseDef->
getOpcode() == TargetOpcode::G_FRAME_INDEX)
1417 if (
Base == St->getValueReg())
1422 if (St->getValueReg() == Addr)
1427 for (
auto &AddrUse :
MRI.use_nodbg_instructions(Addr))
1428 if (AddrUse.getParent() != LdSt.
getParent())
1433 bool RealUse =
false;
1434 for (
auto &AddrUse :
MRI.use_nodbg_instructions(Addr)) {
1452 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1462 assert(
MRI.getType(
MI.getOperand(0).getReg()) == VecEltTy);
1469 if (!LoadMI->isSimple())
1481 const unsigned MaxIter = 20;
1483 for (
const auto &Instr :
1485 if (Instr.isLoadFoldBarrier())
1487 if (Iter++ == MaxIter)
1504 int Elt = CVal->getZExtValue();
1516 Register VecPtr = LoadMI->getPointerReg();
1517 LLT PtrTy =
MRI.getType(VecPtr);
1525 {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}}))
1548 B.buildLoad(Result, finalPtr, PtrInfo, Alignment);
1563 MatchInfo.
IsPre = findPreIndexCandidate(LdSt, MatchInfo.
Addr, MatchInfo.
Base,
1565 if (!MatchInfo.
IsPre &&
1566 !findPostIndexCandidate(LdSt, MatchInfo.
Addr, MatchInfo.
Base,
1576 unsigned Opcode =
MI.getOpcode();
1577 bool IsStore = Opcode == TargetOpcode::G_STORE;
1583 auto *OldCst =
MRI.getVRegDef(MatchInfo.
Offset);
1585 *OldCst->getOperand(1).getCImm());
1586 MatchInfo.
Offset = NewCst.getReg(0);
1589 auto MIB =
Builder.buildInstr(NewOpcode);
1591 MIB.addDef(MatchInfo.
Addr);
1592 MIB.addUse(
MI.getOperand(0).getReg());
1594 MIB.addDef(
MI.getOperand(0).getReg());
1595 MIB.addDef(MatchInfo.
Addr);
1598 MIB.addUse(MatchInfo.
Base);
1599 MIB.addUse(MatchInfo.
Offset);
1600 MIB.addImm(MatchInfo.
IsPre);
1601 MIB->cloneMemRefs(*
MI.getMF(),
MI);
1602 MI.eraseFromParent();
1610 unsigned Opcode =
MI.getOpcode();
1611 bool IsDiv, IsSigned;
1616 case TargetOpcode::G_SDIV:
1617 case TargetOpcode::G_UDIV: {
1619 IsSigned = Opcode == TargetOpcode::G_SDIV;
1622 case TargetOpcode::G_SREM:
1623 case TargetOpcode::G_UREM: {
1625 IsSigned = Opcode == TargetOpcode::G_SREM;
1631 unsigned DivOpcode, RemOpcode, DivremOpcode;
1633 DivOpcode = TargetOpcode::G_SDIV;
1634 RemOpcode = TargetOpcode::G_SREM;
1635 DivremOpcode = TargetOpcode::G_SDIVREM;
1637 DivOpcode = TargetOpcode::G_UDIV;
1638 RemOpcode = TargetOpcode::G_UREM;
1639 DivremOpcode = TargetOpcode::G_UDIVREM;
1657 for (
auto &
UseMI :
MRI.use_nodbg_instructions(Src1)) {
1658 if (
MI.getParent() ==
UseMI.getParent() &&
1659 ((IsDiv &&
UseMI.getOpcode() == RemOpcode) ||
1660 (!IsDiv &&
UseMI.getOpcode() == DivOpcode)) &&
1673 unsigned Opcode =
MI.getOpcode();
1674 assert(OtherMI &&
"OtherMI shouldn't be empty.");
1677 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {
1678 DestDivReg =
MI.getOperand(0).getReg();
1682 DestRemReg =
MI.getOperand(0).getReg();
1686 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;
1693 Builder.setInstrAndDebugLoc(*FirstInst);
1695 Builder.buildInstr(IsSigned ? TargetOpcode::G_SDIVREM
1696 : TargetOpcode::G_UDIVREM,
1697 {DestDivReg, DestRemReg},
1699 MI.eraseFromParent();
1705 assert(
MI.getOpcode() == TargetOpcode::G_BR);
1722 if (BrIt ==
MBB->begin())
1724 assert(std::next(BrIt) ==
MBB->end() &&
"expected G_BR to be a terminator");
1726 BrCond = &*std::prev(BrIt);
1727 if (BrCond->
getOpcode() != TargetOpcode::G_BRCOND)
1733 return BrCondTarget !=
MI.getOperand(0).getMBB() &&
1734 MBB->isLayoutSuccessor(BrCondTarget);
1740 Builder.setInstrAndDebugLoc(*BrCond);
1751 MI.getOperand(0).setMBB(FallthroughBB);
1764 unsigned MaxLen)
const {
1765 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1767 DstAlignCanChange, MemOps);
1772 auto &[Dst, Src, KnownLen, Alignment, DstAlignCanChange, MemOps] = MatchInfo;
1777 DstAlignCanChange, MemOps) ==
1779 assert(
Changed &&
"expected memcpy-family instruction to lower");
1784 unsigned MaxLen)
const {
1800 if (
MI.getOpcode() != TargetOpcode::G_PTR_ADD)
1823 Type *AccessTy =
nullptr;
1824 auto &MF = *
MI.getMF();
1825 for (
auto &
UseMI :
MRI.use_nodbg_instructions(
MI.getOperand(0).getReg())) {
1828 MF.getFunction().getContext());
1833 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;
1838 AMOld.
BaseOffs = MaybeImmVal->Value.getSExtValue();
1840 unsigned AS =
MRI.getType(Add2).getAddressSpace();
1841 const auto &
TLI = *MF.getSubtarget().getTargetLowering();
1842 if (
TLI.isLegalAddressingMode(MF.getDataLayout(), AMOld, AccessTy, AS) &&
1843 !
TLI.isLegalAddressingMode(MF.getDataLayout(), AMNew, AccessTy, AS))
1852 unsigned PtrAddFlags =
MI.getFlags();
1868 MatchInfo.
Flags = Flags;
1874 assert(
MI.getOpcode() == TargetOpcode::G_PTR_ADD &&
"Expected G_PTR_ADD");
1876 LLT OffsetTy =
MRI.getType(
MI.getOperand(2).getReg());
1880 MI.getOperand(1).setReg(MatchInfo.
Base);
1881 MI.getOperand(2).setReg(NewOffset.getReg(0));
1895 unsigned Opcode =
MI.getOpcode();
1896 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1897 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1898 Opcode == TargetOpcode::G_USHLSAT) &&
1899 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1919 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();
1924 if (Opcode == TargetOpcode::G_USHLSAT &&
1925 MatchInfo.
Imm >=
MRI.getType(Shl2).getScalarSizeInBits())
1933 unsigned Opcode =
MI.getOpcode();
1934 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1935 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1936 Opcode == TargetOpcode::G_USHLSAT) &&
1937 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1939 LLT Ty =
MRI.getType(
MI.getOperand(1).getReg());
1940 unsigned const ScalarSizeInBits = Ty.getScalarSizeInBits();
1941 auto Imm = MatchInfo.
Imm;
1943 if (
Imm >= ScalarSizeInBits) {
1945 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {
1946 Builder.buildConstant(
MI.getOperand(0), 0);
1947 MI.eraseFromParent();
1952 Imm = ScalarSizeInBits - 1;
1955 LLT ImmTy =
MRI.getType(
MI.getOperand(2).getReg());
1958 MI.getOperand(1).setReg(MatchInfo.
Reg);
1959 MI.getOperand(2).setReg(NewImm);
1975 unsigned ShiftOpcode =
MI.getOpcode();
1976 assert((ShiftOpcode == TargetOpcode::G_SHL ||
1977 ShiftOpcode == TargetOpcode::G_ASHR ||
1978 ShiftOpcode == TargetOpcode::G_LSHR) &&
1979 "Expected G_SHL, G_ASHR and G_LSHR");
1982 Register LogicDest =
MI.getOperand(1).getReg();
1983 if (!
MRI.hasOneNonDBGUse(LogicDest))
1989 unsigned LogicOpcode = LogicMI->
getOpcode();
1990 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&
1991 LogicOpcode != TargetOpcode::G_XOR)
1997 if (!MaybeImmVal || MaybeImmVal->Value == 0)
2000 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();
2002 auto matchFirstShift = [&](
const MachineInstr *
MI, uint64_t &ShiftVal) {
2004 if (
MI->getOpcode() != ShiftOpcode ||
2005 !
MRI.hasOneNonDBGUse(
MI->getOperand(0).getReg()))
2014 ShiftVal = MaybeImmVal->Value.getSExtValue();
2028 if (matchFirstShift(LogicMIOp1, C0Val)) {
2030 MatchInfo.
Shift2 = LogicMIOp1;
2031 }
else if (matchFirstShift(LogicMIOp2, C0Val)) {
2033 MatchInfo.
Shift2 = LogicMIOp2;
2037 MatchInfo.
ValSum = C0Val + C1Val;
2040 if (MatchInfo.
ValSum >=
MRI.getType(LogicDest).getScalarSizeInBits())
2043 MatchInfo.
Logic = LogicMI;
2049 unsigned Opcode =
MI.getOpcode();
2050 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
2051 Opcode == TargetOpcode::G_LSHR) &&
2052 "Expected G_SHL, G_ASHR and G_LSHR");
2054 LLT ShlType =
MRI.getType(
MI.getOperand(2).getReg());
2055 LLT DestType =
MRI.getType(
MI.getOperand(0).getReg());
2061 Builder.buildInstr(Opcode, {DestType}, {Shift1Base, Const}).
getReg(0);
2070 Register Shift2Const =
MI.getOperand(2).getReg();
2072 .buildInstr(Opcode, {DestType},
2082 MI.eraseFromParent();
2094 assert(
MI.getOpcode() == TargetOpcode::G_LSHR &&
"Expected a G_LSHR");
2098 unsigned OpSizeInBits =
MRI.getType(N0).getScalarSizeInBits();
2113 LLT InnerShiftTy =
MRI.getType(InnerShift);
2115 if ((N1C + N001C).ult(InnerShiftSize)) {
2121 if ((N001C + OpSizeInBits) == InnerShiftSize)
2123 if (
MRI.hasOneNonDBGUse(N0) &&
MRI.hasOneNonDBGUse(InnerShift)) {
2124 MatchInfo.
Mask =
true;
2134 assert(
MI.getOpcode() == TargetOpcode::G_LSHR &&
"Expected a G_LSHR");
2141 if (MatchInfo.
Mask ==
true) {
2149 Builder.buildTrunc(Dst, Shift);
2150 MI.eraseFromParent();
2154 unsigned &ShiftVal)
const {
2155 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
"Expected a G_MUL");
2161 ShiftVal = MaybeImmVal->Value.exactLogBase2();
2162 return (
static_cast<int32_t
>(ShiftVal) != -1);
2166 unsigned &ShiftVal)
const {
2167 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
"Expected a G_MUL");
2169 LLT ShiftTy =
MRI.getType(
MI.getOperand(0).getReg());
2172 MI.setDesc(MIB.
getTII().
get(TargetOpcode::G_SHL));
2173 MI.getOperand(2).setReg(ShiftCst.getReg(0));
2194 auto NegCst =
B.buildConstant(Ty, -
Imm);
2196 MI.setDesc(
B.getTII().get(TargetOpcode::G_ADD));
2197 MI.getOperand(2).setReg(NegCst.getReg(0));
2199 if (
Imm.isMinSignedValue())
2209 assert(
MI.getOpcode() == TargetOpcode::G_SHL &&
VT);
2223 if (!MaybeShiftAmtVal)
2227 LLT SrcTy =
MRI.getType(ExtSrc);
2237 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();
2238 MatchData.
Reg = ExtSrc;
2239 MatchData.
Imm = ShiftAmt;
2241 unsigned MinLeadingZeros =
VT->getKnownZeroes(ExtSrc).countl_one();
2242 unsigned SrcTySize =
MRI.getType(ExtSrc).getScalarSizeInBits();
2243 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;
2249 int64_t ShiftAmtVal = MatchData.
Imm;
2251 LLT ExtSrcTy =
MRI.getType(ExtSrcReg);
2252 auto ShiftAmt =
Builder.buildConstant(ExtSrcTy, ShiftAmtVal);
2254 Builder.buildShl(ExtSrcTy, ExtSrcReg, ShiftAmt,
MI.getFlags());
2255 Builder.buildZExt(
MI.getOperand(0), NarrowShift);
2256 MI.eraseFromParent();
2269 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2270 "Expected an unmerge");
2279 LLT SrcMergeTy =
MRI.getType(SrcInstr->getSourceReg(0));
2280 LLT Dst0Ty =
MRI.getType(Unmerge.getReg(0));
2282 if (SrcMergeTy != Dst0Ty && !SameSize)
2286 for (
unsigned Idx = 0; Idx < SrcInstr->getNumSources(); ++Idx)
2287 Operands.push_back(SrcInstr->getSourceReg(Idx));
2293 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2294 "Expected an unmerge");
2296 "Not enough operands to replace all defs");
2297 unsigned NumElems =
MI.getNumOperands() - 1;
2300 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
2301 bool CanReuseInputDirectly = DstTy == SrcTy;
2302 for (
unsigned Idx = 0; Idx < NumElems; ++Idx) {
2303 Register DstReg =
MI.getOperand(Idx).getReg();
2308 const auto &DstCB =
MRI.getRegClassOrRegBank(DstReg);
2309 if (!DstCB.isNull() && DstCB !=
MRI.getRegClassOrRegBank(SrcReg)) {
2310 SrcReg =
Builder.buildCopy(
MRI.getType(SrcReg), SrcReg).getReg(0);
2311 MRI.setRegClassOrRegBank(SrcReg, DstCB);
2314 if (CanReuseInputDirectly)
2317 Builder.buildCast(DstReg, SrcReg);
2319 MI.eraseFromParent();
2324 unsigned SrcIdx =
MI.getNumOperands() - 1;
2325 Register SrcReg =
MI.getOperand(SrcIdx).getReg();
2331 LLT Dst0Ty =
MRI.getType(
MI.getOperand(0).getReg());
2334 for (
unsigned Idx = 0; Idx != SrcIdx; ++Idx) {
2336 Val = Val.
lshr(ShiftAmt);
2344 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2345 "Expected an unmerge");
2347 "Not enough operands to replace all defs");
2348 unsigned NumElems =
MI.getNumOperands() - 1;
2349 for (
unsigned Idx = 0; Idx < NumElems; ++Idx) {
2350 Register DstReg =
MI.getOperand(Idx).getReg();
2351 Builder.buildConstant(DstReg, Csts[Idx]);
2354 MI.eraseFromParent();
2360 unsigned SrcIdx =
MI.getNumOperands() - 1;
2361 Register SrcReg =
MI.getOperand(SrcIdx).getReg();
2363 unsigned NumElems =
MI.getNumOperands() - 1;
2364 for (
unsigned Idx = 0; Idx < NumElems; ++Idx) {
2365 Register DstReg =
MI.getOperand(Idx).getReg();
2366 B.buildUndef(DstReg);
2374 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2375 "Expected an unmerge");
2376 if (!
MRI.getType(
MI.getOperand(0).getReg()).isScalar() ||
2377 !
MRI.getType(
MI.getOperand(
MI.getNumDefs()).getReg()).isScalar())
2380 for (
unsigned Idx = 1, EndIdx =
MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2381 if (!
MRI.use_nodbg_empty(
MI.getOperand(Idx).getReg()))
2389 Register SrcReg =
MI.getOperand(
MI.getNumDefs()).getReg();
2390 Register Dst0Reg =
MI.getOperand(0).getReg();
2391 Builder.buildTrunc(Dst0Reg, SrcReg);
2392 MI.eraseFromParent();
2396 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2397 "Expected an unmerge");
2398 Register Dst0Reg =
MI.getOperand(0).getReg();
2399 LLT Dst0Ty =
MRI.getType(Dst0Reg);
2405 Register SrcReg =
MI.getOperand(
MI.getNumDefs()).getReg();
2406 LLT SrcTy =
MRI.getType(SrcReg);
2407 if (SrcTy.isVector())
2417 LLT ZExtSrcTy =
MRI.getType(ZExtSrcReg);
2422 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2423 "Expected an unmerge");
2425 Register Dst0Reg =
MI.getOperand(0).getReg();
2430 LLT Dst0Ty =
MRI.getType(Dst0Reg);
2431 LLT ZExtSrcTy =
MRI.getType(ZExtSrcReg);
2434 Builder.buildZExt(Dst0Reg, ZExtSrcReg);
2437 "ZExt src doesn't fit in destination");
2442 for (
unsigned Idx = 1, EndIdx =
MI.getNumDefs(); Idx != EndIdx; ++Idx) {
2444 ZeroReg =
Builder.buildConstant(Dst0Ty, 0).getReg(0);
2447 MI.eraseFromParent();
2451 unsigned TargetShiftSize,
2452 unsigned &ShiftVal)
const {
2453 assert((
MI.getOpcode() == TargetOpcode::G_SHL ||
2454 MI.getOpcode() == TargetOpcode::G_LSHR ||
2455 MI.getOpcode() == TargetOpcode::G_ASHR) &&
"Expected a shift");
2457 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
2462 unsigned Size = Ty.getSizeInBits();
2463 if (
Size <= TargetShiftSize)
2471 ShiftVal = MaybeImmVal->Value.getSExtValue();
2472 return ShiftVal >=
Size / 2 && ShiftVal <
Size;
2479 LLT Ty =
MRI.getType(SrcReg);
2480 unsigned Size = Ty.getSizeInBits();
2481 unsigned HalfSize =
Size / 2;
2482 assert(ShiftVal >= HalfSize);
2486 auto Unmerge =
Builder.buildUnmerge(HalfTy, SrcReg);
2487 unsigned NarrowShiftAmt = ShiftVal - HalfSize;
2489 if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
2490 Register Narrowed = Unmerge.getReg(1);
2497 if (NarrowShiftAmt != 0) {
2498 Narrowed =
Builder.buildLShr(HalfTy, Narrowed,
2499 Builder.buildConstant(HalfTy, NarrowShiftAmt)).getReg(0);
2503 Builder.buildMergeLikeInstr(DstReg, {Narrowed,
Zero});
2504 }
else if (
MI.getOpcode() == TargetOpcode::G_SHL) {
2505 Register Narrowed = Unmerge.getReg(0);
2510 if (NarrowShiftAmt != 0) {
2511 Narrowed =
Builder.buildShl(HalfTy, Narrowed,
2512 Builder.buildConstant(HalfTy, NarrowShiftAmt)).getReg(0);
2516 Builder.buildMergeLikeInstr(DstReg, {
Zero, Narrowed});
2518 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
2520 HalfTy, Unmerge.getReg(1),
2521 Builder.buildConstant(HalfTy, HalfSize - 1));
2523 if (ShiftVal == HalfSize) {
2526 Builder.buildMergeLikeInstr(DstReg, {Unmerge.getReg(1),
Hi});
2527 }
else if (ShiftVal ==
Size - 1) {
2535 HalfTy, Unmerge.getReg(1),
2536 Builder.buildConstant(HalfTy, ShiftVal - HalfSize));
2544 MI.eraseFromParent();
2560 assert(
MI.getOpcode() == TargetOpcode::G_ANYEXT &&
"Expected a G_ANYEXT");
2565 SrcReg = OriginalSrcReg;
2566 LLT DstTy =
MRI.getType(DstReg);
2574 assert(
MI.getOpcode() == TargetOpcode::G_ZEXT &&
"Expected a G_ZEXT");
2577 LLT DstTy =
MRI.getType(DstReg);
2582 unsigned SrcSize =
MRI.getType(SrcReg).getScalarSizeInBits();
2583 return VT->getKnownBits(Reg).countMinLeadingZeros() >= DstSize - SrcSize;
2593 if (ShiftSize > 32 && TruncSize < 32)
2606 MachineInstr &
MI, std::pair<MachineInstr *, LLT> &MatchInfo)
const {
2607 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2611 if (!
MRI.hasOneNonDBGUse(SrcReg))
2614 LLT SrcTy =
MRI.getType(SrcReg);
2615 LLT DstTy =
MRI.getType(DstReg);
2624 case TargetOpcode::G_SHL: {
2633 case TargetOpcode::G_LSHR:
2634 case TargetOpcode::G_ASHR: {
2640 for (
auto &
User :
MRI.use_instructions(DstReg))
2641 if (
User.getOpcode() == TargetOpcode::G_STORE)
2645 if (NewShiftTy == SrcTy)
2659 {NewShiftTy, TL.getPreferredShiftAmountTy(NewShiftTy)}}))
2662 MatchInfo = std::make_pair(SrcMI, NewShiftTy);
2667 MachineInstr &
MI, std::pair<MachineInstr *, LLT> &MatchInfo)
const {
2669 LLT NewShiftTy = MatchInfo.second;
2672 LLT DstTy =
MRI.getType(Dst);
2676 ShiftSrc =
Builder.buildTrunc(NewShiftTy, ShiftSrc).getReg(0);
2679 LLT PrefShiftTy = TL.getPreferredShiftAmountTy(NewShiftTy);
2680 if (
MRI.getType(ShiftAmt) != PrefShiftTy)
2681 ShiftAmt =
Builder.buildZExtOrTrunc(PrefShiftTy, ShiftAmt).getReg(0);
2685 .buildInstr(ShiftMI->
getOpcode(), {NewShiftTy}, {ShiftSrc, ShiftAmt})
2688 if (NewShiftTy == DstTy)
2691 Builder.buildTrunc(Dst, NewShift);
2698 return !MO.isReg() ||
2699 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
2704 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
2706 return all_of(Mask, [](
int Elt) {
return Elt < 0; });
2711 assert((
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
2712 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&
2713 "Expected an insert/extract element op");
2714 LLT VecTy =
MRI.getType(
MI.getOperand(1).getReg());
2719 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
2767 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())
2794 return MO.isReg() && MO.getReg().isPhysical();
2804 return I1->isIdenticalTo(*I2);
2812 if (
Builder.getTII().produceSameValue(*I1, *I2, &
MRI)) {
2819 return I1->findRegisterDefOperandIdx(InstAndDef1->Reg,
nullptr) ==
2840 std::optional<FPValueAndVReg> MaybeCst;
2844 return MaybeCst->Value.isExactlyValue(
C);
2848 unsigned OpIdx)
const {
2849 assert(
MI.getNumExplicitDefs() == 1 &&
"Expected one explicit def?");
2851 Register Replacement =
MI.getOperand(OpIdx).getReg();
2854 MI.eraseFromParent();
2859 assert(
MI.getNumExplicitDefs() == 1 &&
"Expected one explicit def?");
2863 MI.eraseFromParent();
2867 unsigned ConstIdx)
const {
2868 Register ConstReg =
MI.getOperand(ConstIdx).getReg();
2869 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
2881 assert((
MI.getOpcode() == TargetOpcode::G_FSHL ||
2882 MI.getOpcode() == TargetOpcode::G_FSHR) &&
2883 "This is not a funnel shift operation");
2885 Register ConstReg =
MI.getOperand(3).getReg();
2886 LLT ConstTy =
MRI.getType(ConstReg);
2887 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
2890 assert((VRegAndVal) &&
"Value is not a constant");
2893 APInt NewConst = VRegAndVal->Value.
urem(
2898 MI.getOpcode(), {MI.getOperand(0)},
2899 {MI.getOperand(1), MI.getOperand(2), NewConstInstr.getReg(0)});
2901 MI.eraseFromParent();
2905 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
2919 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2921 MI.eraseFromParent();
2926 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2928 MI.eraseFromParent();
2932 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2934 MI.eraseFromParent();
2939 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2941 MI.eraseFromParent();
2945 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2947 MI.eraseFromParent();
2952 assert(
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
2955 LLT DstTy =
MRI.getType(DstReg);
2964 if (
MRI.hasOneNonDBGUse(DstReg) &&
2965 MRI.use_nodbg_instructions(DstReg).begin()->getOpcode() ==
2966 TargetOpcode::G_INSERT_VECTOR_ELT)
2972 MatchInfo.
resize(NumElts);
2976 if (IntImm >= NumElts || IntImm < 0)
2978 if (!MatchInfo[IntImm])
2979 MatchInfo[IntImm] = TmpReg;
2983 if (CurrInst->
getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
2985 if (TmpInst->
getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
2994 return TmpInst->
getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
3001 auto GetUndef = [&]() {
3004 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
3012 Builder.buildBuildVector(
MI.getOperand(0).getReg(), MatchInfo);
3013 MI.eraseFromParent();
3026 unsigned InnerOpc = InnerDef->
getOpcode();
3027 if (InnerOpc != TargetOpcode::G_ADD && InnerOpc != TargetOpcode::G_SUB)
3030 if (!
MRI.hasOneNonDBGUse(MInner))
3041 if (!
MRI.hasOneNonDBGUse(MaybeNot))
3051 if (!TryMatch(InnerLHS, InnerRHS) &&
3052 !(InnerOpc == TargetOpcode::G_ADD && TryMatch(InnerRHS, InnerLHS)))
3056 unsigned FlippedOpc = (InnerOpc == TargetOpcode::G_ADD) ? TargetOpcode::G_SUB
3057 : TargetOpcode::G_ADD;
3060 MatchInfo = [=](MachineIRBuilder &
Builder) {
3061 auto NewInner =
Builder.buildInstr(FlippedOpc, {Ty}, {
B,
C});
3062 auto NewNot =
Builder.buildNot(Ty, NewInner);
3063 Builder.buildInstr(RootOpc, {Dst}, {
A, NewNot});
3075 unsigned RootOpc =
MI.getOpcode();
3077 LLT Ty =
MRI.getType(Dst);
3082 return matchBinopWithNegInner(LHS, RHS, RootOpc, Dst, Ty, MatchInfo) ||
3083 matchBinopWithNegInner(RHS, LHS, RootOpc, Dst, Ty, MatchInfo);
3094 unsigned LogicOpcode =
MI.getOpcode();
3095 assert(LogicOpcode == TargetOpcode::G_AND ||
3096 LogicOpcode == TargetOpcode::G_OR ||
3097 LogicOpcode == TargetOpcode::G_XOR);
3103 if (!
MRI.hasOneNonDBGUse(LHSReg) || !
MRI.hasOneNonDBGUse(RHSReg))
3109 if (!LeftHandInst || !RightHandInst)
3111 unsigned HandOpcode = LeftHandInst->
getOpcode();
3112 if (HandOpcode != RightHandInst->
getOpcode())
3126 if (!XTy.
isValid() || XTy != YTy)
3131 switch (HandOpcode) {
3134 case TargetOpcode::G_ANYEXT:
3135 case TargetOpcode::G_SEXT:
3136 case TargetOpcode::G_ZEXT: {
3140 case TargetOpcode::G_TRUNC: {
3145 LLT DstTy =
MRI.getType(Dst);
3150 if (
TLI.isZExtFree(DstTy, XTy, Ctx) &&
TLI.isTruncateFree(XTy, DstTy, Ctx))
3154 case TargetOpcode::G_AND:
3155 case TargetOpcode::G_ASHR:
3156 case TargetOpcode::G_LSHR:
3157 case TargetOpcode::G_SHL: {
3162 ExtraHandOpSrcReg = ZOp.
getReg();
3173 auto NewLogicDst =
MRI.createGenericVirtualRegister(XTy);
3184 if (ExtraHandOpSrcReg.
isValid())
3196 "Expected at least one instr to build?");
3198 assert(InstrToBuild.Opcode &&
"Expected a valid opcode?");
3199 assert(InstrToBuild.OperandFns.size() &&
"Expected at least one operand?");
3201 for (
auto &OperandFn : InstrToBuild.OperandFns)
3204 MI.eraseFromParent();
3208 MachineInstr &
MI, std::tuple<Register, int64_t> &MatchInfo)
const {
3209 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
3210 int64_t ShlCst, AshrCst;
3216 if (ShlCst != AshrCst)
3219 {TargetOpcode::G_SEXT_INREG,
3222 {
MRI.getType(Src).getScalarSizeInBits() - ShlCst}}))
3224 MatchInfo = std::make_tuple(Src, ShlCst);
3229 MachineInstr &
MI, std::tuple<Register, int64_t> &MatchInfo)
const {
3230 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
3233 std::tie(Src, ShiftAmt) = MatchInfo;
3234 unsigned Size =
MRI.getType(Src).getScalarSizeInBits();
3235 Builder.buildSExtInReg(
MI.getOperand(0).getReg(), Src,
Size - ShiftAmt);
3236 MI.eraseFromParent();
3243 assert(
MI.getOpcode() == TargetOpcode::G_AND);
3246 LLT Ty =
MRI.getType(Dst);
3258 B.buildAnd(Dst, R,
B.buildConstant(Ty,
C1 & C2));
3261 auto Zero =
B.buildConstant(Ty, 0);
3284 assert(
MI.getOpcode() == TargetOpcode::G_AND);
3308 (LHSBits.
Zero | RHSBits.
One).isAllOnes()) {
3315 (LHSBits.
One | RHSBits.
Zero).isAllOnes()) {
3332 assert(
MI.getOpcode() == TargetOpcode::G_OR);
3350 (LHSBits.
One | RHSBits.
Zero).isAllOnes()) {
3357 (LHSBits.
Zero | RHSBits.
One).isAllOnes()) {
3368 unsigned ExtBits =
MI.getOperand(2).getImm();
3369 unsigned TypeSize =
MRI.getType(Src).getScalarSizeInBits();
3370 return VT->computeNumSignBits(Src) >= (
TypeSize - ExtBits + 1);
3374 int64_t Cst,
bool IsVector,
bool IsFP) {
3376 return (ScalarSizeBits == 1 && Cst == -1) ||
3398 unsigned BuildUseCount = BV.getNumSources();
3399 if (BuildUseCount % 2 != 0)
3402 unsigned NumUnmerge = BuildUseCount / 2;
3408 if (!Unmerge || Unmerge->getNumDefs() != NumUnmerge)
3411 UnmergeSrc = Unmerge->getSourceReg();
3413 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
3414 LLT UnmergeSrcTy =
MRI.getType(UnmergeSrc);
3421 !
isLegal({TargetOpcode::G_CONCAT_VECTORS, {DstTy, UnmergeSrcTy}}))
3426 for (
unsigned I = 0;
I < NumUnmerge; ++
I) {
3427 auto MaybeUnmergeReg = BV.getSourceReg(
I);
3430 if (!LoopUnmerge || LoopUnmerge != Unmerge)
3433 if (LoopUnmerge->getOperand(
I).getReg() != MaybeUnmergeReg)
3438 if (Unmerge->getNumDefs() != NumUnmerge)
3442 for (
unsigned I = NumUnmerge;
I < BuildUseCount; ++
I) {
3445 if (
Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
3456 assert(UnmergeSrc &&
"Expected there to be one matching G_UNMERGE_VALUES");
3457 B.setInstrAndDebugLoc(
MI);
3459 Register UndefVec =
B.buildUndef(
MRI.getType(UnmergeSrc)).getReg(0);
3460 B.buildConcatVectors(
MI.getOperand(0), {UnmergeSrc, UndefVec});
3462 MI.eraseFromParent();
3484 unsigned NumOperands =
BuildMI->getNumSources();
3494 for (
I = 0;
I < NumOperands; ++
I) {
3506 UnmergeMI != UnmergeSrcMI)
3517 for (;
I < NumOperands; ++
I) {
3524 LLT UnmergeSrcTy =
MRI.getType(MatchInfo);
3534 LLT UnmergeDstEltTy =
MRI.getType(UnmergeDstReg);
3535 if (UnmergeSrcEltTy != UnmergeDstEltTy)
3543 !
isLegal({TargetOpcode::G_CONCAT_VECTORS, {MidTy, UnmergeSrcTy}}))
3546 if (!
isLegal({TargetOpcode::G_TRUNC, {DstTy, MidTy}}))
3558 LLT DstTy =
MRI.getType(DstReg);
3559 LLT UnmergeSrcTy =
MRI.getType(MatchInfo);
3564 if (DstTyNumElt / UnmergeSrcTyNumElt == 1) {
3569 for (
unsigned I = 1;
I < DstTyNumElt / UnmergeSrcTyNumElt; ++
I)
3573 MidReg =
Builder.buildConcatVectors(MidTy, ConcatRegs).getReg(0);
3576 Builder.buildTrunc(DstReg, MidReg);
3577 MI.eraseFromParent();
3582 assert(
MI.getOpcode() == TargetOpcode::G_XOR);
3583 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
3584 const auto &
TLI = *
Builder.getMF().getSubtarget().getTargetLowering();
3592 if (!
MRI.hasOneNonDBGUse(XorSrc))
3602 for (
unsigned I = 0;
I < RegsToNegate.
size(); ++
I) {
3604 if (!
MRI.hasOneNonDBGUse(Reg))
3609 switch (Def->getOpcode()) {
3614 case TargetOpcode::G_ICMP:
3620 case TargetOpcode::G_FCMP:
3626 case TargetOpcode::G_AND:
3627 case TargetOpcode::G_OR:
3633 RegsToNegate.
push_back(Def->getOperand(1).getReg());
3634 RegsToNegate.
push_back(Def->getOperand(2).getReg());
3642 if (Ty.isVector()) {
3660 for (
Register Reg : RegsToNegate) {
3665 switch (Def->getOpcode()) {
3668 case TargetOpcode::G_ICMP:
3669 case TargetOpcode::G_FCMP: {
3676 case TargetOpcode::G_AND:
3677 Def->setDesc(
Builder.getTII().get(TargetOpcode::G_OR));
3679 case TargetOpcode::G_OR:
3680 Def->setDesc(
Builder.getTII().get(TargetOpcode::G_AND));
3687 MI.eraseFromParent();
3691 MachineInstr &
MI, std::pair<Register, Register> &MatchInfo)
const {
3693 assert(
MI.getOpcode() == TargetOpcode::G_XOR);
3697 Register SharedReg =
MI.getOperand(2).getReg();
3711 if (!
MRI.hasOneNonDBGUse(AndReg))
3718 return Y == SharedReg;
3722 MachineInstr &
MI, std::pair<Register, Register> &MatchInfo)
const {
3725 std::tie(
X,
Y) = MatchInfo;
3728 MI.setDesc(
Builder.getTII().get(TargetOpcode::G_AND));
3729 MI.getOperand(1).setReg(Not->getOperand(0).getReg());
3730 MI.getOperand(2).setReg(
Y);
3736 Register DstReg = PtrAdd.getReg(0);
3737 LLT Ty =
MRI.getType(DstReg);
3740 if (
DL.isNonIntegralAddressSpace(Ty.getScalarType().getAddressSpace()))
3755 Register Pow2Src1 =
MI.getOperand(2).getReg();
3756 LLT Ty =
MRI.getType(DstReg);
3759 auto NegOne =
Builder.buildConstant(Ty, -1);
3760 auto Add =
Builder.buildAdd(Ty, Pow2Src1, NegOne);
3762 MI.eraseFromParent();
3766 unsigned &SelectOpNo)
const {
3778 !
MRI.hasOneNonDBGUse(LHS)) {
3779 OtherOperandReg = LHS;
3783 !
MRI.hasOneNonDBGUse(RHS))
3801 unsigned BinOpcode =
MI.getOpcode();
3806 bool CanFoldNonConst =
3807 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&
3812 if (CanFoldNonConst)
3837 LLT Ty =
MRI.getType(Dst);
3838 unsigned BinOpcode =
MI.getOpcode();
3845 if (SelectOperand == 1) {
3849 FoldTrue =
Builder.buildInstr(BinOpcode, {Ty}, {SelectTrue, RHS}).
getReg(0);
3851 Builder.buildInstr(BinOpcode, {Ty}, {SelectFalse, RHS}).
getReg(0);
3853 FoldTrue =
Builder.buildInstr(BinOpcode, {Ty}, {LHS, SelectTrue}).
getReg(0);
3855 Builder.buildInstr(BinOpcode, {Ty}, {LHS, SelectFalse}).
getReg(0);
3858 Builder.buildSelect(Dst, SelectCond, FoldTrue, FoldFalse,
MI.getFlags());
3859 MI.eraseFromParent();
3862std::optional<SmallVector<Register, 8>>
3863CombinerHelper::findCandidatesForLoadOrCombine(
const MachineInstr *Root)
const {
3864 assert(Root->
getOpcode() == TargetOpcode::G_OR &&
"Expected G_OR only!");
3893 const unsigned MaxIter =
3895 for (
unsigned Iter = 0; Iter < MaxIter; ++Iter) {
3903 if (!
MRI.hasOneNonDBGUse(OrLHS) || !
MRI.hasOneNonDBGUse(OrRHS))
3904 return std::nullopt;
3920 if (RegsToVisit.
empty() || RegsToVisit.
size() % 2 != 0)
3921 return std::nullopt;
3933static std::optional<std::pair<GZExtLoad *, int64_t>>
3937 "Expected Reg to only have one non-debug use?");
3945 if (Shift % MemSizeInBits != 0)
3946 return std::nullopt;
3951 return std::nullopt;
3953 if (!
Load->isUnordered() ||
Load->getMemSizeInBits() != MemSizeInBits)
3954 return std::nullopt;
3956 return std::make_pair(
Load, Shift / MemSizeInBits);
3959std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
3960CombinerHelper::findLoadOffsetsForLoadOrCombine(
3963 const unsigned MemSizeInBits)
const {
3966 SmallSetVector<const MachineInstr *, 8> Loads;
3972 GZExtLoad *LowestIdxLoad =
nullptr;
3975 SmallSet<int64_t, 8> SeenIdx;
3979 MachineBasicBlock *
MBB =
nullptr;
3980 const MachineMemOperand *MMO =
nullptr;
3983 GZExtLoad *EarliestLoad =
nullptr;
3986 GZExtLoad *LatestLoad =
nullptr;
3995 for (
auto Reg : RegsToVisit) {
4000 return std::nullopt;
4003 std::tie(
Load, DstPos) = *LoadAndPos;
4007 MachineBasicBlock *LoadMBB =
Load->getParent();
4011 return std::nullopt;
4014 auto &LoadMMO =
Load->getMMO();
4018 return std::nullopt;
4025 LoadPtr =
Load->getOperand(1).getReg();
4030 if (!SeenIdx.
insert(Idx).second)
4031 return std::nullopt;
4038 if (BasePtr != LoadPtr)
4039 return std::nullopt;
4041 if (Idx < LowestIdx) {
4043 LowestIdxLoad =
Load;
4050 if (!MemOffset2Idx.
try_emplace(DstPos, Idx).second)
4051 return std::nullopt;
4060 EarliestLoad =
Load;
4068 "Expected to find a load for each register?");
4069 assert(EarliestLoad != LatestLoad && EarliestLoad &&
4070 LatestLoad &&
"Expected at least two loads?");
4079 const unsigned MaxIter = 20;
4085 if (
MI.isLoadFoldBarrier())
4086 return std::nullopt;
4087 if (Iter++ == MaxIter)
4088 return std::nullopt;
4091 return std::make_tuple(LowestIdxLoad, LowestIdx, LatestLoad);
4097 assert(
MI.getOpcode() == TargetOpcode::G_OR);
4110 LLT Ty =
MRI.getType(Dst);
4116 const unsigned WideMemSizeInBits = Ty.getSizeInBits();
4117 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)
4121 auto RegsToVisit = findCandidatesForLoadOrCombine(&
MI);
4128 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();
4129 if (NarrowMemSizeInBits % 8 != 0)
4142 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(
4143 MemOffset2Idx, *RegsToVisit, NarrowMemSizeInBits);
4146 std::tie(LowestIdxLoad, LowestIdx, LatestLoad) = *MaybeLoadInfo;
4153 std::optional<bool> IsBigEndian =
isBigEndian(MemOffset2Idx, LowestIdx);
4156 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;
4168 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;
4169 const unsigned ZeroByteOffset =
4173 auto ZeroOffsetIdx = MemOffset2Idx.
find(ZeroByteOffset);
4174 if (ZeroOffsetIdx == MemOffset2Idx.
end() ||
4175 ZeroOffsetIdx->second != LowestIdx)
4185 {TargetOpcode::G_LOAD, {Ty,
MRI.getType(Ptr)}, {MMDesc}}))
4199 MIB.setInstrAndDebugLoc(*LatestLoad);
4200 Register LoadDst = NeedsBSwap ?
MRI.cloneVirtualRegister(Dst) : Dst;
4201 MIB.buildLoad(LoadDst, Ptr, *NewMMO);
4203 MIB.buildBSwap(Dst, LoadDst);
4215 if (
MRI.getType(DstReg).isVector())
4219 if (!
MRI.hasOneNonDBGUse(DstReg))
4221 ExtMI = &*
MRI.use_instr_nodbg_begin(DstReg);
4223 case TargetOpcode::G_ANYEXT:
4225 case TargetOpcode::G_ZEXT:
4226 case TargetOpcode::G_SEXT:
4233 if (
Builder.getTII().isExtendLikelyToBeFolded(*ExtMI,
MRI))
4240 for (
unsigned I = 0;
I <
PHI.getNumIncomingValues(); ++
I) {
4242 switch (
DefMI->getOpcode()) {
4243 case TargetOpcode::G_LOAD:
4244 case TargetOpcode::G_TRUNC:
4245 case TargetOpcode::G_SEXT:
4246 case TargetOpcode::G_ZEXT:
4247 case TargetOpcode::G_ANYEXT:
4248 case TargetOpcode::G_CONSTANT:
4252 if (InSrcs.
size() > 2)
4266 LLT ExtTy =
MRI.getType(DstReg);
4273 for (
unsigned I = 0;
I <
PHI.getNumIncomingValues(); ++
I) {
4274 auto SrcReg =
PHI.getIncomingValue(
I);
4278 if (!SrcMIs.
insert(SrcMI))
4284 if (InsertPt !=
MBB->end() && InsertPt->isPHI())
4285 InsertPt =
MBB->getFirstNonPHI();
4290 OldToNewSrcMap[SrcMI] = NewExt;
4295 auto NewPhi =
Builder.buildInstrNoInsert(TargetOpcode::G_PHI);
4296 NewPhi.addDef(DstReg);
4299 NewPhi.addMBB(MO.getMBB());
4302 auto *NewSrc = OldToNewSrcMap[
MRI.getVRegDef(MO.getReg())];
4303 NewPhi.addUse(NewSrc->getOperand(0).getReg());
4311 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
4315 LLT SrcTy =
MRI.getType(SrcVec);
4316 if (SrcTy.isScalableVector())
4320 if (!Cst || Cst->Value.getZExtValue() >= SrcTy.getNumElements())
4323 unsigned VecIdx = Cst->Value.getZExtValue();
4335 if (SrcVecMI->
getOpcode() != TargetOpcode::G_BUILD_VECTOR &&
4336 SrcVecMI->
getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)
4340 if (!
MRI.hasOneNonDBGUse(SrcVec) &&
4352 LLT ScalarTy =
MRI.getType(Reg);
4354 LLT DstTy =
MRI.getType(DstReg);
4356 if (ScalarTy != DstTy) {
4358 Builder.buildTrunc(DstReg, Reg);
4359 MI.eraseFromParent();
4367 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs)
const {
4368 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4386 LLT DstTy =
MRI.getType(DstReg);
4391 if (
II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
4396 unsigned Idx = Cst->getZExtValue();
4399 ExtractedElts.
set(Idx);
4400 SrcDstPairs.emplace_back(
4401 std::make_pair(
MI.getOperand(Idx + 1).getReg(), &
II));
4404 return ExtractedElts.
all();
4409 SmallVectorImpl<std::pair<Register, MachineInstr *>> &SrcDstPairs)
const {
4410 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4411 for (
auto &Pair : SrcDstPairs) {
4412 auto *ExtMI = Pair.second;
4414 ExtMI->eraseFromParent();
4416 MI.eraseFromParent();
4423 MI.eraseFromParent();
4433 bool AllowScalarConstants,
4435 assert(
MI.getOpcode() == TargetOpcode::G_OR);
4438 LLT Ty =
MRI.getType(Dst);
4439 unsigned BitWidth = Ty.getScalarSizeInBits();
4441 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;
4442 unsigned FshOpc = 0;
4453 int64_t CstShlAmt = 0, CstLShrAmt;
4456 CstShlAmt + CstLShrAmt ==
BitWidth) {
4457 FshOpc = TargetOpcode::G_FSHR;
4463 FshOpc = TargetOpcode::G_FSHL;
4468 FshOpc = TargetOpcode::G_FSHR;
4473 LLT AmtTy =
MRI.getType(Amt);
4475 (!AllowScalarConstants || CstShlAmt == 0 || !Ty.isScalar()))
4479 B.buildInstr(FshOpc, {Dst}, {ShlSrc, LShrSrc, Amt});
4486 assert(
MI.getOpcode() == TargetOpcode::G_ROTL ||
4487 MI.getOpcode() == TargetOpcode::G_ROTR);
4489 MRI.getType(
MI.getOperand(0).getReg()).getScalarSizeInBits();
4491 bool OutOfRange =
false;
4492 auto MatchOutOfRange = [Bitsize, &OutOfRange](
const Constant *
C) {
4494 OutOfRange |= CI->getValue().uge(Bitsize);
4501 assert(
MI.getOpcode() == TargetOpcode::G_ROTL ||
4502 MI.getOpcode() == TargetOpcode::G_ROTR);
4504 MRI.getType(
MI.getOperand(0).getReg()).getScalarSizeInBits();
4506 LLT AmtTy =
MRI.getType(Amt);
4507 auto Bits =
Builder.buildConstant(AmtTy, Bitsize);
4508 Amt =
Builder.buildURem(AmtTy,
MI.getOperand(2).getReg(), Bits).getReg(0);
4510 MI.getOperand(2).setReg(Amt);
4515 int64_t &MatchInfo)
const {
4516 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
4527 auto KnownRHS =
VT->getKnownBits(
MI.getOperand(3).getReg());
4528 if (KnownRHS.isUnknown())
4531 std::optional<bool> KnownVal;
4532 if (KnownRHS.isZero()) {
4542 auto KnownLHS =
VT->getKnownBits(
MI.getOperand(2).getReg());
4552 MRI.getType(
MI.getOperand(0).getReg()).isVector(),
4561 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
4577 LLT DstTy =
MRI.getType(Dst);
4585 auto KnownLHS =
VT->getKnownBits(LHS);
4586 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)
4589 LLT LHSTy =
MRI.getType(LHS);
4592 unsigned Op = TargetOpcode::COPY;
4593 if (DstSize != LHSSize)
4594 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;
4605 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4609 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
4615 int64_t AndMaskBits;
4623 if (AndMaskBits & OrMaskBits)
4629 if (
MI.getOperand(1).getReg() == AndMaskReg)
4630 MI.getOperand(2).setReg(AndMaskReg);
4631 MI.getOperand(1).setReg(Src);
4641 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
4644 LLT Ty =
MRI.getType(Src);
4646 if (!
LI || !
LI->isLegalOrCustom({TargetOpcode::G_SBFX, {Ty, ExtractTy}}))
4648 int64_t Width =
MI.getOperand(2).getImm();
4655 if (ShiftImm < 0 || ShiftImm + Width > Ty.getScalarSizeInBits())
4659 auto Cst1 =
B.buildConstant(ExtractTy, ShiftImm);
4660 auto Cst2 =
B.buildConstant(ExtractTy, Width);
4661 B.buildSbfx(Dst, ShiftSrc, Cst1, Cst2);
4671 LLT Ty =
MRI.getType(Dst);
4675 if (
LI && !
LI->isLegalOrCustom({TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4678 int64_t AndImm, LSBImm;
4680 const unsigned Size = Ty.getScalarSizeInBits();
4688 uint64_t MaybeMask =
static_cast<uint64_t
>(AndImm);
4693 if (MaybeMask & (MaybeMask + 1))
4697 if (
static_cast<uint64_t
>(LSBImm) >=
Size)
4705 if (
static_cast<uint64_t
>(LSBImm) + Width >
Size)
4709 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4710 auto LSBCst =
B.buildConstant(ExtractTy, LSBImm);
4711 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {ShiftSrc, LSBCst, WidthCst});
4719 const unsigned Opcode =
MI.getOpcode();
4720 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);
4722 const Register Dst =
MI.getOperand(0).getReg();
4724 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR
4725 ? TargetOpcode::G_SBFX
4726 : TargetOpcode::G_UBFX;
4729 LLT Ty =
MRI.getType(Dst);
4731 if (!
LI || !
LI->isLegalOrCustom({ExtrOpcode, {Ty, ExtractTy}}))
4737 const unsigned Size = Ty.getScalarSizeInBits();
4746 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >=
Size)
4750 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)
4754 const int64_t Pos = ShrAmt - ShlAmt;
4755 const int64_t Width =
Size - ShrAmt;
4758 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4759 auto PosCst =
B.buildConstant(ExtractTy, Pos);
4760 B.buildInstr(ExtrOpcode, {Dst}, {ShlSrc, PosCst, WidthCst});
4768 const unsigned Opcode =
MI.getOpcode();
4769 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);
4771 const Register Dst =
MI.getOperand(0).getReg();
4772 LLT Ty =
MRI.getType(Dst);
4774 if (
LI && !
LI->isLegalOrCustom({TargetOpcode::G_UBFX, {Ty, ExtractTy}}))
4786 const unsigned Size = Ty.getScalarSizeInBits();
4787 if (ShrAmt < 0 || ShrAmt >=
Size)
4791 if (0 == (SMask >> ShrAmt)) {
4793 B.buildConstant(Dst, 0);
4806 const int64_t Pos = ShrAmt;
4811 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt ==
Size)
4815 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4816 auto PosCst =
B.buildConstant(ExtractTy, Pos);
4817 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {AndSrc, PosCst, WidthCst});
4822bool CombinerHelper::reassociationCanBreakAddressingModePattern(
4826 Register Src1Reg = PtrAdd.getBaseReg();
4831 Register Src2Reg = PtrAdd.getOffsetReg();
4833 if (
MRI.hasOneNonDBGUse(Src1Reg))
4843 const APInt &C1APIntVal = *
C1;
4844 const APInt &C2APIntVal = *C2;
4845 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();
4847 for (
auto &
UseMI :
MRI.use_nodbg_instructions(PtrAdd.getReg(0))) {
4850 MachineInstr *ConvUseMI = &
UseMI;
4851 unsigned ConvUseOpc = ConvUseMI->
getOpcode();
4852 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||
4853 ConvUseOpc == TargetOpcode::G_PTRTOINT) {
4855 if (!
MRI.hasOneNonDBGUse(DefReg))
4857 ConvUseMI = &*
MRI.use_instr_nodbg_begin(DefReg);
4866 TargetLoweringBase::AddrMode AM;
4869 unsigned AS =
MRI.getType(LdStMI->getPointerReg()).getAddressSpace();
4871 PtrAdd.getMF()->getFunction().getContext());
4872 const auto &
TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();
4891 Register Src1Reg =
MI.getOperand(1).getReg();
4892 if (RHS->getOpcode() != TargetOpcode::G_ADD)
4904 unsigned PtrAddFlags =
MI.getFlags();
4905 unsigned AddFlags = RHS->getFlags();
4918 LLT PtrTy =
MRI.getType(
MI.getOperand(0).getReg());
4921 Builder.buildPtrAdd(PtrTy, Src1Reg, RHS->getOperand(1).getReg(), Flags);
4923 MI.getOperand(1).setReg(NewBase.getReg(0));
4924 MI.getOperand(2).setReg(RHS->getOperand(2).getReg());
4928 return !reassociationCanBreakAddressingModePattern(
MI);
4938 std::optional<ValueAndVReg> LHSCstOff;
4948 unsigned PtrAddFlags =
MI.getFlags();
4949 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
4951 bool IsNoUSWrap = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
4953 bool IsInBounds = IsNoUWrap && (PtrAddFlags & LHSPtrAddFlags &
4967 LHSPtrAdd->moveBefore(&
MI);
4970 auto NewCst =
B.buildConstant(
MRI.getType(RHSReg), LHSCstOff->Value);
4972 MI.getOperand(2).setReg(NewCst.getReg(0));
4975 Observer.changingInstr(*LHSPtrAdd);
4976 LHSPtrAdd->getOperand(2).setReg(RHSReg);
4977 LHSPtrAdd->setFlags(Flags);
4980 return !reassociationCanBreakAddressingModePattern(
MI);
4991 Register Src2Reg =
MI.getOperand(2).getReg();
4992 Register LHSSrc1 = LHSPtrAdd->getBaseReg();
4993 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();
5006 unsigned PtrAddFlags =
MI.getFlags();
5007 unsigned LHSPtrAddFlags = LHSPtrAdd->getFlags();
5020 auto NewCst =
B.buildConstant(
MRI.getType(Src2Reg), *
C1 + *C2);
5022 MI.getOperand(1).setReg(LHSSrc1);
5023 MI.getOperand(2).setReg(NewCst.getReg(0));
5027 return !reassociationCanBreakAddressingModePattern(
MI);
5067 LLT OpRHSTy =
MRI.getType(OpRHS);
5084 auto NewCst =
B.buildInstr(
Opc, {OpRHSTy}, {OpLHSRHS, OpRHS});
5085 B.buildInstr(
Opc, {DstReg}, {OpLHSLHS, NewCst});
5093 auto NewLHSLHS =
B.buildInstr(
Opc, {OpRHSTy}, {OpLHSLHS, OpRHS});
5094 B.buildInstr(
Opc, {DstReg}, {NewLHSLHS, OpLHSRHS});
5107 unsigned Opc =
MI.getOpcode();
5123 LLT DstTy =
MRI.getType(Dst);
5124 unsigned Opc =
MI.getOpcode();
5137 for (
unsigned I = 0, E = BV->getNumSources();
I != E; ++
I) {
5144 B.buildBuildVectorConstant(Dst, Csts);
5151 B.buildConstant(Dst, Cst);
5162 LLT DstTy =
MRI.getType(Dst);
5165 MI.getOpcode(), DstTy,
MI.getOperand(1).getReg(),
MRI);
5170 if (Csts.
size() == 1)
5171 B.buildConstant(Dst, Csts[0]);
5173 B.buildBuildVectorConstant(Dst, Csts);
5183 LLT DstTy =
MRI.getType(Dst);
5197 B.buildBuildVectorConstant(Dst, Csts);
5206 B.buildConstant(Dst, Cst);
5231 LLT DstTy =
MRI.getType(Dst);
5244 B.buildBuildVectorFConstant(Dst, Csts);
5253 B.buildFConstant(Dst, Cst);
5262 LLT DstTy =
MRI.getType(Dst);
5263 LLT SrcTy =
MRI.getType(Src);
5270 if (SrcTy.isVector() &&
5280 if (Csts.
size() == 1)
5281 B.buildFConstant(Dst, Csts[0]);
5283 B.buildBuildVectorFConstant(Dst, Csts);
5290 assert(
MI.getOpcode() == TargetOpcode::G_FMA ||
5291 MI.getOpcode() == TargetOpcode::G_FMAD);
5293 auto [
_, Op1, Op2, Op3] =
MI.getFirst4Regs();
5294 LLT DstTy =
MRI.getType(Dst);
5312 if (!BV1 || !BV2 || !BV3)
5314 unsigned NumElts = BV1->getNumSources();
5315 if (BV2->getNumSources() != NumElts || BV3->getNumSources() != NumElts)
5318 for (
unsigned I = 0;
I != NumElts; ++
I) {
5322 if (!
C1 || !C2 || !C3)
5328 B.buildBuildVectorFConstant(Dst, Csts);
5336 if (!Op1Cst || !Op2Cst || !Op3Cst)
5364 assert(
MI.getOpcode() == TargetOpcode::G_AND);
5368 LLT WideTy =
MRI.getType(Dst);
5372 if (!WideTy.
isScalar() || !
MRI.hasOneNonDBGUse(AndLHS))
5388 case TargetOpcode::G_ADD:
5389 case TargetOpcode::G_SUB:
5390 case TargetOpcode::G_MUL:
5391 case TargetOpcode::G_AND:
5392 case TargetOpcode::G_OR:
5393 case TargetOpcode::G_XOR:
5401 auto Mask = Cst->Value;
5406 unsigned NarrowWidth = Mask.countr_one();
5412 auto &MF = *
MI.getMF();
5415 if (!
TLI.isTruncateFree(WideTy, NarrowTy, Ctx) ||
5416 !
TLI.isZExtFree(NarrowTy, WideTy, Ctx))
5424 auto NarrowLHS =
Builder.buildTrunc(NarrowTy, BinOpLHS);
5425 auto NarrowRHS =
Builder.buildTrunc(NarrowTy, BinOpRHS);
5427 Builder.buildInstr(LHSOpc, {NarrowTy}, {NarrowLHS, NarrowRHS});
5428 auto Ext =
Builder.buildZExt(WideTy, NarrowBinOp);
5430 MI.getOperand(1).setReg(Ext.getReg(0));
5439 assert(
MI.getOpcode() == TargetOpcode::G_UMULO ||
5440 MI.getOpcode() == TargetOpcode::G_SMULO);
5449 B.buildConstant(Dst, 0);
5450 B.buildConstant(Carry, 0);
5458 if (SrcA.
isValid() && SrcA == SrcB)
5466 unsigned Opcode =
MI.getOpcode();
5467 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5469 Register Dst = UDivorRem.getReg(0);
5470 Register LHS = UDivorRem.getReg(1);
5471 Register RHS = UDivorRem.getReg(2);
5472 LLT Ty =
MRI.getType(Dst);
5480 bool UseSRL =
false;
5485 auto BuildExactUDIVPattern = [&](
const Constant *
C) {
5487 if (IsSplat && !Factors.
empty()) {
5494 APInt Divisor = CI->getValue();
5503 Shifts.
push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));
5504 Factors.
push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));
5514 if (Ty.isVector()) {
5515 Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);
5516 Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);
5519 Factor = Factors[0];
5527 return MIB.buildMul(Ty, Res, Factor);
5530 unsigned KnownLeadingZeros =
5531 VT ?
VT->getKnownBits(LHS).countMinLeadingZeros() : 0;
5533 bool UseNPQ =
false;
5535 auto BuildUDIVPattern = [&](
const Constant *
C) {
5537 const APInt &Divisor = CI->getValue();
5539 bool SelNPQ =
false;
5541 unsigned PreShift = 0, PostShift = 0;
5546 if (!Divisor.
isOne()) {
5552 Divisor, std::min(KnownLeadingZeros, Divisor.
countl_zero()));
5554 Magic = std::move(magics.
Magic);
5557 "We shouldn't generate an undefined shift!");
5559 "We shouldn't generate an undefined shift!");
5563 SelNPQ = magics.
IsAdd;
5567 MIB.buildConstant(ScalarShiftAmtTy, PreShift).getReg(0));
5568 MagicFactors.
push_back(MIB.buildConstant(ScalarTy, Magic).getReg(0));
5570 MIB.buildConstant(ScalarTy,
5575 MIB.buildConstant(ScalarShiftAmtTy, PostShift).getReg(0));
5583 assert(Matched &&
"Expected unary predicate match to succeed");
5585 Register PreShift, PostShift, MagicFactor, NPQFactor;
5588 PreShift = MIB.buildBuildVector(ShiftAmtTy, PreShifts).getReg(0);
5589 MagicFactor = MIB.buildBuildVector(Ty, MagicFactors).getReg(0);
5590 NPQFactor = MIB.buildBuildVector(Ty, NPQFactors).getReg(0);
5591 PostShift = MIB.buildBuildVector(ShiftAmtTy, PostShifts).getReg(0);
5594 "Non-build_vector operation should have been a scalar");
5595 PreShift = PreShifts[0];
5596 MagicFactor = MagicFactors[0];
5597 PostShift = PostShifts[0];
5601 Q = MIB.buildLShr(Ty, Q, PreShift).getReg(0);
5604 Q = MIB.buildUMulH(Ty, Q, MagicFactor).getReg(0);
5607 Register NPQ = MIB.buildSub(Ty, LHS, Q).getReg(0);
5612 NPQ = MIB.buildUMulH(Ty, NPQ, NPQFactor).getReg(0);
5614 NPQ = MIB.buildLShr(Ty, NPQ, MIB.buildConstant(ShiftAmtTy, 1)).getReg(0);
5616 Q = MIB.buildAdd(Ty, NPQ, Q).getReg(0);
5619 Q = MIB.buildLShr(Ty, Q, PostShift).getReg(0);
5620 auto One = MIB.buildConstant(Ty, 1);
5621 auto IsOne = MIB.buildICmp(
5625 auto ret = MIB.buildSelect(Ty, IsOne, LHS, Q);
5627 if (Opcode == TargetOpcode::G_UREM) {
5628 auto Prod = MIB.buildMul(Ty, ret, RHS);
5629 return MIB.buildSub(Ty, LHS, Prod);
5635 unsigned Opcode =
MI.getOpcode();
5636 assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5639 LLT DstTy =
MRI.getType(Dst);
5641 auto &MF = *
MI.getMF();
5651 if (MF.getFunction().hasMinSize())
5654 if (Opcode == TargetOpcode::G_UDIV &&
5657 MRI, RHS, [](
const Constant *
C) {
return C && !
C->isNullValue(); });
5672 {TargetOpcode::G_ICMP,
5676 if (Opcode == TargetOpcode::G_UREM &&
5682 MRI, RHS, [](
const Constant *
C) {
return C && !
C->isNullValue(); });
5691 unsigned Opcode =
MI.getOpcode();
5692 assert(Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM);
5695 LLT DstTy =
MRI.getType(Dst);
5699 auto &MF = *
MI.getMF();
5709 if (MF.getFunction().hasMinSize())
5713 if (Opcode == TargetOpcode::G_SDIV &&
5716 MRI, RHS, [](
const Constant *
C) {
return C && !
C->isNullValue(); });
5734 }
else if (!
isLegal({TargetOpcode::G_SMULH, {DstTy}})) {
5738 if (Opcode == TargetOpcode::G_SREM &&
5744 MRI, RHS, [](
const Constant *
C) {
return C && !
C->isNullValue(); });
5753 unsigned Opcode =
MI.getOpcode();
5754 assert(
MI.getOpcode() == TargetOpcode::G_SDIV ||
5755 Opcode == TargetOpcode::G_SREM);
5757 Register Dst = SDivorRem.getReg(0);
5758 Register LHS = SDivorRem.getReg(1);
5759 Register RHS = SDivorRem.getReg(2);
5760 LLT Ty =
MRI.getType(Dst);
5767 bool UseSRA =
false;
5773 auto BuildExactSDIVPattern = [&](
const Constant *
C) {
5775 if (IsSplat && !ExactFactors.
empty()) {
5777 ExactFactors.
push_back(ExactFactors[0]);
5782 APInt Divisor = CI->getValue();
5792 ExactShifts.
push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));
5793 ExactFactors.
push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));
5801 assert(Matched &&
"Expected unary predicate match to succeed");
5804 if (Ty.isVector()) {
5805 Shift = MIB.buildBuildVector(ShiftAmtTy, ExactShifts).getReg(0);
5806 Factor = MIB.buildBuildVector(Ty, ExactFactors).getReg(0);
5808 Shift = ExactShifts[0];
5809 Factor = ExactFactors[0];
5817 return MIB.buildMul(Ty, Res, Factor);
5822 auto BuildSDIVPattern = [&](
const Constant *
C) {
5824 const APInt &Divisor = CI->getValue();
5828 int NumeratorFactor = 0;
5839 NumeratorFactor = 1;
5842 NumeratorFactor = -1;
5845 MagicFactors.
push_back(MIB.buildConstant(ScalarTy, Magics.
Magic).getReg(0));
5846 Factors.
push_back(MIB.buildConstant(ScalarTy, NumeratorFactor).getReg(0));
5848 MIB.buildConstant(ScalarShiftAmtTy, Magics.
ShiftAmount).getReg(0));
5849 ShiftMasks.
push_back(MIB.buildConstant(ScalarTy, ShiftMask).getReg(0));
5857 assert(Matched &&
"Expected unary predicate match to succeed");
5859 Register MagicFactor, Factor, Shift, ShiftMask;
5862 MagicFactor = MIB.buildBuildVector(Ty, MagicFactors).getReg(0);
5863 Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);
5864 Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);
5865 ShiftMask = MIB.buildBuildVector(Ty, ShiftMasks).getReg(0);
5868 "Non-build_vector operation should have been a scalar");
5869 MagicFactor = MagicFactors[0];
5870 Factor = Factors[0];
5872 ShiftMask = ShiftMasks[0];
5876 Q = MIB.buildSMulH(Ty, LHS, MagicFactor).getReg(0);
5879 Factor = MIB.buildMul(Ty, LHS, Factor).getReg(0);
5880 Q = MIB.buildAdd(Ty, Q, Factor).getReg(0);
5883 Q = MIB.buildAShr(Ty, Q, Shift).getReg(0);
5886 auto SignShift = MIB.buildConstant(ShiftAmtTy, EltBits - 1);
5887 auto T = MIB.buildLShr(Ty, Q, SignShift);
5888 T = MIB.buildAnd(Ty,
T, ShiftMask);
5889 auto ret = MIB.buildAdd(Ty, Q,
T);
5891 if (Opcode == TargetOpcode::G_SREM) {
5892 auto Prod = MIB.buildMul(Ty, ret, RHS);
5893 return MIB.buildSub(Ty, LHS, Prod);
5899 assert((
MI.getOpcode() == TargetOpcode::G_SDIV ||
5900 MI.getOpcode() == TargetOpcode::G_UDIV) &&
5901 "Expected SDIV or UDIV");
5904 auto MatchPow2 = [&](
const Constant *
C) {
5906 return CI && (CI->getValue().isPowerOf2() ||
5907 (IsSigned && CI->getValue().isNegatedPowerOf2()));
5913 assert(
MI.getOpcode() == TargetOpcode::G_SDIV &&
"Expected SDIV");
5918 LLT Ty =
MRI.getType(Dst);
5938 unsigned BitWidth = Ty.getScalarSizeInBits();
5942 auto C1 =
Builder.buildCTTZ(ShiftAmtTy, RHS);
5943 auto Inexact =
Builder.buildSub(ShiftAmtTy, Bits,
C1);
5955 auto One =
Builder.buildConstant(Ty, 1);
5956 auto MinusOne =
Builder.buildConstant(Ty, -1);
5960 auto IsOneOrMinusOne =
Builder.buildOr(CCVT, IsOne, IsMinusOne);
5961 AShr =
Builder.buildSelect(Ty, IsOneOrMinusOne, LHS, AShr);
5965 auto Neg =
Builder.buildNeg(Ty, AShr);
5967 Builder.buildSelect(
MI.getOperand(0).getReg(), IsNeg, Neg, AShr);
5968 MI.eraseFromParent();
5972 assert(
MI.getOpcode() == TargetOpcode::G_UDIV &&
"Expected UDIV");
5977 LLT Ty =
MRI.getType(Dst);
5980 auto C1 =
Builder.buildCTTZ(ShiftAmtTy, RHS);
5981 Builder.buildLShr(
MI.getOperand(0).getReg(), LHS,
C1);
5982 MI.eraseFromParent();
5996 return CI && CI->getValue().isPowerOf2();
6000 LLT Ty =
MRI.getType(Dst);
6001 LLT ShiftTy =
MRI.getType(Shift);
6014 auto Amount =
B.buildAdd(ShiftTy, Shift,
Log2);
6015 B.buildLShr(Dst, LHS, Amount, Flags);
6025 if (!
VT || !
VT->signBitIsZero(LHS) || !
VT->signBitIsZero(RHS))
6028 LLT Ty =
MRI.getType(Dst);
6033 assert(
MI.getOpcode() == TargetOpcode::G_SREM &&
"Expected SREM");
6038 LLT Ty =
MRI.getType(Dst);
6057 unsigned BitWidth = Ty.getScalarSizeInBits();
6058 auto AbsRHS =
Builder.buildAbs(Ty, RHS);
6059 auto Mask =
Builder.buildSub(Ty, AbsRHS,
Builder.buildConstant(Ty, 1));
6061 auto Sign =
Builder.buildAShr(Ty, LHS, BWMinusOne);
6063 auto Biased =
Builder.buildAdd(Ty, LHS, Bias);
6066 MI.eraseFromParent();
6070 assert(
MI.getOpcode() == TargetOpcode::G_UMULH);
6073 LLT Ty =
MRI.getType(Dst);
6074 LLT RHSTy =
MRI.getType(RHS);
6076 auto MatchPow2ExceptOne = [&](
const Constant *
C) {
6078 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();
6093 LLT Ty =
MRI.getType(Dst);
6099 Builder.buildSub(Ty,
Builder.buildConstant(Ty, NumEltBits), LogBase2);
6100 auto Trunc =
Builder.buildZExtOrTrunc(ShiftAmtTy, ShiftAmt);
6101 Builder.buildLShr(Dst, LHS, Trunc);
6102 MI.eraseFromParent();
6109 LLT DstTy =
MRI.getType(Dst);
6110 LLT SrcTy =
MRI.getType(Src);
6112 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6113 assert(NumSrcBits > NumDstBits &&
"Unexpected types for truncate operation");
6116 {TargetOpcode::G_TRUNC_SSAT_S, {DstTy, SrcTy}}))
6145 Builder.buildTruncSSatS(Dst, MatchInfo);
6146 MI.eraseFromParent();
6153 LLT DstTy =
MRI.getType(Dst);
6154 LLT SrcTy =
MRI.getType(Src);
6156 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6157 assert(NumSrcBits > NumDstBits &&
"Unexpected types for truncate operation");
6160 {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6178 Builder.buildTruncSSatU(Dst, MatchInfo);
6179 MI.eraseFromParent();
6186 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
6187 LLT SrcTy =
MRI.getType(Val);
6189 unsigned NumSrcBits = SrcTy.getScalarSizeInBits();
6190 assert(NumSrcBits > NumDstBits &&
"Unexpected types for truncate operation");
6193 {TargetOpcode::G_TRUNC_SSAT_U, {DstTy, SrcTy}}))
6202 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
6210 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
6213 LLT Ty =
MRI.getType(
MI.getOperand(0).getReg());
6215 const auto LHSCst = Ty.isVector()
6229 if (LHSCst->Value.isNegZero())
6233 if (LHSCst->Value.isPosZero())
6242 if (
MI.getOpcode() != TargetOpcode::G_FMUL)
6250 MRI.use_instr_nodbg_end()) >
6252 MRI.use_instr_nodbg_end());
6256 bool &AllowFusionGlobally,
6258 bool CanReassociate)
const {
6260 auto *MF =
MI.getMF();
6261 const auto &
TLI = *MF->getSubtarget().getTargetLowering();
6262 LLT DstType =
MRI.getType(
MI.getOperand(0).getReg());
6270 bool HasFMA =
TLI.isFMAFasterThanFMulAndFAdd(*MF, DstType) &&
6273 if (!HasFMAD && !HasFMA)
6278 AllowFusionGlobally = HasFMAD;
6290 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
6292 bool AllowFusionGlobally, HasFMAD,
Aggressive;
6304 unsigned PreferredFusedOpcode =
6305 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6318 unsigned Flags =
MI.getFlags() & LHS.MI->getFlags();
6320 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6321 {LHS.MI->getOperand(1).getReg(),
6322 LHS.MI->getOperand(2).getReg(), RHS.Reg},
6331 unsigned Flags =
MI.getFlags() & RHS.MI->getFlags();
6333 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6334 {RHS.MI->getOperand(1).getReg(),
6335 RHS.MI->getOperand(2).getReg(), LHS.Reg},
6347 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
6349 bool AllowFusionGlobally, HasFMAD,
Aggressive;
6353 const auto &
TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
6362 LLT DstType =
MRI.getType(
MI.getOperand(0).getReg());
6364 unsigned PreferredFusedOpcode =
6365 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6368 bool LHSContractable =
6371 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
6374 bool RHSContractable =
6377 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
6381 if (LHSContractable || RHSContractable) {
6384 if (!LHSContractable ||
6385 (RHSContractable &&
hasMoreUses(*LHSFpExtSrc, *RHSFpExtSrc,
MRI))) {
6387 LHSFpExtSrc = RHSFpExtSrc;
6390 unsigned Flags =
MI.getFlags() & LHSFpExtSrc->
getFlags();
6394 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6395 {FpExtX.getReg(0), FpExtY.getReg(0), RHS.Reg}, Flags);
6406 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
6408 bool AllowFusionGlobally, HasFMAD,
Aggressive;
6420 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
6422 unsigned PreferredFusedOpcode =
6423 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6428 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6431 MRI.hasOneNonDBGUse(LHS.MI->getOperand(0).getReg()) &&
6432 MRI.hasOneNonDBGUse(LHS.MI->getOperand(3).getReg())) {
6437 else if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6440 MRI.hasOneNonDBGUse(RHS.MI->getOperand(0).getReg()) &&
6441 MRI.hasOneNonDBGUse(RHS.MI->getOperand(3).getReg())) {
6450 Register X = FMA->getOperand(1).getReg();
6451 Register Y = FMA->getOperand(2).getReg();
6454 unsigned InnerFlags =
MI.getFlags() & FMulMI->
getFlags();
6455 unsigned OuterFlags =
MI.getFlags() & FMA->getFlags();
6458 Register InnerFMA =
MRI.createGenericVirtualRegister(DstTy);
6459 B.buildInstr(PreferredFusedOpcode, {InnerFMA}, {U, V, Z}, InnerFlags);
6460 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6461 {
X,
Y, InnerFMA}, OuterFlags);
6472 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
6474 bool AllowFusionGlobally, HasFMAD,
Aggressive;
6481 const auto &
TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
6482 LLT DstType =
MRI.getType(
MI.getOperand(0).getReg());
6492 unsigned PreferredFusedOpcode =
6493 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6507 Register FpExtU =
B.buildFPExt(DstType, U).getReg(0);
6508 Register FpExtV =
B.buildFPExt(DstType, V).getReg(0);
6509 Register InnerFMA =
B.buildInstr(PreferredFusedOpcode, {DstType},
6510 {FpExtU, FpExtV, Z}, InnerFlags)
6512 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6513 {
X,
Y, InnerFMA}, OuterFlags);
6519 if (LHS.MI->getOpcode() == PreferredFusedOpcode &&
6523 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
6525 unsigned InnerFlags =
MI.getFlags() & FMulMI->
getFlags();
6526 unsigned OuterFlags =
MI.getFlags() & LHS.MI->getFlags();
6530 LHS.MI->getOperand(1).getReg(),
6531 LHS.MI->getOperand(2).getReg(), InnerFlags, OuterFlags,
B);
6542 FMAMI->
getOpcode() == PreferredFusedOpcode) {
6547 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
6549 unsigned InnerFlags =
MI.getFlags() & FMulMI->
getFlags();
6550 unsigned OuterFlags =
MI.getFlags() & FMAMI->
getFlags();
6554 X =
B.buildFPExt(DstType,
X).getReg(0);
6555 Y =
B.buildFPExt(DstType,
Y).getReg(0);
6558 InnerFlags, OuterFlags,
B);
6567 if (RHS.MI->getOpcode() == PreferredFusedOpcode &&
6571 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
6573 unsigned InnerFlags =
MI.getFlags() & FMulMI->
getFlags();
6574 unsigned OuterFlags =
MI.getFlags() & RHS.MI->getFlags();
6578 RHS.MI->getOperand(1).getReg(),
6579 RHS.MI->getOperand(2).getReg(), InnerFlags, OuterFlags,
B);
6590 FMAMI->
getOpcode() == PreferredFusedOpcode) {
6595 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
6597 unsigned InnerFlags =
MI.getFlags() & FMulMI->
getFlags();
6598 unsigned OuterFlags =
MI.getFlags() & FMAMI->
getFlags();
6602 X =
B.buildFPExt(DstType,
X).getReg(0);
6603 Y =
B.buildFPExt(DstType,
Y).getReg(0);
6606 InnerFlags, OuterFlags,
B);
6618 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
6620 bool AllowFusionGlobally, HasFMAD,
Aggressive;
6632 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
6636 int FirstMulHasFewerUses =
true;
6640 FirstMulHasFewerUses =
false;
6642 unsigned PreferredFusedOpcode =
6643 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6646 if (FirstMulHasFewerUses &&
6649 unsigned Flags =
MI.getFlags() & LHS.MI->getFlags();
6651 Register NegZ =
B.buildFNeg(DstTy, RHS.Reg).getReg(0);
6652 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6653 {LHS.MI->getOperand(1).getReg(),
6654 LHS.MI->getOperand(2).getReg(), NegZ},
6662 unsigned Flags =
MI.getFlags() & RHS.MI->getFlags();
6665 B.buildFNeg(DstTy, RHS.MI->getOperand(1).getReg()).getReg(0);
6666 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6667 {NegY, RHS.MI->getOperand(2).getReg(), LHS.Reg}, Flags);
6678 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
6680 bool AllowFusionGlobally, HasFMAD,
Aggressive;
6686 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
6688 unsigned PreferredFusedOpcode =
6689 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6697 unsigned Flags =
MI.getFlags() & FMulMI->
getFlags();
6701 Register NegZ =
B.buildFNeg(DstTy, RHSReg).getReg(0);
6702 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6713 unsigned Flags =
MI.getFlags() & FMulMI->
getFlags();
6715 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6729 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
6731 bool AllowFusionGlobally, HasFMAD,
Aggressive;
6737 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
6739 unsigned PreferredFusedOpcode =
6740 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6747 unsigned Flags =
MI.getFlags() & FMulMI->
getFlags();
6753 Register NegZ =
B.buildFNeg(DstTy, RHSReg).getReg(0);
6754 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6755 {FpExtX, FpExtY, NegZ}, Flags);
6764 unsigned Flags =
MI.getFlags() & FMulMI->
getFlags();
6768 Register NegY =
B.buildFNeg(DstTy, FpExtY).getReg(0);
6771 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
6772 {NegY, FpExtZ, LHSReg}, Flags);
6783 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
6785 bool AllowFusionGlobally, HasFMAD,
Aggressive;
6789 const auto &
TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
6790 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
6794 unsigned PreferredFusedOpcode =
6795 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6799 Register FpExtX =
B.buildFPExt(DstTy,
X).getReg(0);
6800 Register FpExtY =
B.buildFPExt(DstTy,
Y).getReg(0);
6801 B.buildInstr(PreferredFusedOpcode, {Dst}, {FpExtX, FpExtY, Z}, Flags);
6812 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstTy,
6814 unsigned Flags =
MI.getFlags() & FMulMI->
getFlags();
6816 Register FMAReg =
MRI.createGenericVirtualRegister(DstTy);
6819 B.buildFNeg(
MI.getOperand(0).getReg(), FMAReg);
6829 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstTy,
6831 unsigned Flags =
MI.getFlags() & FMulMI->
getFlags();
6847 assert(
MI.getOpcode() == TargetOpcode::G_FDIV);
6857 return N0CFP && (N0CFP->isOne() || N0CFP->isMinusOne());
6874 for (
auto &U :
MRI.use_nodbg_instructions(
Y)) {
6875 if (&U == &
MI || U.getParent() !=
MI.getParent())
6877 if (U.getOpcode() == TargetOpcode::G_FDIV &&
6878 U.getOperand(2).getReg() ==
Y && U.getOperand(1).getReg() !=
Y &&
6879 !IsOne(U.getOperand(1).getReg())) {
6892 return MatchInfo.
size() >= MinUses;
6900 LLT Ty =
MRI.getType(MatchInfo[0]->getOperand(0).
getReg());
6901 auto Div =
Builder.buildFDiv(Ty,
Builder.buildFConstant(Ty, 1.0),
6902 MatchInfo[0]->getOperand(2).getReg(),
6903 MatchInfo[0]->getFlags());
6908 Builder.buildFMul(
MI->getOperand(0).getReg(),
MI->getOperand(1).getReg(),
6909 Div->getOperand(0).getReg(),
MI->getFlags());
6910 MI->eraseFromParent();
6932 LLT DstVecTy =
MRI.getType(
MI.getOperand(0).getReg());
6941 return MRI.getType(MatchInfo) == DstVecTy;
6944 std::optional<ValueAndVReg> ShiftAmount;
6953 return MRI.getType(MatchInfo) == DstVecTy;
6968 return MRI.getType(MatchInfo) ==
MRI.getType(
MI.getOperand(0).getReg());
6975 std::optional<ValueAndVReg> ShiftAmt;
6981 LLT MatchTy =
MRI.getType(MatchInfo);
6982 return ShiftAmt->Value.getZExtValue() == MatchTy.
getSizeInBits() &&
6983 MatchTy ==
MRI.getType(
MI.getOperand(0).getReg());
6986unsigned CombinerHelper::getFPMinMaxOpcForSelect(
6988 SelectPatternNaNBehaviour VsNaNRetVal)
const {
6989 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&
6990 "Expected a NaN behaviour?");
7000 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7001 return TargetOpcode::G_FMAXNUM;
7002 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7003 return TargetOpcode::G_FMAXIMUM;
7004 if (
isLegal({TargetOpcode::G_FMAXNUM, {DstTy}}))
7005 return TargetOpcode::G_FMAXNUM;
7006 if (
isLegal({TargetOpcode::G_FMAXIMUM, {DstTy}}))
7007 return TargetOpcode::G_FMAXIMUM;
7013 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
7014 return TargetOpcode::G_FMINNUM;
7015 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
7016 return TargetOpcode::G_FMINIMUM;
7017 if (
isLegal({TargetOpcode::G_FMINNUM, {DstTy}}))
7018 return TargetOpcode::G_FMINNUM;
7019 if (!
isLegal({TargetOpcode::G_FMINIMUM, {DstTy}}))
7021 return TargetOpcode::G_FMINIMUM;
7025CombinerHelper::SelectPatternNaNBehaviour
7027 bool IsOrderedComparison)
const {
7028 bool LHSSafe =
VT->isKnownNeverNaN(
LHS);
7029 bool RHSSafe =
VT->isKnownNeverNaN(
RHS);
7031 if (!LHSSafe && !RHSSafe)
7032 return SelectPatternNaNBehaviour::NOT_APPLICABLE;
7033 if (LHSSafe && RHSSafe)
7034 return SelectPatternNaNBehaviour::RETURNS_ANY;
7037 if (IsOrderedComparison)
7038 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN
7039 : SelectPatternNaNBehaviour::RETURNS_OTHER;
7042 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER
7043 : SelectPatternNaNBehaviour::RETURNS_NAN;
7052 LLT DstTy =
MRI.getType(Dst);
7065 SelectPatternNaNBehaviour ResWithKnownNaNInfo =
7067 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)
7069 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
7072 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)
7073 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;
7074 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)
7075 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;
7077 if (TrueVal != CmpLHS || FalseVal != CmpRHS)
7080 unsigned Opc = getFPMinMaxOpcForSelect(
Pred, DstTy, ResWithKnownNaNInfo);
7085 if (
Opc != TargetOpcode::G_FMAXIMUM &&
Opc != TargetOpcode::G_FMINIMUM) {
7090 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {
7092 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())
7096 MatchInfo = [=](MachineIRBuilder &
B) {
7097 B.buildInstr(
Opc, {Dst}, {CmpLHS, CmpRHS});
7105 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
7112 Register TrueVal =
MI.getOperand(2).getReg();
7113 Register FalseVal =
MI.getOperand(3).getReg();
7114 return matchFPSelectToMinMax(Dst,
Cond, TrueVal, FalseVal, MatchInfo);
7119 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
7132 if (MatchedSub &&
X != OpLHS)
7140 Y =
X == OpLHS ? OpRHS :
X == OpRHS ? OpLHS :
Register();
7143 auto Zero =
B.buildConstant(
MRI.getType(
Y), 0);
7144 B.buildICmp(Pred, Dst,
Y,
Zero);
7151static std::optional<unsigned>
7153 std::optional<int64_t> &Result) {
7154 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR ||
7155 Opcode == TargetOpcode::G_ASHR) &&
7156 "Expect G_SHL, G_LSHR or G_ASHR.");
7157 auto SignificantBits = 0;
7159 case TargetOpcode::G_SHL:
7163 case TargetOpcode::G_LSHR:
7167 case TargetOpcode::G_ASHR:
7176 Result = std::nullopt;
7188 Register ShiftVal =
MI.getOperand(1).getReg();
7189 Register ShiftReg =
MI.getOperand(2).getReg();
7190 LLT ResTy =
MRI.getType(Dst);
7191 std::optional<int64_t> Result;
7192 auto IsShiftTooBig = [&](
const Constant *
C) {
7197 Result = std::nullopt;
7200 auto OptMaxUsefulShift =
7202 return OptMaxUsefulShift && CI->uge(*OptMaxUsefulShift);
7212 int64_t Val = *Result;
7218 unsigned LHSOpndIdx = 1;
7219 unsigned RHSOpndIdx = 2;
7220 switch (
MI.getOpcode()) {
7221 case TargetOpcode::G_UADDO:
7222 case TargetOpcode::G_SADDO:
7223 case TargetOpcode::G_UMULO:
7224 case TargetOpcode::G_SMULO:
7231 Register LHS =
MI.getOperand(LHSOpndIdx).getReg();
7232 Register RHS =
MI.getOperand(RHSOpndIdx).getReg();
7243 if (LHSDef->
getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER)
7248 return RHSDef->
getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER &&
7256 std::optional<FPValueAndVReg> ValAndVReg;
7262bool CombinerHelper::isOneOrOneSplat(
Register Src,
bool AllowUndefs)
const {
7263 LLT SrcTy =
MRI.getType(Src);
7264 if (SrcTy.isFixedVector())
7266 if (SrcTy.isScalar()) {
7270 return IConstant && IConstant->Value == 1;
7275bool CombinerHelper::isZeroOrZeroSplat(
Register Src,
bool AllowUndefs)
const {
7276 LLT SrcTy =
MRI.getType(Src);
7278 return isConstantSplatVector(Src, 0, AllowUndefs);
7283 return IConstant && IConstant->Value == 0;
7290bool CombinerHelper::isConstantSplatVector(
Register Src, int64_t SplatValue,
7291 bool AllowUndefs)
const {
7297 for (
unsigned I = 0;
I < NumSources; ++
I) {
7298 GImplicitDef *ImplicitDef =
7300 if (ImplicitDef && AllowUndefs)
7302 if (ImplicitDef && !AllowUndefs)
7304 std::optional<ValueAndVReg> IConstant =
7306 if (IConstant && IConstant->Value == SplatValue)
7316CombinerHelper::getConstantOrConstantSplatVector(
Register Src)
const {
7319 return IConstant->Value;
7323 return std::nullopt;
7326 std::optional<APInt>
Value = std::nullopt;
7327 for (
unsigned I = 0;
I < NumSources; ++
I) {
7328 std::optional<ValueAndVReg> IConstant =
7331 return std::nullopt;
7333 Value = IConstant->Value;
7334 else if (*
Value != IConstant->Value)
7335 return std::nullopt;
7341bool CombinerHelper::isConstantOrConstantVectorI(
Register Src)
const {
7351 for (
unsigned I = 0;
I < NumSources; ++
I) {
7352 std::optional<ValueAndVReg> IConstant =
7361bool CombinerHelper::tryFoldSelectOfConstants(
GSelect *
Select,
7368 LLT CondTy =
MRI.getType(
Select->getCondReg());
7369 LLT TrueTy =
MRI.getType(
Select->getTrueReg());
7379 std::optional<ValueAndVReg> TrueOpt =
7381 std::optional<ValueAndVReg> FalseOpt =
7384 if (!TrueOpt || !FalseOpt)
7387 APInt TrueValue = TrueOpt->Value;
7388 APInt FalseValue = FalseOpt->Value;
7392 MatchInfo = [=](MachineIRBuilder &
B) {
7393 B.setInstrAndDebugLoc(*
Select);
7394 B.buildZExtOrTrunc(Dest,
Cond);
7401 MatchInfo = [=](MachineIRBuilder &
B) {
7402 B.setInstrAndDebugLoc(*
Select);
7403 B.buildSExtOrTrunc(Dest,
Cond);
7410 MatchInfo = [=](MachineIRBuilder &
B) {
7411 B.setInstrAndDebugLoc(*
Select);
7412 Register Inner =
MRI.createGenericVirtualRegister(CondTy);
7413 B.buildNot(Inner,
Cond);
7414 B.buildZExtOrTrunc(Dest, Inner);
7421 MatchInfo = [=](MachineIRBuilder &
B) {
7422 B.setInstrAndDebugLoc(*
Select);
7423 Register Inner =
MRI.createGenericVirtualRegister(CondTy);
7424 B.buildNot(Inner,
Cond);
7425 B.buildSExtOrTrunc(Dest, Inner);
7431 if (TrueValue - 1 == FalseValue) {
7432 MatchInfo = [=](MachineIRBuilder &
B) {
7433 B.setInstrAndDebugLoc(*
Select);
7434 Register Inner =
MRI.createGenericVirtualRegister(TrueTy);
7435 B.buildZExtOrTrunc(Inner,
Cond);
7436 B.buildAdd(Dest, Inner,
False);
7442 if (TrueValue + 1 == FalseValue) {
7443 MatchInfo = [=](MachineIRBuilder &
B) {
7444 B.setInstrAndDebugLoc(*
Select);
7445 Register Inner =
MRI.createGenericVirtualRegister(TrueTy);
7446 B.buildSExtOrTrunc(Inner,
Cond);
7447 B.buildAdd(Dest, Inner,
False);
7454 MatchInfo = [=](MachineIRBuilder &
B) {
7455 B.setInstrAndDebugLoc(*
Select);
7456 Register Inner =
MRI.createGenericVirtualRegister(TrueTy);
7457 B.buildZExtOrTrunc(Inner,
Cond);
7460 auto ShAmtC =
B.buildConstant(ShiftTy, TrueValue.
exactLogBase2());
7461 B.buildShl(Dest, Inner, ShAmtC, Flags);
7468 MatchInfo = [=](MachineIRBuilder &
B) {
7469 B.setInstrAndDebugLoc(*
Select);
7471 B.buildNot(Not,
Cond);
7472 Register Inner =
MRI.createGenericVirtualRegister(TrueTy);
7473 B.buildZExtOrTrunc(Inner, Not);
7476 auto ShAmtC =
B.buildConstant(ShiftTy, FalseValue.
exactLogBase2());
7477 B.buildShl(Dest, Inner, ShAmtC, Flags);
7484 MatchInfo = [=](MachineIRBuilder &
B) {
7485 B.setInstrAndDebugLoc(*
Select);
7486 Register Inner =
MRI.createGenericVirtualRegister(TrueTy);
7487 B.buildSExtOrTrunc(Inner,
Cond);
7488 B.buildOr(Dest, Inner,
False, Flags);
7495 MatchInfo = [=](MachineIRBuilder &
B) {
7496 B.setInstrAndDebugLoc(*
Select);
7498 B.buildNot(Not,
Cond);
7499 Register Inner =
MRI.createGenericVirtualRegister(TrueTy);
7500 B.buildSExtOrTrunc(Inner, Not);
7501 B.buildOr(Dest, Inner,
True, Flags);
7510bool CombinerHelper::tryFoldBoolSelectToLogic(
GSelect *
Select,
7517 LLT CondTy =
MRI.getType(
Select->getCondReg());
7518 LLT TrueTy =
MRI.getType(
Select->getTrueReg());
7527 if (CondTy != TrueTy)
7533 MatchInfo = [=](MachineIRBuilder &
B) {
7534 B.setInstrAndDebugLoc(*
Select);
7535 Register Ext =
MRI.createGenericVirtualRegister(TrueTy);
7536 B.buildZExtOrTrunc(Ext,
Cond);
7537 auto FreezeFalse =
B.buildFreeze(TrueTy,
False);
7538 B.buildOr(DstReg, Ext, FreezeFalse, Flags);
7546 MatchInfo = [=](MachineIRBuilder &
B) {
7547 B.setInstrAndDebugLoc(*
Select);
7548 Register Ext =
MRI.createGenericVirtualRegister(TrueTy);
7549 B.buildZExtOrTrunc(Ext,
Cond);
7550 auto FreezeTrue =
B.buildFreeze(TrueTy,
True);
7551 B.buildAnd(DstReg, Ext, FreezeTrue);
7557 if (isOneOrOneSplat(
False,
true)) {
7558 MatchInfo = [=](MachineIRBuilder &
B) {
7559 B.setInstrAndDebugLoc(*
Select);
7561 Register Inner =
MRI.createGenericVirtualRegister(CondTy);
7562 B.buildNot(Inner,
Cond);
7564 Register Ext =
MRI.createGenericVirtualRegister(TrueTy);
7565 B.buildZExtOrTrunc(Ext, Inner);
7566 auto FreezeTrue =
B.buildFreeze(TrueTy,
True);
7567 B.buildOr(DstReg, Ext, FreezeTrue, Flags);
7573 if (isZeroOrZeroSplat(
True,
true)) {
7574 MatchInfo = [=](MachineIRBuilder &
B) {
7575 B.setInstrAndDebugLoc(*
Select);
7577 Register Inner =
MRI.createGenericVirtualRegister(CondTy);
7578 B.buildNot(Inner,
Cond);
7580 Register Ext =
MRI.createGenericVirtualRegister(TrueTy);
7581 B.buildZExtOrTrunc(Ext, Inner);
7582 auto FreezeFalse =
B.buildFreeze(TrueTy,
False);
7583 B.buildAnd(DstReg, Ext, FreezeFalse);
7605 LLT DstTy =
MRI.getType(DstReg);
7610 if (!
MRI.hasOneNonDBGUse(CondReg))
7619 if (
True == CmpRHS &&
False == CmpLHS) {
7667 assert(
MI.getOpcode() == TargetOpcode::G_SUB);
7668 Register DestReg =
MI.getOperand(0).getReg();
7669 LLT DestTy =
MRI.getType(DestReg);
7683 if (
isLegal({NewOpc, {DestTy}})) {
7685 B.buildInstr(NewOpc, {DestReg}, {
X, Sub0});
7697 if (tryFoldSelectOfConstants(
Select, MatchInfo))
7700 if (tryFoldBoolSelectToLogic(
Select, MatchInfo))
7710bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(
7712 assert(Logic->
getOpcode() != TargetOpcode::G_XOR &&
"unexpected xor");
7713 bool IsAnd = Logic->
getOpcode() == TargetOpcode::G_AND;
7717 unsigned Flags = Logic->
getFlags();
7730 if (!
MRI.hasOneNonDBGUse(Cmp1->
getReg(0)) ||
7736 std::optional<ValueAndVReg> MaybeC1 =
7740 C1 = MaybeC1->Value;
7742 std::optional<ValueAndVReg> MaybeC2 =
7746 C2 = MaybeC2->Value;
7753 LLT CmpOperandTy =
MRI.getType(R1);
7767 std::optional<APInt> Offset1;
7768 std::optional<APInt> Offset2;
7771 std::optional<ValueAndVReg> MaybeOffset1 =
7774 R1 =
Add->getLHSReg();
7775 Offset1 = MaybeOffset1->Value;
7779 std::optional<ValueAndVReg> MaybeOffset2 =
7782 R2 =
Add->getLHSReg();
7783 Offset2 = MaybeOffset2->Value;
7802 bool CreateMask =
false;
7815 if (!LowerDiff.
isPowerOf2() || LowerDiff != UpperDiff ||
7828 CR->getEquivalentICmp(NewPred, NewC,
Offset);
7837 MatchInfo = [=](MachineIRBuilder &
B) {
7838 if (CreateMask &&
Offset != 0) {
7839 auto TildeLowerDiff =
B.buildConstant(CmpOperandTy, ~LowerDiff);
7840 auto And =
B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);
7841 auto OffsetC =
B.buildConstant(CmpOperandTy,
Offset);
7842 auto Add =
B.buildAdd(CmpOperandTy,
And, OffsetC, Flags);
7843 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
7844 auto ICmp =
B.buildICmp(NewPred, CmpTy,
Add, NewCon);
7845 B.buildZExtOrTrunc(DstReg, ICmp);
7846 }
else if (CreateMask &&
Offset == 0) {
7847 auto TildeLowerDiff =
B.buildConstant(CmpOperandTy, ~LowerDiff);
7848 auto And =
B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);
7849 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
7850 auto ICmp =
B.buildICmp(NewPred, CmpTy,
And, NewCon);
7851 B.buildZExtOrTrunc(DstReg, ICmp);
7852 }
else if (!CreateMask &&
Offset != 0) {
7853 auto OffsetC =
B.buildConstant(CmpOperandTy,
Offset);
7854 auto Add =
B.buildAdd(CmpOperandTy, R1, OffsetC, Flags);
7855 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
7856 auto ICmp =
B.buildICmp(NewPred, CmpTy,
Add, NewCon);
7857 B.buildZExtOrTrunc(DstReg, ICmp);
7858 }
else if (!CreateMask &&
Offset == 0) {
7859 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
7860 auto ICmp =
B.buildICmp(NewPred, CmpTy, R1, NewCon);
7861 B.buildZExtOrTrunc(DstReg, ICmp);
7869bool CombinerHelper::tryFoldLogicOfFCmps(
GLogicalBinOp *Logic,
7875 bool IsAnd = Logic->
getOpcode() == TargetOpcode::G_AND;
7887 LLT CmpTy =
MRI.getType(Cmp1->
getReg(0));
7893 {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||
7894 !
MRI.hasOneNonDBGUse(Logic->
getReg(0)) ||
7895 !
MRI.hasOneNonDBGUse(Cmp1->
getReg(0)) ||
7896 !
MRI.hasOneNonDBGUse(Cmp2->
getReg(0)) ||
7907 if (LHS0 == RHS1 && LHS1 == RHS0) {
7913 if (LHS0 == RHS0 && LHS1 == RHS1) {
7917 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;
7919 MatchInfo = [=](MachineIRBuilder &
B) {
7924 auto False =
B.buildConstant(CmpTy, 0);
7925 B.buildZExtOrTrunc(DestReg,
False);
7932 B.buildZExtOrTrunc(DestReg,
True);
7934 auto Cmp =
B.buildFCmp(
Pred, CmpTy, LHS0, LHS1, Flags);
7935 B.buildZExtOrTrunc(DestReg, Cmp);
7947 if (tryFoldAndOrOrICmpsUsingRanges(
And, MatchInfo))
7950 if (tryFoldLogicOfFCmps(
And, MatchInfo))
7959 if (tryFoldAndOrOrICmpsUsingRanges(
Or, MatchInfo))
7962 if (tryFoldLogicOfFCmps(
Or, MatchInfo))
7977 bool IsSigned =
Add->isSigned();
7978 LLT DstTy =
MRI.getType(Dst);
7979 LLT CarryTy =
MRI.getType(Carry);
7982 if (isConstantOrConstantVectorI(LHS) && !isConstantOrConstantVectorI(RHS)) {
7985 B.buildSAddo(Dst, Carry, RHS, LHS);
7991 B.buildUAddo(Dst, Carry, RHS, LHS);
7996 std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(LHS);
7997 std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(RHS);
8003 APInt Result = IsSigned ? MaybeLHS->sadd_ov(*MaybeRHS, Overflow)
8004 : MaybeLHS->uadd_ov(*MaybeRHS, Overflow);
8006 B.buildConstant(Dst, Result);
8007 B.buildConstant(Carry, Overflow);
8015 B.buildCopy(Dst, LHS);
8016 B.buildConstant(Carry, 0);
8025 if (MaybeRHS && AddLHS &&
MRI.hasOneNonDBGUse(
Add->getReg(0)) &&
8028 std::optional<APInt> MaybeAddRHS =
8029 getConstantOrConstantSplatVector(AddLHS->
getRHSReg());
8032 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(*MaybeRHS, Overflow)
8033 : MaybeAddRHS->uadd_ov(*MaybeRHS, Overflow);
8037 auto ConstRHS =
B.buildConstant(DstTy, NewC);
8038 B.buildSAddo(Dst, Carry, AddLHS->
getLHSReg(), ConstRHS);
8044 auto ConstRHS =
B.buildConstant(DstTy, NewC);
8045 B.buildUAddo(Dst, Carry, AddLHS->
getLHSReg(), ConstRHS);
8070 B.buildConstant(Carry, 0);
8077 B.buildAdd(Dst, LHS, RHS);
8078 B.buildConstant(Carry, 1);
8090 if (
VT->computeNumSignBits(RHS) > 1 &&
VT->computeNumSignBits(LHS) > 1) {
8093 B.buildConstant(Carry, 0);
8109 B.buildConstant(Carry, 0);
8116 B.buildAdd(Dst, LHS, RHS);
8117 B.buildConstant(Carry, 1);
8135 bool OptForSize =
MI.getMF()->getFunction().hasOptSize();
8141 auto [Dst,
Base] =
MI.getFirst2Regs();
8142 LLT Ty =
MRI.getType(Dst);
8146 Builder.buildFConstant(Dst, 1.0);
8147 MI.removeFromParent();
8159 std::optional<SrcOp> Res;
8161 while (ExpVal > 0) {
8166 Res =
Builder.buildFMul(Ty, *Res, CurSquare);
8169 CurSquare =
Builder.buildFMul(Ty, CurSquare, CurSquare);
8176 Res =
Builder.buildFDiv(Ty,
Builder.buildFConstant(Ty, 1.0), *Res,
8180 MI.eraseFromParent();
8191 if (!
MRI.hasOneNonDBGUse(
Sub->getLHSReg()))
8198 LLT DstTy =
MRI.getType(Dst);
8201 auto Const =
B.buildConstant(DstTy,
C1 - C2);
8202 B.buildAdd(Dst,
A, Const);
8216 if (!
MRI.hasOneNonDBGUse(
Sub->getRHSReg()))
8223 LLT DstTy =
MRI.getType(Dst);
8226 auto Const =
B.buildConstant(DstTy, C2 -
C1);
8227 B.buildSub(Dst, Const,
A);
8248 LLT DstTy =
MRI.getType(Dst);
8251 auto Const =
B.buildConstant(DstTy,
C1 + C2);
8252 B.buildSub(Dst,
A, Const);
8273 LLT DstTy =
MRI.getType(Dst);
8276 auto Const =
B.buildConstant(DstTy,
C1 - C2);
8277 B.buildSub(Dst, Const,
A);
8291 if (!
MRI.hasOneNonDBGUse(
Add->getLHSReg()))
8298 LLT DstTy =
MRI.getType(Dst);
8301 auto Const =
B.buildConstant(DstTy, C2 -
C1);
8302 B.buildAdd(Dst,
A, Const);
8346 if (!
MRI.hasOneNonDBGUse(BV->
getReg(0)))
8354 LLT SmallBvTy = DstTy;
8358 {TargetOpcode::G_BUILD_VECTOR, {SmallBvTy, SmallBvElemenTy}}))
8363 {TargetOpcode::G_ANYEXT,
8375 auto AnyExt =
B.buildAnyExt(SmallBvElemenTy, SourceArray);
8376 Ops.push_back(AnyExt.getReg(0));
8394 const LLT SrcTy =
MRI.getType(Shuffle.getSrc1Reg());
8395 const unsigned NumSrcElems = SrcTy.isVector() ? SrcTy.getNumElements() : 1;
8396 const unsigned NumDstElts = OrigMask.
size();
8397 for (
unsigned i = 0; i != NumDstElts; ++i) {
8398 int Idx = OrigMask[i];
8399 if (Idx >= (
int)NumSrcElems) {
8410 B.buildShuffleVector(
MI.getOperand(0),
MI.getOperand(1),
MI.getOperand(2),
8411 std::move(NewMask));
8418 const unsigned MaskSize = Mask.size();
8419 for (
unsigned I = 0;
I < MaskSize; ++
I) {
8424 if (Idx < (
int)NumElems)
8425 Mask[
I] = Idx + NumElems;
8427 Mask[
I] = Idx - NumElems;
8432 if (!
MRI.getType(Reg).isFixedVector())
8436 return CI && (CI->isZero() || CI->isOne());
8446 if (
getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Shuffle.getSrc1Reg(),
MRI))
8449 if (
getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Shuffle.getSrc2Reg(),
MRI))
8452 const LLT DstTy =
MRI.getType(Shuffle.getReg(0));
8453 const LLT Src1Ty =
MRI.getType(Shuffle.getSrc1Reg());
8455 {TargetOpcode::G_SHUFFLE_VECTOR, {DstTy, Src1Ty}}))
8459 const unsigned NumSrcElems = Src1Ty.getNumElements();
8461 bool TouchesSrc1 =
false;
8462 bool TouchesSrc2 =
false;
8463 const unsigned NumElems = Mask.size();
8464 for (
unsigned Idx = 0; Idx < NumElems; ++Idx) {
8468 if (Mask[Idx] < (
int)NumSrcElems)
8474 if (TouchesSrc1 == TouchesSrc2)
8477 Register NewSrc1 = Shuffle.getSrc1Reg();
8480 NewSrc1 = Shuffle.getSrc2Reg();
8485 auto Undef =
B.buildUndef(Src1Ty);
8486 B.buildShuffleVector(Shuffle.getReg(0), NewSrc1,
Undef, NewMask);
8500 LLT DstTy =
MRI.getType(Dst);
8501 LLT CarryTy =
MRI.getType(Carry);
8523 B.buildConstant(Carry, 0);
8530 B.buildSub(Dst, LHS, RHS);
8548 B.buildConstant(Carry, 0);
8555 B.buildSub(Dst, LHS, RHS);
8572 CtlzMI.
getOpcode() == TargetOpcode::G_CTLZ_ZERO_POISON) &&
8573 "Expected G_CTLZ variant");
8578 LLT Ty =
MRI.getType(Dst);
8579 LLT SrcTy =
MRI.getType(Src);
8581 if (!(Ty.isValid() && Ty.isScalar()))
8590 switch (
LI->getAction(Query).Action) {
8601 bool NeedAdd =
true;
8609 unsigned BitWidth = Ty.getScalarSizeInBits();
8620 B.buildCTLS(Dst,
X);
8624 auto Ctls =
B.buildCTLS(Ty,
X);
8625 auto One =
B.buildConstant(Ty, 1);
8627 B.buildAdd(Dst, Ctls, One);
8635 return C &&
C->getValueAPF().isNaN();
8639 unsigned ZeroPoisonOpc)
const {
8640 const bool IsCtlzPair =
Opc == TargetOpcode::G_CTLZ &&
8641 ZeroPoisonOpc == TargetOpcode::G_CTLZ_ZERO_POISON;
8642 const bool IsCttzPair =
Opc == TargetOpcode::G_CTTZ &&
8643 ZeroPoisonOpc == TargetOpcode::G_CTTZ_ZERO_POISON;
8644 if (
MI.getOpcode() !=
Opc || (!IsCtlzPair && !IsCttzPair))
8650 if (!
VT->isKnownNeverZero(Src))
8653 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
8654 LLT SrcTy =
MRI.getType(Src);
8660 assert((Opcode == TargetOpcode::G_AND || Opcode == TargetOpcode::G_OR) &&
8661 "Expected G_AND or G_OR");
8663 ~(Opcode == TargetOpcode::G_AND ? OtherKnown.
Zero : OtherKnown.
One);
8666static std::optional<unsigned>
8668 if (!Amt || Amt->uge(BW))
8669 return std::nullopt;
8670 return Amt->getZExtValue();
8677 "shift amount must be less than the bit width");
8679 case TargetOpcode::G_SHL:
8681 case TargetOpcode::G_LSHR:
8683 case TargetOpcode::G_ASHR: {
8694Register CombinerHelper::simplifyMultipleUseDemandedBits(
8696 assert(
R.isVirtual() &&
"Expected a virtual register");
8697 assert(!DemandedBits.
isZero() &&
"Zero demand is handled by the caller");
8701 [[maybe_unused]] LLT Ty =
MRI.getType(R);
8704 "DemandedBits width must match the register scalar type");
8706 MachineInstr *
DefMI =
MRI.getVRegDef(R);
8707 assert(
DefMI &&
"Expected a definition in generic SSA MIR");
8716 "Logical operands must match the result type");
8719 std::optional<APInt>
C = getConstantOrConstantSplatVector(CReg);
8722 bool BypassToX = Opcode == TargetOpcode::G_AND
8727 simplifyMultipleUseDemandedBits(
X, DemandedBits,
Depth + 1))
8731 bool BypassToC = Opcode == TargetOpcode::G_AND
8741 return LookThrough(
RHS,
LHS);
8744bool CombinerHelper::simplifyDemandedBitsImpl(
MachineInstr &
MI,
unsigned OpNo,
8747 bool DoRewrite)
const {
8749 assert(
MI.getOperand(OpNo).isReg() && !
MI.getOperand(OpNo).isDef() &&
8750 "Expected a register use");
8751 Register OpReg =
MI.getOperand(OpNo).getReg();
8753 LLT OpTy =
MRI.getType(OpReg);
8756 "DemandedBits width must match the operand scalar type");
8758 MachineInstr *
DefMI =
MRI.getVRegDef(OpReg);
8759 assert(
DefMI &&
"Expected a definition in generic SSA MIR");
8761 auto DropPoisonFlags = [&]() {
8770 auto InvalidateDebugUses = [&](
const APInt &Demanded) {
8771 if (DoRewrite && !Demanded.isAllOnes())
8772 MRI.markUsesInDebugValueAsUndef(OpReg);
8774 auto Rewrite = [&](
Register Repl) {
8775 if (!DoRewrite || Repl == OpReg)
8776 return Repl != OpReg;
8779 InvalidateDebugUses(DemandedBits);
8788 if (DemandedBits.
isZero()) {
8793 MachineBasicBlock &SaveMBB =
Builder.getMBB();
8798 Builder.setInsertPt(SaveMBB, SavePt);
8810 auto GiveUp = [&]() {
8823 APInt Demanded = DemandedBits;
8824 if (!
MRI.hasOneNonDBGUse(OpReg) && !Demanded.
isAllOnes()) {
8826 simplifyMultipleUseDemandedBits(OpReg, Demanded,
Depth)) {
8827 Known =
VT->getKnownBits(Repl);
8836 case TargetOpcode::G_AND:
8837 case TargetOpcode::G_OR: {
8842 "Logical operands must match the result type");
8843 auto SimplifyWithConst = [&](
Register X,
8844 Register CReg) -> std::optional<Register> {
8845 std::optional<APInt>
C = getConstantOrConstantSplatVector(CReg);
8847 return std::nullopt;
8848 assert(
C->getBitWidth() == BW &&
"Constant width must match its type");
8850 if (Opcode == TargetOpcode::G_AND) {
8855 return std::nullopt;
8861 return std::nullopt;
8865 if (std::optional<Register> Repl = SimplifyWithConst(
LHS,
RHS)) {
8866 Known =
VT->getKnownBits(*Repl);
8870 if (std::optional<Register> Repl = SimplifyWithConst(
RHS,
LHS)) {
8871 Known =
VT->getKnownBits(*Repl);
8876 KnownBits RHSKnown(BW);
8877 bool Changed = simplifyDemandedBitsImpl(*
DefMI, 2, Demanded,
8878 RHSKnown,
Depth + 1, DoRewrite);
8880 KnownBits LHSKnown(BW);
8881 Changed |= simplifyDemandedBitsImpl(*
DefMI, 1, LHSDemand, LHSKnown,
8882 Depth + 1, DoRewrite);
8884 !getConstantOrConstantSplatVector(
RHS)) {
8887 LHSDemand == Demanded
8893 OpTy.isFixedVector()
8897 if (RHSDemand != Demanded) {
8899 KnownBits RetryKnown(BW);
8900 Changed = simplifyDemandedBitsImpl(*
DefMI, 2, RHSDemand,
8901 RetryKnown,
Depth + 1, DoRewrite);
8906 InvalidateDebugUses(Demanded);
8908 Known = Opcode == TargetOpcode::G_AND ? LHSKnown & RHSKnown
8909 : LHSKnown | RHSKnown;
8912 case TargetOpcode::G_SHL:
8913 case TargetOpcode::G_LSHR:
8914 case TargetOpcode::G_ASHR: {
8921 KnownBits SrcKnown(BW);
8922 bool Changed = simplifyDemandedBitsImpl(*
DefMI, 1, SrcDemand,
8923 SrcKnown,
Depth + 1, DoRewrite);
8926 InvalidateDebugUses(Demanded);
8930 case TargetOpcode::G_SHL:
8933 case TargetOpcode::G_LSHR:
8936 case TargetOpcode::G_ASHR: {
8940 isLegal({TargetOpcode::G_LSHR, {OpTy, AmtTy}});
8942 SrcKnown.isNonNegative()) &&
8945 MachineBasicBlock &SaveMBB =
Builder.getMBB();
8951 Builder.setInsertPt(SaveMBB, SavePt);
8953 Rewrite(Lshr.getReg(0));
8972 unsigned Depth)
const {
8980 LLT Ty =
MRI.getType(Dst);
8981 assert(Ty.isValid() &&
"Expected a typed generic result");
8982 if (!
VT || Ty.isVector())
8987 if (!simplifyDemandedBitsImpl(
MI, OpNo, OpDemand,
Known, 0,
8998 unsigned Opcode =
MI.getOpcode();
9000 case TargetOpcode::G_SHL:
9001 case TargetOpcode::G_LSHR:
9002 case TargetOpcode::G_ASHR: {
9004 getConstantOrConstantSplatVector(
MI.getOperand(2).getReg()),
9011 return Probe(1, SrcDemand, SrcKnown);
9015 case TargetOpcode::G_AND:
9016 case TargetOpcode::G_OR: {
9021 bool RootRedundant = Opcode == TargetOpcode::G_AND
9030 if (Probe(2, RootDemand, RHSKnown))
9036 if (Probe(1, LHSDemand, LHSKnown))
9040 if (!
MRI.hasOneNonDBGUse(RHS) || getConstantOrConstantSplatVector(RHS))
9044 LHSDemand == RootDemand
9048 VT->getKnownBits(
MI.getOperand(1).getReg()));
9050 return RHSDemand != RootDemand && Probe(2, RHSDemand, RetryKnown);
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const Function * getParent(const Value *V)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static APInt getDemandedBitsForLogicalOp(unsigned Opcode, const APInt &Demanded, const KnownBits &OtherKnown)
static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1, const MachineRegisterInfo &MRI)
static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally)
Checks if MI is TargetOpcode::G_FMUL and contractable either due to global flags or MachineInstr flag...
static unsigned getIndexedOpc(unsigned LdStOpc)
static std::optional< unsigned > getValidConstShiftAmt(const std::optional< APInt > &Amt, unsigned BW)
static APInt getDemandedSrcBitsForShiftConst(unsigned Opcode, const APInt &DemandedBits, unsigned ShAmt)
static std::optional< std::pair< GZExtLoad *, int64_t > > matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits, const MachineRegisterInfo &MRI)
Helper function for findLoadOffsetsForLoadOrCombine.
static std::optional< unsigned > getMinUselessShift(KnownBits ValueKB, unsigned Opcode, std::optional< int64_t > &Result)
Return the minimum useless shift amount that results in complete loss of the source value.
static Register peekThroughBitcast(Register Reg, const MachineRegisterInfo &MRI)
static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I)
static cl::opt< bool > ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(false), cl::desc("Force all indexed operations to be " "legal for the GlobalISel combiner"))
static void commuteMask(MutableArrayRef< int > Mask, const unsigned NumElems)
static cl::opt< unsigned > PostIndexUseThreshold("post-index-use-threshold", cl::Hidden, cl::init(32), cl::desc("Number of uses of a base pointer to check before it is no longer " "considered for post-indexing."))
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static bool vectorFPFoldUsesIEEEDenormals(const MachineFunction &MF, LLT Ty)
static unsigned getExtLoadOpcForExtend(unsigned ExtOpc)
static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits, int64_t Cst, bool IsVector, bool IsFP)
static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy)
static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI, MachineRegisterInfo &MRI)
Return true if 'MI' is a load or a store that may be fold it's address operand into the load / store ...
static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I)
static Register buildLogBase2(Register V, MachineIRBuilder &MIB)
Determines the LogBase2 value for a non-null input value using the transform: LogBase2(V) = (EltBits ...
This contains common combine transformations that may be used in a combine pass,or by the target else...
Utilities for dealing with flags related to floating point properties and mode controls.
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Interface for Targets to specify which operations they can successfully select and how the others sho...
static bool isConstantSplatVector(SDValue N, APInt &SplatValue, unsigned MinSizeInBits)
Implement a low-level type suitable for MachineInstr level instruction selection.
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
Promote Memory to Register
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
uint64_t IntrinsicInst * II
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
Remove Loads Into Fake Uses
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
This file describes how to lower LLVM code to machine code.
static constexpr roundingMode rmNearestTiesToEven
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
uint64_t getZExtValue() const
Get zero extended value.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
int32_t exactLogBase2() const
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countLeadingZeros() const
bool isStrictlyPositive() const
Determine if this APInt Value is positive.
static bool isSameValue(const APInt &I1, const APInt &I2, bool SignedCompare=false)
Determine if two APInts have the same value, after zero-extending or sign-extending (if SignedCompare...
LLVM_ABI APInt multiplicativeInverse() const
bool isMask(unsigned numBits) const
LLVM_ABI APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
bool isOne() const
Determine if this is a value of 1.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
bool isEquality() const
Determine if this is an equals/not equals predicate.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ ICMP_SGE
signed greater or equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
static LLVM_ABI bool isEquality(Predicate pred)
Determine if this is an equals/not equals predicate.
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
static LLVM_ABI bool isOrdered(Predicate predicate)
Determine if the predicate is an ordered operation.
LLVM_ABI void applyCombineBuildVectorOfBitcast(MachineInstr &MI, SmallVector< Register > &Ops) const
LLVM_ABI void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo) const
LLVM_ABI bool matchRepeatedFPDivisor(MachineInstr &MI, SmallVector< MachineInstr * > &MatchInfo) const
LLVM_ABI bool isFPConstantLegalOrBeforeLegalizer(const LLT Ty) const
LLVM_ABI bool matchFoldC2MinusAPlusC1(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchLoadOrCombine(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match expression trees of the form.
LLVM_ABI const RegisterBank * getRegBank(Register Reg) const
Get the register bank of Reg.
LLVM_ABI bool matchEqualDefs(const MachineOperand &MOP1, const MachineOperand &MOP2) const
Return true if MOP1 and MOP2 are register operands are defined by equivalent instructions.
LLVM_ABI void applyUDivOrURemByConst(MachineInstr &MI) const
LLVM_ABI void applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI) const
LLVM_ABI bool matchUnmergeValuesAnyExtBuildVector(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchFsubToFneg(MachineInstr &MI) const
LLVM_ABI bool matchCtls(MachineInstr &CtlzMI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchSelectSameVal(MachineInstr &MI) const
Optimize (cond ? x : x) -> x.
LLVM_ABI bool matchReassocConstantInnerRHS(GPtrAdd &MI, MachineInstr *RHS, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchBitfieldExtractFromShr(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match: shr (shl x, n), k -> sbfx/ubfx x, pos, width.
LLVM_ABI bool matchFoldAMinusC1PlusC2(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchTruncSSatU(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI void applySimplifyURemByPow2(MachineInstr &MI) const
Combine G_UREM x, (known power of 2) to an add and bitmasking.
LLVM_ABI bool matchCombineUnmergeZExtToZExt(MachineInstr &MI) const
Transform X, Y = G_UNMERGE(G_ZEXT(Z)) -> X = G_ZEXT(Z); Y = G_CONSTANT 0.
LLVM_ABI bool matchSimplifyDemandedBits(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match demanded-bits simplification of a root's register use.
const TargetInstrInfo * TII
LLVM_ABI void applyCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops) const
Replace MI with a flattened build_vector with Ops or an implicit_def if Ops is empty.
LLVM_ABI void applyXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo) const
LLVM_ABI bool canCombineFMadOrFMA(MachineInstr &MI, bool &AllowFusionGlobally, bool &HasFMAD, bool &Aggressive, bool CanReassociate=false) const
LLVM_ABI bool matchFoldAPlusC1MinusC2(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchExtractVecEltBuildVec(MachineInstr &MI, Register &Reg) const
LLVM_ABI void applyCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts) const
LLVM_ABI bool matchCombineFAddFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z) (fadd (fpext (fmul x,...
LLVM_ABI bool matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const
LLVM_ABI void applyCombineShuffleConcat(MachineInstr &MI, SmallVector< Register > &Ops) const
Replace MI with a flattened build_vector with Ops or an implicit_def if Ops is empty.
LLVM_ABI void replaceSingleDefInstWithReg(MachineInstr &MI, Register Replacement) const
Delete MI and replace all of its uses with Replacement.
LLVM_ABI void applyCombineShuffleToBuildVector(MachineInstr &MI) const
Replace shuffle MI with a build_vector.
LLVM_ABI bool matchCombineExtractedVectorLoad(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine a G_EXTRACT_VECTOR_ELT of a load into a narrowed load.
LLVM_ABI void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const
MachineRegisterInfo::replaceRegWith() and inform the observer of the changes.
LLVM_ABI void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, Register ToReg) const
Replace a single register operand with a new register and inform the observer of the changes.
LLVM_ABI void applyCombineMemCpyFamily(MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo) const
LLVM_ABI bool matchReassocCommBinOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Reassociate commutative binary operations like G_ADD.
LLVM_ABI void applyBuildFnMO(const MachineOperand &MO, BuildFnTy &MatchInfo) const
Use a function which takes in a MachineIRBuilder to perform a combine.
LLVM_ABI bool matchCommuteConstantToRHS(MachineInstr &MI) const
Match constant LHS ops that should be commuted.
LLVM_ABI bool matchSimplifyNegMinMax(MachineInstr &MI, BuildFnTy &MatchInfo) const
Tranform (neg (min/max x, (neg x))) into (max/min x, (neg x)).
LLVM_ABI bool matchCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI) const
Try to combine G_[SU]DIV and G_[SU]REM into a single G_[SU]DIVREM when their source operands are iden...
LLVM_ABI void applyUMulHToLShr(MachineInstr &MI) const
LLVM_ABI void applyNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate) const
LLVM_ABI bool isLegalOrHasFewerElements(const LegalityQuery &Query) const
LLVM_ABI bool matchShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo) const
Fold (shift (shift base, x), y) -> (shift base (x+y))
LLVM_ABI bool matchTruncLshrBuildVectorFold(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI bool matchAllExplicitUsesAreUndef(MachineInstr &MI) const
Return true if all register explicit use operands on MI are defined by a G_IMPLICIT_DEF.
LLVM_ABI bool isPredecessor(const MachineInstr &DefMI, const MachineInstr &UseMI) const
Returns true if DefMI precedes UseMI or they are the same instruction.
LLVM_ABI bool matchPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo) const
LLVM_ABI bool matchTruncSSatS(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI const TargetLowering & getTargetLowering() const
LLVM_ABI bool matchShuffleUndefRHS(MachineInstr &MI, BuildFnTy &MatchInfo) const
Remove references to rhs if it is undef.
LLVM_ABI void applyBuildInstructionSteps(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const
Replace MI with a series of instructions described in MatchInfo.
LLVM_ABI void applySDivByPow2(MachineInstr &MI) const
LLVM_ABI bool matchConstantFoldFPBinOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Do constant FP folding when opportunities are exposed after MIR building.
LLVM_ABI void applyUDivByPow2(MachineInstr &MI) const
Given an G_UDIV MI expressing an unsigned divided by a pow2 constant, return expressions that impleme...
LLVM_ABI bool matchOr(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine ors.
LLVM_ABI bool matchLshrOfTruncOfLshr(MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo, MachineInstr &ShiftMI) const
Fold (lshr (trunc (lshr x, C1)), C2) -> trunc (shift x, (C1 + C2))
LLVM_ABI void replaceInstWithConstant(MachineInstr &MI, int64_t C) const
Replace an instruction with a G_CONSTANT with value C.
LLVM_ABI bool matchCombineFSubFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z)) (fsub (fpext (fmul x,...
LLVM_ABI bool matchConstantLargerBitWidth(MachineInstr &MI, unsigned ConstIdx) const
Checks if constant at ConstIdx is larger than MI 's bitwidth.
LLVM_ABI void applyCombineCopy(MachineInstr &MI) const
LLVM_ABI bool matchCombineShlOfExtend(MachineInstr &MI, RegisterImmPair &MatchData) const
LLVM_ABI bool matchCombineFSubFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fsub (fmul x, y), z) -> (fma x, y, -z) (fsub (fmul x, y), z) -> (fmad x,...
LLVM_ABI bool matchCombineFAddFMAFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z)) (fadd (fmad x,...
LLVM_ABI bool matchSextTruncSextLoad(MachineInstr &MI) const
LLVM_ABI bool matchCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo) const
LLVM_ABI bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI, Register &UnmergeSrc) const
LLVM_ABI bool matchDivByPow2(MachineInstr &MI, bool IsSigned) const
Given an G_SDIV MI expressing a signed divided by a pow2 constant, return expressions that implements...
LLVM_ABI bool matchNarrowBinopFeedingAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchCombineLoadWithAndMask(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match (and (load x), mask) -> zextload x.
LLVM_ABI bool matchCombineFAddFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fadd (fmul x, y), z) -> (fma x, y, z) (fadd (fmul x, y), z) -> (fmad x,...
LLVM_ABI bool matchCombineCopy(MachineInstr &MI) const
LLVM_ABI bool matchExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI) const
LLVM_ABI void applyShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo) const
LLVM_ABI bool matchXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo) const
Fold (xor (and x, y), y) -> (and (not x), y) {.
LLVM_ABI bool matchCombineShuffleVector(MachineInstr &MI, SmallVectorImpl< Register > &Ops) const
Check if the G_SHUFFLE_VECTOR MI can be replaced by a concat_vectors.
LLVM_ABI void replaceInstWithFConstant(MachineInstr &MI, double C) const
Replace an instruction with a G_FCONSTANT with value C.
LLVM_ABI bool matchOrShiftToFunnelShift(MachineInstr &MI, bool AllowScalarConstants, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchRedundantSExtInReg(MachineInstr &MI) const
LLVM_ABI void replaceOpcodeWith(MachineInstr &FromMI, unsigned ToOpcode) const
Replace the opcode in instruction with a new opcode and inform the observer of the changes.
LLVM_ABI void applyFunnelShiftConstantModulo(MachineInstr &MI) const
Replaces the shift amount in MI with ShiftAmt % BW.
LLVM_ABI bool matchFoldC1Minus2MinusC2(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void applyCombineShlOfExtend(MachineInstr &MI, const RegisterImmPair &MatchData) const
LLVM_ABI void applyUseVectorTruncate(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize, GISelValueTracking *VT=nullptr, MachineDominatorTree *MDT=nullptr, const LegalizerInfo *LI=nullptr)
LLVM_ABI bool matchShuffleDisjointMask(MachineInstr &MI, BuildFnTy &MatchInfo) const
Turn shuffle a, b, mask -> shuffle undef, b, mask iff mask does not reference a.
LLVM_ABI bool matchCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal) const
Transform a multiply by a power-of-2 value to a left shift.
LLVM_ABI void applyCombineShuffleVector(MachineInstr &MI, ArrayRef< Register > Ops) const
Replace MI with a concat_vectors with Ops.
LLVM_ABI bool matchCombineUnmergeUndef(MachineInstr &MI, std::function< void(MachineIRBuilder &)> &MatchInfo) const
Transform G_UNMERGE G_IMPLICIT_DEF -> G_IMPLICIT_DEF, G_IMPLICIT_DEF, ...
LLVM_ABI void applyFoldBinOpIntoSelect(MachineInstr &MI, const unsigned &SelectOpNo) const
SelectOperand is the operand in binary operator MI that is the select to fold.
LLVM_ABI bool matchFoldAMinusC1MinusC2(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) const
LLVM_ABI bool matchCombineShuffleConcat(MachineInstr &MI, SmallVector< Register > &Ops) const
LLVM_ABI void applySextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo) const
LLVM_ABI bool matchTruncUSatU(MachineInstr &MI, MachineInstr &MinMI) const
LLVM_ABI bool matchICmpToLHSKnownBits(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchReassocPtrAdd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Reassociate pointer calculations with G_ADD involved, to allow better addressing mode usage.
LLVM_ABI bool isPreLegalize() const
LLVM_ABI bool matchUndefShuffleVectorMask(MachineInstr &MI) const
Return true if a G_SHUFFLE_VECTOR instruction MI has an undef mask.
LLVM_ABI bool matchCombineSubToAdd(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
LLVM_ABI bool matchCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops) const
If MI is G_CONCAT_VECTORS, try to combine it.
LLVM_ABI bool matchInsertExtractVecEltOutOfBounds(MachineInstr &MI) const
Return true if a G_{EXTRACT,INSERT}_VECTOR_ELT has an out of range index.
LLVM_ABI bool matchOperandIsKnownToBeAPowerOfTwo(const MachineOperand &MO, bool OrNegative=false, bool OrZero=false) const
Check if operand MO is known to be a power of 2.
LLVM_ABI bool matchExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo) const
LLVM_ABI LLVMContext & getContext() const
LLVM_ABI void applyPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo) const
LLVM_ABI bool isConstantLegalOrBeforeLegalizer(const LLT Ty) const
LLVM_ABI bool matchNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate) const
Combine inverting a result of a compare into the opposite cond code.
LLVM_ABI bool matchSextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo) const
Match sext_inreg(load p), imm -> sextload p.
LLVM_ABI bool matchSelectIMinMax(const MachineOperand &MO, BuildFnTy &MatchInfo) const
Combine select to integer min/max.
LLVM_ABI bool simplifyDemandedBits(MachineInstr &MI, unsigned OpNo, const APInt &DemandedBits, KnownBits &Known, unsigned Depth=0) const
Simplify a register use under DemandedBits and compute Known in the same walk.
LLVM_ABI bool matchConstantFoldUnaryIntOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Constant fold a unary integer op (G_CTLZ, G_CTTZ, G_CTPOP and their _ZERO_POISON variants,...
LLVM_ABI bool isLegal(const LegalityQuery &Query) const
LLVM_ABI bool matchShiftsTooBig(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match shifts greater or equal to the range (the bitwidth of the result datatype, or the effective bit...
LLVM_ABI void applyCombineShuffleOfBuildVector(MachineInstr &MI) const
Replace shuffle of build_vector with a build_vector.
LLVM_ABI bool matchICmpToTrueFalseKnownBits(MachineInstr &MI, int64_t &MatchInfo) const
LLVM_ABI bool tryReassocBinOp(unsigned Opc, Register DstReg, Register Op0, Register Op1, BuildFnTy &MatchInfo) const
Try to reassociate to reassociate operands of a commutative binop.
LLVM_ABI void eraseInst(MachineInstr &MI) const
Erase MI.
LLVM_ABI void applyBuildFnNoErase(MachineInstr &MI, BuildFnTy &MatchInfo) const
Use a function which takes in a MachineIRBuilder to perform a combine.
LLVM_ABI bool matchUseVectorTruncate(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI bool matchConstantFoldBinOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Do constant folding when opportunities are exposed after MIR building.
MachineRegisterInfo & MRI
LLVM_ABI void applyExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI) const
LLVM_ABI bool matchConstantFPOp(const MachineOperand &MOP, double C) const
Return true if MOP is defined by a G_FCONSTANT or splat with a value exactly equal to C.
LLVM_ABI MachineInstr * buildUDivOrURemUsingMul(MachineInstr &MI) const
Given an G_UDIV MI or G_UREM MI expressing a divide by constant, return an expression that implements...
LLVM_ABI void applyExtractVecEltBuildVec(MachineInstr &MI, Register &Reg) const
LLVM_ABI bool matchFoldBinOpIntoSelect(MachineInstr &MI, unsigned &SelectOpNo) const
Push a binary operator through a select on constants.
LLVM_ABI bool tryCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftAmount) const
LLVM_ABI bool isConstantNaN(Register Reg) const
LLVM_ABI bool matchPtrAddZeroVector(MachineInstr &MI) const
}
LLVM_ABI bool tryCombineExtendingLoads(MachineInstr &MI) const
If MI is extend that consumes the result of a load, try to combine it.
LLVM_ABI bool isLegalOrBeforeLegalizer(const LegalityQuery &Query) const
LLVM_ABI bool matchBuildVectorIdentityFold(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI bool matchBitfieldExtractFromShrAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match: shr (and x, n), k -> ubfx x, pos, width.
LLVM_ABI void applyTruncSSatS(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI void applyRotateOutOfRange(MachineInstr &MI) const
LLVM_ABI bool matchReassocFoldConstantsInSubTree(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchHoistLogicOpWithSameOpcodeHands(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo) const
Match (logic_op (op x...), (op y...)) -> (op (logic_op x, y))
LLVM_ABI bool matchBitfieldExtractFromAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match: and (lshr x, cst), mask -> ubfx x, cst, width.
LLVM_ABI bool matchBitfieldExtractFromSExtInReg(MachineInstr &MI, BuildFnTy &MatchInfo) const
Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
LLVM_ABI bool matchAndOrDisjointMask(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void replaceInstWithUndef(MachineInstr &MI) const
Replace an instruction with a G_IMPLICIT_DEF.
LLVM_ABI bool isDesirableToCommuteWithShift(const MachineInstr &MI) const
LLVM_ABI bool matchRedundantBinOpInEquality(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform: (X + Y) == X -> Y == 0 (X - Y) == X -> Y == 0 (X ^ Y) == X -> Y == 0 (X + Y) !...
LLVM_ABI bool matchConstantFoldFPUnary(MachineInstr &MI, BuildFnTy &MatchInfo) const
Constant fold a unary FP operation when the source is constant.
LLVM_ABI bool matchOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond) const
If a brcond's true block is not the fallthrough, make it so by inverting the condition and swapping o...
LLVM_ABI bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine addos.
LLVM_ABI void applyAshShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo) const
LLVM_ABI bool matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine selects.
LLVM_ABI bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo) const
LLVM_ABI bool matchConstantFoldCastOp(MachineInstr &MI, BuildFnTy &MatchInfo) const
Do constant folding when opportunities are exposed after MIR building.
LLVM_ABI bool matchCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI) const
Transform X, Y<dead> = G_UNMERGE Z -> X = G_TRUNC Z.
LLVM_ABI bool matchRotateOutOfRange(MachineInstr &MI) const
LLVM_ABI void applyExpandFPowI(MachineInstr &MI, int64_t Exponent) const
Expands FPOWI into a series of multiplications and a division if the exponent is negative.
LLVM_ABI void setRegBank(Register Reg, const RegisterBank *RegBank) const
Set the register bank of Reg.
LLVM_ABI bool matchCommuteFPConstantToRHS(MachineInstr &MI) const
Match constant LHS FP ops that should be commuted.
LLVM_ABI void applyCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI) const
LLVM_ABI bool matchRedundantOr(MachineInstr &MI, Register &Replacement) const
LLVM_ABI void applyTruncSSatU(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI void applySimplifySRemByPow2(MachineInstr &MI) const
Combine G_SREM x, (+/-2^k) to a bias-and-mask sequence.
LLVM_ABI bool matchCombineFSubFpExtFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fsub (fpext (fneg (fmul x, y))), z) -> (fneg (fma (fpext x), (fpext y),...
LLVM_ABI bool matchTruncBuildVectorFold(MachineInstr &MI, Register &MatchInfo) const
LLVM_ABI void applyCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo) const
LLVM_ABI bool matchConstantOp(const MachineOperand &MOP, int64_t C) const
Return true if MOP is defined by a G_CONSTANT or splat with a value equal to C.
LLVM_ABI void applyCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal) const
LLVM_ABI void applyCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, Register &UnmergeSrc) const
LLVM_ABI bool matchUMulHToLShr(MachineInstr &MI) const
MachineDominatorTree * MDT
LLVM_ABI bool matchSimplifySelectToMinMax(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void applyRepeatedFPDivisor(SmallVector< MachineInstr * > &MatchInfo) const
LLVM_ABI bool matchTruncUSatUToFPTOUISat(MachineInstr &MI, MachineInstr &SrcMI) const
const RegisterBankInfo * RBI
LLVM_ABI bool matchMulOBy0(MachineInstr &MI, BuildFnTy &MatchInfo) const
Match: (G_*MULO x, 0) -> 0 + no carry out.
LLVM_ABI bool matchBinopWithNeg(MachineInstr &MI, BuildFnTy &MatchInfo) const
Fold a bitwiseop (~b +/- c) -> a bitwiseop ~(b -/+ c)
LLVM_ABI bool matchCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts) const
Transform G_UNMERGE Constant -> Constant1, Constant2, ...
LLVM_ABI bool matchCountZeroToZeroPoison(MachineInstr &MI, unsigned Opc, unsigned ZeroPoisonOpc) const
LLVM_ABI void applyShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo) const
const TargetRegisterInfo * TRI
LLVM_ABI bool matchRedundantAnd(MachineInstr &MI, Register &Replacement) const
LLVM_ABI bool dominates(const MachineInstr &DefMI, const MachineInstr &UseMI) const
Returns true if DefMI dominates UseMI.
GISelChangeObserver & Observer
LLVM_ABI void applyBuildFn(MachineInstr &MI, BuildFnTy &MatchInfo) const
Use a function which takes in a MachineIRBuilder to perform a combine.
LLVM_ABI bool matchCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo) const
Transform trunc (shl x, K) to shl (trunc x), K if K < VT.getScalarSizeInBits().
LLVM_ABI bool matchCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftSize, unsigned &ShiftVal) const
Reduce a shift by a constant to an unmerge and a shift on a half sized type.
LLVM_ABI bool matchUDivOrURemByConst(MachineInstr &MI) const
Combine G_UDIV or G_UREM by constant into a multiply by magic constant.
LLVM_ABI bool matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine ands.
LLVM_ABI bool matchSuboCarryOut(const MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchCombineFSubFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Transform (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z)) (fsub (fneg (fmul,...
LLVM_ABI bool matchCombineZextTrunc(MachineInstr &MI, Register &Reg) const
Transform zext(trunc(x)) to x.
LLVM_ABI bool matchUDivByShiftedPow2(MachineInstr &MI, BuildFnTy &MatchInfo) const
Combine unsigned division by a shifted power of two into a right shift.
LLVM_ABI bool matchSameRegOrICstOrSplat(Register A, Register B) const
True when A and B are the same register, including through copies, or different registers holding the...
LLVM_ABI void applyLshrOfTruncOfLshr(MachineInstr &MI, LshrOfTruncOfLshr &MatchInfo) const
LLVM_ABI bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0) const
Optimize memcpy intrinsics et al, e.g.
LLVM_ABI bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI void applySDivOrSRemByConst(MachineInstr &MI) const
LLVM_ABI bool matchCombineMemCpyFamily(MachineInstr &MI, MemCpyFamilyLoweringInfo &MatchInfo, unsigned MaxLen=0) const
LLVM_ABI MachineInstr * buildSDivOrSRemUsingMul(MachineInstr &MI) const
Given an G_SDIV MI or G_SREM MI expressing a signed divide by constant, return an expression that imp...
LLVM_ABI bool isLegalOrHasWidenScalar(const LegalityQuery &Query) const
LLVM_ABI bool matchSignedDivRemToUnsigned(MachineInstr &MI, unsigned UnsignedOpcode) const
Convert signed division or remainder of nonnegative operands to unsigned.
LLVM_ABI bool matchReassocConstantInnerLHS(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchOverlappingAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const
Fold and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0.
LLVM_ABI bool matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg) const
Transform anyext(trunc(x)) to x.
LLVM_ABI void applyExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo) const
MachineIRBuilder & Builder
LLVM_ABI bool isZeroOrOneVector(Register Reg) const
LLVM_ABI void replaceSingleDefInstWithOperand(MachineInstr &MI, unsigned OpIdx) const
Delete MI and replace all of its uses with its OpIdx-th operand.
LLVM_ABI const MachineFunction & getMachineFunction() const
LLVM_ABI bool matchCombineBuildVectorOfBitcast(MachineInstr &MI, SmallVector< Register > &Ops) const
Combine G_BUILD_VECTOR(G_UNMERGE(G_BITCAST), Undef) to G_BITCAST(G_BUILD_VECTOR(.....
LLVM_ABI bool matchCombineFAddFpExtFMulToFMadOrFMAAggressive(MachineInstr &MI, BuildFnTy &MatchInfo) const
LLVM_ABI bool matchConstantFoldFMA(MachineInstr &MI, BuildFnTy &MatchInfo) const
Constant fold G_FMA/G_FMAD.
LLVM_ABI bool matchSDivOrSRemByConst(MachineInstr &MI) const
Combine G_SDIV or G_SREM by constant into a multiply by magic constant.
LLVM_ABI void applyOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond) const
LLVM_ABI void applyCombineShiftToUnmerge(MachineInstr &MI, const unsigned &ShiftVal) const
LLVM_ABI bool matchFPowIExpansion(MachineInstr &MI, int64_t Exponent) const
Match FPOWI if it's safe to extend it into a series of multiplications.
LLVM_ABI void applyCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo) const
LLVM_ABI bool matchCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands) const
Transform <ty,...> G_UNMERGE(G_MERGE ty X, Y, Z) -> ty X, Y, Z.
LLVM_ABI void applyCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands) const
LLVM_ABI bool matchAshrShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo) const
Match ashr (shl x, C), C -> sext_inreg (C)
LLVM_ABI void applyCombineUnmergeZExtToZExt(MachineInstr &MI) const
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
This class represents a range of values.
LLVM_ABI std::optional< ConstantRange > exactUnionWith(const ConstantRange &CR) const
Union the two ranges and return the result if it can be represented exactly, otherwise return std::nu...
LLVM_ABI ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
static LLVM_ABI ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
const APInt & getLower() const
Return the lower value for this range.
LLVM_ABI OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const
Return whether unsigned sub of the two ranges always/never overflows.
LLVM_ABI OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
LLVM_ABI bool isWrappedSet() const
Return true if this set wraps around the unsigned domain.
const APInt & getUpper() const
Return the upper value for this range.
static LLVM_ABI ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
LLVM_ABI OverflowResult signedAddMayOverflow(const ConstantRange &Other) const
Return whether signed add of the two ranges always/never overflows.
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
LLVM_ABI OverflowResult signedSubMayOverflow(const ConstantRange &Other) const
Return whether signed sub of the two ranges always/never overflows.
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Represents overflowing add operations.
Represents an integer addition.
Represents a logical and.
CmpInst::Predicate getCond() const
Register getLHSReg() const
Register getRHSReg() const
Represents any generic load, including sign/zero extending variants.
Register getDstReg() const
Get the definition register of the loaded value.
Register getCarryOutReg() const
Register getRHSReg() const
Register getLHSReg() const
Register getLHSReg() const
Register getRHSReg() const
Represents a G_BUILD_VECTOR.
Register getSrcReg() const
Represents a G_CONCAT_VECTORS.
Abstract class that contains various methods for clients to notify about changes.
Simple wrapper observer that takes several observers, and calls each one for each event.
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
Represents a logical binary operation.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
bool isAtomic() const
Returns true if the attached MachineMemOperand has the atomic flag set.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
Register getSourceReg(unsigned I) const
Returns the I'th source register.
unsigned getNumSources() const
Returns the number of source registers.
Represents overflowing sub operations.
Represents an integer subtraction.
Represents a G_UNMERGE_VALUES.
unsigned getNumDefs() const
Returns the number of def registers.
Register getSourceReg() const
Get the unmerge source register.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
constexpr bool isScalableVector() const
Returns true if the LLT is a scalable vector.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
LLT getScalarType() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isFloat() const
constexpr bool isVector() const
constexpr bool isByteSized() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr bool isAnyScalar() const
constexpr ElementCount getElementCount() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr bool isPointerOrPointerVector() const
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
static LLT integer(unsigned SizeInBits)
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
This is an important class for using LLVM in a threaded context.
LLVM_ABI LegalizeResult lowerMemCpyFamily(MachineInstr &MI, Register Dst, Register Src, uint64_t KnownLen, Align Alignment, bool DstAlignCanChange, ArrayRef< LLT > MemOps)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
LLVM_ABI Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index)
Get a pointer to vector element Index located in memory for a vector of type VecTy starting at a base...
TypeSize getValue() const
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Helper class to build MachineInstr.
const TargetInstrInfo & getTII()
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineFunction & getMF()
Getter for the function we currently build.
MachineRegisterInfo * getMRI()
Getter for MRI.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
LLVM_ABI bool isDereferenceableInvariantLoad() const
Return true if this load instruction never traps and points to a memory location whose value doesn't ...
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
unsigned getNumOperands() const
Retuns the total number of operands.
static constexpr uint32_t getPoisonGeneratingFlags()
LLVM_ABI void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
LLVM_ABI unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
mop_range uses()
Returns all operands which may be register uses.
MachineOperand * findRegisterUseOperand(Register Reg, const TargetRegisterInfo *TRI, bool isKill=false)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
A description of a memory reference used in the backend.
LLT getMemoryType() const
Return the memory type of the memory reference.
unsigned getAddrSpace() const
bool isAtomic() const
Returns true if this operation has an atomic ordering requirement of unordered or higher,...
const MachinePointerInfo & getPointerInfo() const
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
LocationSize getSizeInBits() const
Return the size in bits of the memory reference.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
void setMBB(MachineBasicBlock *MBB)
void setPredicate(unsigned Predicate)
Register getReg() const
getReg - Returns the register number.
unsigned getPredicate() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
This class implements the register bank concept.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
size_type size() const
Determine the number of elements in the SetVector.
size_type count(const_arg_type key) const
Count the number of elements of a given key in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool all() const
Returns true if all bits are set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
virtual LLVM_READONLY LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
bool isBeneficialToExpandPowI(int64_t Exponent, bool OptForSize) const
Return true if it is beneficial to expand an @llvm.powi.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool isIndexingLegal(MachineInstr &MI, Register Base, Register Offset, bool IsPre, MachineRegisterInfo &MRI) const
Returns true if the specified base+offset is a legal indexed addressing mode for this target.
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount through its operand,...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
The instances of the Type class are immutable: once they are created, they are never changed.
A Use represents the edge between a Value definition and its users.
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ FewerElements
The (vector) operation should be implemented by splitting it into sub-vectors where the operation is ...
@ Legal
The operation is expected to be selectable directly by the target, and no transformation is necessary...
@ WidenScalar
The operation should be implemented in terms of a wider scalar base-type.
@ Custom
The target wants to do something special with this combination of operand and type.
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(const APInt &RequestedValue)
Matches a constant equal to RequestedValue.
GInstrBind< GBuildVector > m_GBuildVector(GBuildVector *&Inst)
GCstAndRegMatch m_GCst(std::optional< ValueAndVReg > &ValReg)
LoadOp_match< GLoad, PtrP > m_GLoad(const PtrP &Ptr)
MIFlagsRef m_MIFlags(uint32_t &Flags)
operand_type_match m_Pred()
BinaryOp_match< LHS, RHS, TargetOpcode::G_UMIN, true > m_GUMin(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_XOR, true > m_GXor(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_SEXT > m_GSExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_FPEXT > m_GFPExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
BinaryOp_match< SpecificConstantMatch, SrcTy, TargetOpcode::G_SUB > m_Neg(const SrcTy &&Src)
Matches a register negated by a G_SUB.
ICstOrSplatMatch< APInt > m_ICstOrSplat(APInt &Cst)
ImplicitDefMatch m_GImplicitDef()
GInstrBind< GConcatVectors > m_GConcatVectors(GConcatVectors *&Inst)
GConstantBitsMatch m_GConstantOrFConstantBits(APInt &Bits)
CheckType m_SpecificType(LLT Ty)
deferred_ty< Register > m_DeferredReg(Register &R)
Similar to m_SpecificReg/Type, but the specific value to match originated from an earlier sub-pattern...
BinaryOp_match< LHS, RHS, TargetOpcode::G_UMAX, true > m_GUMax(const LHS &L, const RHS &R)
BinaryOp_match< SrcTy, SpecificConstantMatch, TargetOpcode::G_XOR, true > m_Not(const SrcTy &&Src)
Matches a register not-ed by a G_XOR.
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP > m_GICmp(const Pred &P, const LHS &L, const RHS &R)
GInstrBind< GUnmerge > m_GUnmerge(GUnmerge *&Inst)
Instruction binders for ops with no operand-form matcher (constant-immediate or variadic-source ops).
MMORef m_MMO(const MachineMemOperand *&MMO)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ASHR, false > m_GAShr(const LHS &L, const RHS &R)
TernaryOp_match< Src0Ty, Src1Ty, Src2Ty, TargetOpcode::G_SELECT > m_GISelect(const Src0Ty &Src0, const Src1Ty &Src1, const Src2Ty &Src2)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
SpecificConstantOrSplatMatch m_SpecificICstOrSplat(const APInt &RequestedValue)
Matches a RequestedValue constant or a constant splat of RequestedValue.
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_BITCAST > m_GBitcast(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_BUILD_VECTOR_TRUNC, false > m_GBuildVectorTrunc(const LHS &L, const RHS &R)
bind_ty< MachineInstr * > m_MInstr(MachineInstr *&MI)
UnaryOp_match< SrcTy, TargetOpcode::G_FNEG > m_GFNeg(const SrcTy &Src)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP, true > m_c_GICmp(const Pred &P, const LHS &L, const RHS &R)
G_ICMP matcher that also matches commuted compares.
LoadOp_match< GAnyLoad, PtrP > m_GAnyLoad(const PtrP &Ptr)
TernaryOp_match< Src0Ty, Src1Ty, Src2Ty, TargetOpcode::G_INSERT_VECTOR_ELT > m_GInsertVecElt(const Src0Ty &Src0, const Src1Ty &Src1, const Src2Ty &Src2)
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
And< Preds... > m_all_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SMIN, true > m_GSMin(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_ANYEXT > m_GAnyExt(const SrcTy &Src)
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
BinaryOp_match< LHS, RHS, TargetOpcode::G_FMUL, true > m_GFMul(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SMAX, true > m_GSMax(const LHS &L, const RHS &R)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_FCMP > m_GFCmp(const Pred &P, const LHS &L, const RHS &R)
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
Not(const Pred &P) -> Not< Pred >
initializer< Ty > init(const Ty &Val)
unsigned getNumElements(Type *Ty)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
LLVM_ABI std::optional< APInt > isConstantOrConstantSplatVector(Register Def, const MachineRegisterInfo &MRI)
Determines if Def defines a constant integer or a splat vector of constant integers.
LLVM_ABI bool isBuildVectorAllZeros(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
LLVM_ABI Type * getTypeForLLT(LLT Ty, LLVMContext &C)
Get the type back from LLT.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
LLVM_ABI std::optional< APFloat > isConstantOrConstantSplatVectorFP(Register Def, const MachineRegisterInfo &MRI)
Determines if Def defines a float constant integer or a splat vector of float constant integers.
LLVM_ABI const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI std::optional< APInt > getIConstantVRegVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT, return the corresponding value.
LLVM_ABI std::optional< APInt > getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI)
LLVM_ABI bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
std::function< void(MachineIRBuilder &)> BuildFnTy
LLVM_ABI const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
LLVM_ABI std::optional< APFloat > ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
@ Load
The value being inserted comes from a load (InsertElement only).
LLVM_ABI bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg, llvm::function_ref< bool(const Constant *ConstVal)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant G_B...
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
LLVM_ABI MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
LLVM_ABI MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
LLVM_ABI bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Returns true if given the TargetLowering's boolean contents information, the value Val contains a tru...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)
Return true if the given value is known to have exactly one bit set when defined.
LLVM_ABI std::optional< APInt > ConstantFoldBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
auto dyn_cast_or_null(const Y &Val)
constexpr bool has_single_bit(T Value) noexcept
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI const APInt & getIConstantFromReg(Register VReg, const MachineRegisterInfo &MRI)
VReg is defined by a G_CONSTANT, return the corresponding value.
LLVM_ABI bool isConstantOrConstantVector(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowFP=true, bool AllowOpaqueConstants=true)
Return true if the specified instruction is known to be a constant, or a vector of constants.
constexpr unsigned MaxAnalysisRecursionDepth
SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > OperandBuildSteps
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM_ABI bool canReplaceReg(Register DstReg, Register SrcReg, MachineRegisterInfo &MRI)
Check if DstReg can be replaced with SrcReg depending on the register constraints.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
std::tuple< Register, Register, uint64_t, Align, bool, std::vector< LLT > > MemCpyFamilyLoweringInfo
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
LLVM_ABI bool canCreateUndefOrPoison(const Operator *Op, bool ConsiderFlagsAndMetadata=true)
canCreateUndefOrPoison returns true if Op can create undef or poison from non-undef & non-poison oper...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI SmallVector< APInt > ConstantFoldVectorBinop(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
Tries to constant fold a vector binop with sources Op1 and Op2.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI std::optional< FPValueAndVReg > getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI, bool AllowUndef=true)
Returns a floating point scalar constant of a build vector splat if it exists.
LLVM_ABI EVT getApproximateEVTForLLT(LLT Ty, LLVMContext &Ctx)
LLVM_ABI std::optional< APInt > ConstantFoldCastOp(unsigned Opcode, LLT DstTy, const Register Op0, const MachineRegisterInfo &MRI)
LLVM_ABI bool canLowerMemCpyFamily(const MachineInstr &MI, const MachineRegisterInfo &MRI, unsigned MaxLen, Register &Dst, Register &Src, uint64_t &KnownLen, Align &Alignment, bool &DstAlignCanChange, std::vector< LLT > &MemOps)
Matcher for memcpy-like instructions.
LLVM_ABI SmallVector< APFloat > ConstantFoldUnaryFPOp(unsigned Opcode, LLT DstTy, Register Src, const MachineRegisterInfo &MRI)
Tries to constant fold a unary floating-point operation on Src.
LLVM_ABI unsigned getInverseGMinMaxOpcode(unsigned MinMaxOpc)
Returns the inverse opcode of MinMaxOpc, which is a generic min/max opcode like G_SMIN.
LLVM_ABI SmallVector< APFloat > ConstantFoldVectorFPBinop(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
Tries to constant fold a vector floating-point binop with sources Op1 and Op2.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ Sub
Subtraction of integers.
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
LLVM_ABI std::optional< FPValueAndVReg > getFConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_FCONSTANT returns it...
constexpr unsigned BitWidth
LLVM_ABI int64_t getICmpTrueVal(const TargetLowering &TLI, bool IsVector, bool IsFP)
Returns an integer representing true, as defined by the TargetBooleanContents.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
LLVM_ABI std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
LLVM_ABI SmallVector< APInt > ConstantFoldUnaryIntOp(unsigned Opcode, LLT DstTy, Register Src, const MachineRegisterInfo &MRI)
Tries to constant fold a unary integer operation (G_CTLZ, G_CTTZ, G_CTPOP and their _ZERO_POISON vari...
unsigned Log2(Align A)
Returns the log2 of the alignment.
LLVM_ABI Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
unsigned getFCmpCode(CmpInst::Predicate CC)
Similar to getICmpCode but for FCmpInst.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Simple struct used to hold a Register value and the instruction which defines it.
static constexpr DenormalMode getIEEE()
SmallVector< InstructionBuildSteps, 2 > InstrsToBuild
Describes instructions to be built during a combine.
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
bool isNonNegative() const
Returns true if this value is known to be non-negative.
unsigned countMinLeadingOnes() const
Returns the minimum number of leading one bits.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
static LLVM_ABI KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned getBitWidth() const
Get the bit width of this value.
static LLVM_ABI KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
bool isNegative() const
Returns true if this value is known to be negative.
static LLVM_ABI KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
This class contains a discriminated union of information about pointers in memory operands,...
LLVM_ABI unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
MachinePointerInfo getWithOffset(int64_t O) const
const RegisterBank * Bank
Register LogicNonShiftReg
Magic data for optimising signed division by a constant.
unsigned ShiftAmount
shift amount
static LLVM_ABI SignedDivisionByConstantInfo get(const APInt &D)
Calculate the magic numbers required to implement a signed integer division by a constant as a sequen...
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
Magic data for optimising unsigned division by a constant.
unsigned PreShift
pre-shift amount
unsigned PostShift
post-shift amount
static LLVM_ABI UnsignedDivisionByConstantInfo get(const APInt &D, unsigned LeadingZeros=0, bool AllowEvenDivisorOptimization=true, bool AllowWidenOptimization=false)
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...