51#define GET_GICOMBINER_DEPS
52#include "AArch64GenPostLegalizeGILowering.inc"
53#undef GET_GICOMBINER_DEPS
55#define DEBUG_TYPE "aarch64-postlegalizer-lowering"
58using namespace MIPatternMatch;
59using namespace AArch64GISelUtils;
63#define GET_GICOMBINER_TYPES
64#include "AArch64GenPostLegalizeGILowering.inc"
65#undef GET_GICOMBINER_TYPES
70struct ShuffleVectorPseudo {
74 ShuffleVectorPseudo(
unsigned Opc,
Register Dst,
75 std::initializer_list<SrcOp> SrcOps)
76 : Opc(Opc), Dst(Dst), SrcOps(SrcOps){};
77 ShuffleVectorPseudo() =
default;
85 "Only possible block sizes for REV are: 16, 32, 64");
86 assert(EltSize != 64 &&
"EltSize cannot be 64 for REV mask.");
88 unsigned BlockElts =
M[0] + 1;
97 for (
unsigned i = 0; i < NumElts; ++i) {
101 if (
static_cast<unsigned>(M[i]) !=
102 (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
112 if (NumElts % 2 != 0)
114 WhichResult = (
M[0] == 0 ? 0 : 1);
115 for (
unsigned i = 0; i < NumElts; i += 2) {
116 if ((M[i] >= 0 &&
static_cast<unsigned>(M[i]) != i + WhichResult) ||
118 static_cast<unsigned>(
M[i + 1]) != i + NumElts + WhichResult))
126std::optional<std::pair<bool, uint64_t>> getExtMask(
ArrayRef<int> M,
129 auto FirstRealElt =
find_if(M, [](
int Elt) {
return Elt >= 0; });
130 if (FirstRealElt ==
M.end())
135 APInt ExpectedElt =
APInt(MaskBits, *FirstRealElt + 1);
141 [&ExpectedElt](
int Elt) { return Elt != ExpectedElt++ && Elt >= 0; }))
151 bool ReverseExt =
false;
163 return std::make_pair(ReverseExt, Imm);
169 WhichResult = (
M[0] == 0 ? 0 : 1);
170 for (
unsigned i = 0; i != NumElts; ++i) {
174 if (
static_cast<unsigned>(M[i]) != 2 * i + WhichResult)
182bool isZipMask(
ArrayRef<int> M,
unsigned NumElts,
unsigned &WhichResult) {
183 if (NumElts % 2 != 0)
187 WhichResult = (
M[0] == 0 ? 0 : 1);
188 unsigned Idx = WhichResult * NumElts / 2;
189 for (
unsigned i = 0; i != NumElts; i += 2) {
190 if ((M[i] >= 0 &&
static_cast<unsigned>(M[i]) !=
Idx) ||
191 (
M[i + 1] >= 0 &&
static_cast<unsigned>(
M[i + 1]) !=
Idx + NumElts))
207 int NumInputElements) {
208 if (
M.size() !=
static_cast<size_t>(NumInputElements))
210 int NumLHSMatch = 0, NumRHSMatch = 0;
211 int LastLHSMismatch = -1, LastRHSMismatch = -1;
212 for (
int Idx = 0;
Idx < NumInputElements; ++
Idx) {
218 M[
Idx] ==
Idx ? ++NumLHSMatch : LastLHSMismatch =
Idx;
219 M[
Idx] ==
Idx + NumInputElements ? ++NumRHSMatch : LastRHSMismatch =
Idx;
221 const int NumNeededToMatch = NumInputElements - 1;
222 if (NumLHSMatch == NumNeededToMatch)
223 return std::make_pair(
true, LastLHSMismatch);
224 if (NumRHSMatch == NumNeededToMatch)
225 return std::make_pair(
false, LastRHSMismatch);
232 ShuffleVectorPseudo &MatchInfo) {
233 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
237 LLT Ty =
MRI.getType(Dst);
247 for (
unsigned LaneSize : {64U, 32U, 16U}) {
248 if (
isREVMask(ShuffleMask, EltSize, NumElts, LaneSize)) {
251 Opcode = AArch64::G_REV64;
252 else if (LaneSize == 32U)
253 Opcode = AArch64::G_REV32;
255 Opcode = AArch64::G_REV16;
257 MatchInfo = ShuffleVectorPseudo(Opcode, Dst, {Src});
268 ShuffleVectorPseudo &MatchInfo) {
269 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
270 unsigned WhichResult;
273 unsigned NumElts =
MRI.getType(Dst).getNumElements();
274 if (!
isTRNMask(ShuffleMask, NumElts, WhichResult))
276 unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;
279 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1,
V2});
289 ShuffleVectorPseudo &MatchInfo) {
290 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
291 unsigned WhichResult;
294 unsigned NumElts =
MRI.getType(Dst).getNumElements();
295 if (!
isUZPMask(ShuffleMask, NumElts, WhichResult))
297 unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;
300 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1,
V2});
305 ShuffleVectorPseudo &MatchInfo) {
306 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
307 unsigned WhichResult;
310 unsigned NumElts =
MRI.getType(Dst).getNumElements();
311 if (!isZipMask(ShuffleMask, NumElts, WhichResult))
313 unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;
316 MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1,
V2});
323 ShuffleVectorPseudo &MatchInfo) {
342 auto *InsMI =
getOpcodeDef(TargetOpcode::G_INSERT_VECTOR_ELT,
343 MI.getOperand(1).getReg(),
MRI);
347 if (!
getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(),
355 MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP,
MI.getOperand(0).getReg(),
356 {InsMI->getOperand(2).getReg()});
363 ShuffleVectorPseudo &MatchInfo) {
364 assert(Lane >= 0 &&
"Expected positive lane?");
367 auto *BuildVecMI =
getOpcodeDef(TargetOpcode::G_BUILD_VECTOR,
368 MI.getOperand(1).getReg(),
MRI);
371 Register Reg = BuildVecMI->getOperand(Lane + 1).getReg();
373 ShuffleVectorPseudo(AArch64::G_DUP,
MI.getOperand(0).getReg(), {Reg});
378 ShuffleVectorPseudo &MatchInfo) {
379 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
383 int Lane = *MaybeLane;
387 if (matchDupFromInsertVectorElt(Lane,
MI,
MRI, MatchInfo))
389 if (matchDupFromBuildVector(Lane,
MI,
MRI, MatchInfo))
406 unsigned ExpectedElt =
M[0];
407 for (
unsigned I = 1;
I < NumElts; ++
I) {
411 if (ExpectedElt == NumElts)
416 if (ExpectedElt !=
static_cast<unsigned>(M[
I]))
424 ShuffleVectorPseudo &MatchInfo) {
425 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
427 LLT DstTy =
MRI.getType(Dst);
430 auto Mask =
MI.getOperand(3).getShuffleMask();
433 uint64_t ExtFactor =
MRI.getType(V1).getScalarSizeInBits() / 8;
436 if (!getOpcodeDef<GImplicitDef>(V2,
MRI) ||
437 !isSingletonExtMask(Mask, DstTy))
441 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V1,
Imm});
445 std::tie(ReverseExt, Imm) = *ExtInfo;
449 MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1,
V2,
Imm});
456 ShuffleVectorPseudo &MatchInfo) {
458 MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps);
459 MI.eraseFromParent();
467 if (MatchInfo.SrcOps[2].getImm() == 0)
468 MIRBuilder.buildCopy(MatchInfo.Dst, MatchInfo.SrcOps[0]);
472 MIRBuilder.buildConstant(
LLT::scalar(32), MatchInfo.SrcOps[2].getImm());
473 MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst},
474 {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst});
476 MI.eraseFromParent();
480 assert(
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT);
489 auto &
Insert = cast<GInsertVectorElement>(
MI);
512 "Expected a power-2 vector size");
540 std::tuple<Register, int, Register, int> &MatchInfo) {
541 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
544 int NumElts =
MRI.getType(Dst).getNumElements();
545 auto DstIsLeftAndDstLane =
isINSMask(ShuffleMask, NumElts);
546 if (!DstIsLeftAndDstLane)
550 std::tie(DstIsLeft, DstLane) = *DstIsLeftAndDstLane;
556 int SrcLane = ShuffleMask[DstLane];
557 if (SrcLane >= NumElts) {
562 MatchInfo = std::make_tuple(DstVec, DstLane, SrcVec, SrcLane);
568 std::tuple<Register, int, Register, int> &MatchInfo) {
571 auto ScalarTy =
MRI.getType(Dst).getElementType();
573 int DstLane, SrcLane;
574 std::tie(DstVec, DstLane, SrcVec, SrcLane) = MatchInfo;
579 MI.eraseFromParent();
587 assert(Ty.
isVector() &&
"vector shift count is not a vector type");
594 return Cnt >= 1 && Cnt <= ElementBits;
600 assert(
MI.getOpcode() == TargetOpcode::G_ASHR ||
601 MI.getOpcode() == TargetOpcode::G_LSHR);
602 LLT Ty =
MRI.getType(
MI.getOperand(1).getReg());
610 unsigned Opc =
MI.getOpcode();
611 assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR);
613 Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR;
615 auto ImmDef = MIB.buildConstant(
LLT::scalar(32), Imm);
616 MIB.buildInstr(NewOpc, {
MI.getOperand(0)}, {
MI.getOperand(1), ImmDef});
617 MI.eraseFromParent();
627std::optional<std::pair<uint64_t, CmpInst::Predicate>>
630 const auto &Ty =
MRI.getType(RHS);
634 assert((
Size == 32 ||
Size == 64) &&
"Expected 32 or 64 bit compare only?");
641 uint64_t C = ValAndVReg->Value.getZExtValue();
659 (
Size == 32 &&
static_cast<int32_t
>(
C) == INT32_MIN))
685 if ((
Size == 32 &&
static_cast<int32_t
>(
C) == INT32_MAX) ||
699 if ((
Size == 32 &&
static_cast<uint32_t>(
C) == UINT32_MAX) ||
724bool matchAdjustICmpImmAndPred(
726 std::pair<uint64_t, CmpInst::Predicate> &MatchInfo) {
727 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
730 if (
auto MaybeNewImmAndPred = tryAdjustICmpImmAndPred(RHS, Pred,
MRI)) {
731 MatchInfo = *MaybeNewImmAndPred;
737void applyAdjustICmpImmAndPred(
738 MachineInstr &
MI, std::pair<uint64_t, CmpInst::Predicate> &MatchInfo,
746 RHS.setReg(Cst->getOperand(0).getReg());
747 MI.getOperand(1).setPredicate(MatchInfo.second);
752 std::pair<unsigned, int> &MatchInfo) {
753 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
755 const LLT SrcTy =
MRI.getType(Src1Reg);
756 const LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
775 if (ScalarSize == 64)
776 Opc = AArch64::G_DUPLANE64;
777 else if (ScalarSize == 32)
778 Opc = AArch64::G_DUPLANE32;
781 if (ScalarSize == 32)
782 Opc = AArch64::G_DUPLANE32;
783 else if (ScalarSize == 16)
784 Opc = AArch64::G_DUPLANE16;
788 Opc = AArch64::G_DUPLANE8;
789 else if (ScalarSize == 16)
790 Opc = AArch64::G_DUPLANE16;
794 Opc = AArch64::G_DUPLANE8;
802 MatchInfo.first = Opc;
803 MatchInfo.second = *LaneIdx;
809 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
811 const LLT SrcTy =
MRI.getType(Src1Reg);
813 B.setInstrAndDebugLoc(
MI);
814 auto Lane =
B.buildConstant(
LLT::scalar(64), MatchInfo.second);
820 auto Undef =
B.buildUndef(SrcTy);
822 {Src1Reg, Undef.getReg(0)})
825 B.buildInstr(MatchInfo.first, {MI.getOperand(0).getReg()}, {DupSrc, Lane});
826 MI.eraseFromParent();
830 auto &Unmerge = cast<GUnmerge>(
MI);
831 Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1);
832 const LLT SrcTy =
MRI.getType(Src1Reg);
841 auto &Unmerge = cast<GUnmerge>(
MI);
842 Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1);
843 const LLT SrcTy =
MRI.getType(Src1Reg);
845 "Expected a fixed length vector");
848 B.buildExtractVectorElementConstant(Unmerge.getReg(
I), Src1Reg,
I);
849 MI.eraseFromParent();
853 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
862 int64_t Cst =
Splat->getCst();
863 return (Cst != 0 && Cst != -1);
868 B.setInstrAndDebugLoc(
MI);
869 B.buildInstr(AArch64::G_DUP, {
MI.getOperand(0).
getReg()},
871 MI.eraseFromParent();
878 if (!
MRI.hasOneNonDBGUse(CmpOp))
883 if (
MI.getOpcode() == TargetOpcode::G_SEXT_INREG)
885 if (
MI.getOpcode() != TargetOpcode::G_AND)
892 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
896 if (IsSupportedExtend(*Def))
899 unsigned Opc =
Def->getOpcode();
900 if (Opc != TargetOpcode::G_SHL && Opc != TargetOpcode::G_ASHR &&
901 Opc != TargetOpcode::G_LSHR)
908 uint64_t ShiftAmt = MaybeShiftAmt->Value.getZExtValue();
915 if (IsSupportedExtend(*ShiftLHS))
916 return (ShiftAmt <= 4) ? 2 : 1;
918 LLT Ty =
MRI.getType(
Def->getOperand(0).getReg());
922 if ((ShiftSize == 32 && ShiftAmt <= 31) ||
923 (ShiftSize == 64 && ShiftAmt <= 63))
931 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
957 Register TheLHS = GetRegForProfit(LHS);
958 Register TheRHS = GetRegForProfit(RHS);
972 MI.getOperand(2).setReg(RHS);
973 MI.getOperand(3).setReg(LHS);
984 LLT DstTy =
MRI.getType(LHS);
986 assert(DstTy ==
MRI.getType(RHS) &&
"Src and Dst types must match!");
1038 assert(
MI.getOpcode() == TargetOpcode::G_FCMP);
1042 LLT DstTy =
MRI.getType(Dst);
1046 unsigned EltSize =
MRI.getType(LHS).getScalarSizeInBits();
1047 if (EltSize == 16 && !
ST.hasFullFP16())
1049 if (EltSize != 16 && EltSize != 32 && EltSize != 64)
1058 assert(
MI.getOpcode() == TargetOpcode::G_FCMP);
1061 const auto &CmpMI = cast<GFCmp>(
MI);
1068 LLT DstTy =
MRI.getType(Dst);
1075 bool Invert =
false;
1077 if ((Pred == CmpInst::Predicate::FCMP_ORD ||
1078 Pred == CmpInst::Predicate::FCMP_UNO) &&
1095 ST.getTargetLowering()->getTargetMachine().Options.NoNaNsFPMath;
1097 auto Cmp = getVectorFCMP(
CC, LHS, RHS, IsZero, NoNans,
MRI);
1102 auto Cmp2 = getVectorFCMP(CC2, LHS, RHS, IsZero, NoNans,
MRI);
1103 auto Cmp2Dst = Cmp2(MIB);
1104 auto Cmp1Dst =
Cmp(MIB);
1109 MRI.replaceRegWith(Dst, CmpRes);
1110 MI.eraseFromParent();
1115 assert(
MI.getOpcode() == TargetOpcode::G_STORE);
1117 if (
MRI.getType(DstReg).isVector())
1123 return MRI.getType(SrcReg).getSizeInBits() <= 64;
1129 assert(
MI.getOpcode() == TargetOpcode::G_STORE);
1131 MI.getOperand(0).setReg(SrcReg);
1139 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1141 LLT DstTy =
MRI.getType(DstReg);
1147 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1148 B.setInstrAndDebugLoc(
MI);
1150 Helper.lower(
MI, 0,
LLT());
1157 auto &Unmerge = cast<GUnmerge>(
MI);
1158 if (Unmerge.getNumDefs() != 2)
1160 if (!
MRI.use_nodbg_empty(Unmerge.getReg(1)))
1163 LLT DstTy =
MRI.getType(Unmerge.getReg(0));
1175 if (!LowestVal || LowestVal->Value.getZExtValue() != DstTy.
getSizeInBytes())
1178 if (!getOpcodeDef<GImplicitDef>(ExtSrc2,
MRI))
1181 MatchInfo = ExtSrc1;
1191 MI.getOperand(0).setReg(
MI.getOperand(1).getReg());
1192 MI.getOperand(1).setReg(Dst1);
1193 MI.getOperand(2).setReg(SrcReg);
1204 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
1210 unsigned I1Opc =
I1->getOpcode();
1212 if (((I1Opc == TargetOpcode::G_ZEXT && I2Opc == TargetOpcode::G_ZEXT) ||
1213 (I1Opc == TargetOpcode::G_SEXT && I2Opc == TargetOpcode::G_SEXT)) &&
1214 (
MRI.getType(
I1->getOperand(0).getReg()).getScalarSizeInBits() ==
1215 MRI.getType(
I1->getOperand(1).getReg()).getScalarSizeInBits() * 2) &&
1230 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
1231 "Expected a G_MUL instruction");
1234 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
1239 unsigned I1Opc =
I1->getOpcode();
1241 if (((I1Opc == TargetOpcode::G_ZEXT && I2Opc == TargetOpcode::G_ZEXT) ||
1242 (I1Opc == TargetOpcode::G_SEXT && I2Opc == TargetOpcode::G_SEXT)) &&
1243 (
MRI.getType(
I1->getOperand(0).getReg()).getScalarSizeInBits() ==
1244 MRI.getType(
I1->getOperand(1).getReg()).getScalarSizeInBits() * 2) &&
1248 B.setInstrAndDebugLoc(
MI);
1249 B.buildInstr(
I1->getOpcode() == TargetOpcode::G_ZEXT ? AArch64::G_UMULL
1251 {MI.getOperand(0).getReg()},
1252 {I1->getOperand(1).getReg(), I2->getOperand(1).getReg()});
1253 MI.eraseFromParent();
1258 B.setInstrAndDebugLoc(
MI);
1259 Helper.fewerElementsVector(
1266class AArch64PostLegalizerLoweringImpl :
public Combiner {
1270 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig;
1274 AArch64PostLegalizerLoweringImpl(
1277 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,
1280 static const char *
getName() {
return "AArch6400PreLegalizerCombiner"; }
1285#define GET_GICOMBINER_CLASS_MEMBERS
1286#include "AArch64GenPostLegalizeGILowering.inc"
1287#undef GET_GICOMBINER_CLASS_MEMBERS
1290#define GET_GICOMBINER_IMPL
1291#include "AArch64GenPostLegalizeGILowering.inc"
1292#undef GET_GICOMBINER_IMPL
1294AArch64PostLegalizerLoweringImpl::AArch64PostLegalizerLoweringImpl(
1297 const AArch64PostLegalizerLoweringImplRuleConfig &RuleConfig,
1299 :
Combiner(MF, CInfo, TPC, nullptr, CSEInfo),
1300 Helper(Observer,
B,
true), RuleConfig(RuleConfig),
1303#include
"AArch64GenPostLegalizeGILowering.inc"
1312 AArch64PostLegalizerLowering();
1315 return "AArch64PostLegalizerLowering";
1322 AArch64PostLegalizerLoweringImplRuleConfig RuleConfig;
1326void AArch64PostLegalizerLowering::getAnalysisUsage(
AnalysisUsage &AU)
const {
1333AArch64PostLegalizerLowering::AArch64PostLegalizerLowering()
1337 if (!RuleConfig.parseCommandLineOption())
1341bool AArch64PostLegalizerLowering::runOnMachineFunction(
MachineFunction &MF) {
1343 MachineFunctionProperties::Property::FailedISel))
1346 MachineFunctionProperties::Property::Legalized) &&
1347 "Expected a legalized function?");
1348 auto *TPC = &getAnalysis<TargetPassConfig>();
1354 F.hasOptSize(),
F.hasMinSize());
1355 AArch64PostLegalizerLoweringImpl Impl(MF, CInfo, TPC,
nullptr,
1357 return Impl.combineMachineInstrs();
1360char AArch64PostLegalizerLowering::ID = 0;
1362 "Lower AArch64 MachineInstrs after legalization",
false,
1371 return new AArch64PostLegalizerLowering();
unsigned const MachineRegisterInfo * MRI
static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt)
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift righ...
static bool isLegalArithImmed(uint64_t C)
static bool isREVMask(ArrayRef< int > M, EVT VT, unsigned BlockSize)
isREVMask - Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
static bool isTRNMask(ArrayRef< int > M, EVT VT, unsigned &WhichResult)
static bool isINSMask(ArrayRef< int > M, int NumInputElements, bool &DstIsLeft, int &Anomaly)
static unsigned getCmpOperandFoldingProfit(SDValue Op)
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
static bool isCMN(SDValue Op, ISD::CondCode CC)
This file declares the targeting of the Machinelegalizer class for AArch64.
#define GET_GICOMBINER_CONSTRUCTOR_INITS
Lower AArch64 MachineInstrs after legalization
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This contains common combine transformations that may be used in a combine pass,or by the target else...
Option class for Targets to specify which operations are combined how and when.
This contains the base class for all Combiners generated by TableGen.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This contains common code to allow clients to notify changes to machine instr.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static const int BlockSize
Target-Independent Code Generator Pass Configuration Options pass.
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
unsigned logBase2() const
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ ICMP_ULT
unsigned less than
@ ICMP_SGE
signed greater or equal
@ ICMP_ULE
unsigned less or equal
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
virtual bool tryCombineAll(MachineInstr &I) const =0
FunctionPass class - This class is used to implement most global optimizations.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
constexpr unsigned getScalarSizeInBits() const
constexpr LLT multiplyElements(int Factor) const
Produce a vector type that is Factor times bigger, preserving the element type.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr bool isScalable() const
Returns true if the LLT is a scalable vector.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
bool hasProperty(Property P) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineFunctionProperties & getProperties() const
Get the function properties.
Helper class to build MachineInstr.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineFunction & getMF()
Getter for the function we currently build.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Target-Independent Code Generator Pass Configuration Options.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::optional< RegOrConstant > getAArch64VectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI)
void changeVectorFCMPPredToAArch64CC(const CmpInst::Predicate P, AArch64CC::CondCode &CondCode, AArch64CC::CondCode &CondCode2, bool &Invert)
Find the AArch64 condition codes necessary to represent P for a vector floating point comparison.
std::optional< int64_t > getAArch64VectorSplatScalar(const MachineInstr &MI, const MachineRegisterInfo &MRI)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
operand_type_match m_Reg()
SpecificConstantMatch m_ZeroInt()
{ Convenience matchers for specific integer values.
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
void initializeAArch64PostLegalizerLoweringPass(PassRegistry &)
bool isUZPMask(ArrayRef< int > M, EVT VT, unsigned &WhichResultOut)
Return true for uzp1 or uzp2 masks of the form: <0, 2, 4, 6, 8, 10, 12, 14> or <1,...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
FunctionPass * createAArch64PostLegalizerLowering()
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.