42#define DEBUG_TYPE "legalizer"
45using namespace LegalizeActions;
46using namespace MIPatternMatch;
55static std::pair<int, int>
61 unsigned NumParts =
Size / NarrowSize;
62 unsigned LeftoverSize =
Size - NumParts * NarrowSize;
65 if (LeftoverSize == 0)
70 if (LeftoverSize % EltSize != 0)
80 return std::make_pair(NumParts, NumLeftover);
107 : MIRBuilder(Builder), Observer(Observer),
MRI(MF.getRegInfo()),
108 LI(*MF.getSubtarget().getLegalizerInfo()),
109 TLI(*MF.getSubtarget().getTargetLowering()), KB(nullptr) {}
114 : MIRBuilder(
B), Observer(Observer),
MRI(MF.getRegInfo()), LI(LI),
115 TLI(*MF.getSubtarget().getTargetLowering()), KB(KB) {}
124 if (isa<GIntrinsic>(
MI))
127 switch (Step.Action) {
142 return bitcast(
MI, Step.TypeIdx, Step.NewType);
145 return lower(
MI, Step.TypeIdx, Step.NewType);
162void LegalizerHelper::insertParts(
Register DstReg,
184 assert(LeftoverRegs.
size() == 1 &&
"Expected one leftover register");
186 for (
auto Reg : concat<const Register>(PartRegs, LeftoverRegs))
188 return mergeMixedSubvectors(DstReg, AllRegs);
193 for (
auto PartReg : concat<const Register>(PartRegs, LeftoverRegs))
194 extractGCDType(GCDRegs, GCDTy, PartReg);
195 LLT ResultLCMTy = buildLCMMergePieces(ResultTy, LeftoverTy, GCDTy, GCDRegs);
196 buildWidenedRemergeToDst(DstReg, ResultLCMTy, GCDRegs);
209void LegalizerHelper::mergeMixedSubvectors(
Register DstReg,
212 for (
unsigned i = 0; i < PartRegs.
size() - 1; ++i)
213 appendVectorElts(AllElts, PartRegs[i]);
219 appendVectorElts(AllElts, Leftover);
227 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
229 const int StartIdx = Regs.
size();
230 const int NumResults =
MI.getNumOperands() - 1;
232 for (
int I = 0;
I != NumResults; ++
I)
233 Regs[StartIdx +
I] =
MI.getOperand(
I).getReg();
239 if (SrcTy == GCDTy) {
254 extractGCDType(Parts, GCDTy, SrcReg);
258LLT LegalizerHelper::buildLCMMergePieces(
LLT DstTy,
LLT NarrowTy,
LLT GCDTy,
260 unsigned PadStrategy) {
265 int NumOrigSrc = VRegs.
size();
271 if (NumOrigSrc < NumParts * NumSubParts) {
272 if (PadStrategy == TargetOpcode::G_ZEXT)
274 else if (PadStrategy == TargetOpcode::G_ANYEXT)
277 assert(PadStrategy == TargetOpcode::G_SEXT);
298 for (
int I = 0;
I != NumParts; ++
I) {
299 bool AllMergePartsArePadding =
true;
302 for (
int J = 0; J != NumSubParts; ++J) {
303 int Idx =
I * NumSubParts + J;
304 if (
Idx >= NumOrigSrc) {
305 SubMerge[J] = PadReg;
309 SubMerge[J] = VRegs[
Idx];
312 AllMergePartsArePadding =
false;
318 if (AllMergePartsArePadding && !AllPadReg) {
319 if (PadStrategy == TargetOpcode::G_ANYEXT)
321 else if (PadStrategy == TargetOpcode::G_ZEXT)
331 Remerge[
I] = AllPadReg;
335 if (NumSubParts == 1)
336 Remerge[
I] = SubMerge[0];
341 if (AllMergePartsArePadding && !AllPadReg)
342 AllPadReg = Remerge[
I];
345 VRegs = std::move(Remerge);
349void LegalizerHelper::buildWidenedRemergeToDst(
Register DstReg,
LLT LCMTy,
356 if (DstTy == LCMTy) {
370 UnmergeDefs[0] = DstReg;
371 for (
unsigned I = 1;
I != NumDefs; ++
I)
383#define RTLIBCASE_INT(LibcallPrefix) \
387 return RTLIB::LibcallPrefix##32; \
389 return RTLIB::LibcallPrefix##64; \
391 return RTLIB::LibcallPrefix##128; \
393 llvm_unreachable("unexpected size"); \
397#define RTLIBCASE(LibcallPrefix) \
401 return RTLIB::LibcallPrefix##32; \
403 return RTLIB::LibcallPrefix##64; \
405 return RTLIB::LibcallPrefix##80; \
407 return RTLIB::LibcallPrefix##128; \
409 llvm_unreachable("unexpected size"); \
414 case TargetOpcode::G_MUL:
416 case TargetOpcode::G_SDIV:
418 case TargetOpcode::G_UDIV:
420 case TargetOpcode::G_SREM:
422 case TargetOpcode::G_UREM:
424 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
426 case TargetOpcode::G_FADD:
428 case TargetOpcode::G_FSUB:
430 case TargetOpcode::G_FMUL:
432 case TargetOpcode::G_FDIV:
434 case TargetOpcode::G_FEXP:
436 case TargetOpcode::G_FEXP2:
438 case TargetOpcode::G_FEXP10:
440 case TargetOpcode::G_FREM:
442 case TargetOpcode::G_FPOW:
444 case TargetOpcode::G_FPOWI:
446 case TargetOpcode::G_FMA:
448 case TargetOpcode::G_FSIN:
450 case TargetOpcode::G_FCOS:
452 case TargetOpcode::G_FTAN:
454 case TargetOpcode::G_FLOG10:
456 case TargetOpcode::G_FLOG:
458 case TargetOpcode::G_FLOG2:
460 case TargetOpcode::G_FLDEXP:
462 case TargetOpcode::G_FCEIL:
464 case TargetOpcode::G_FFLOOR:
466 case TargetOpcode::G_FMINNUM:
468 case TargetOpcode::G_FMAXNUM:
470 case TargetOpcode::G_FSQRT:
472 case TargetOpcode::G_FRINT:
474 case TargetOpcode::G_FNEARBYINT:
476 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
478 case TargetOpcode::G_INTRINSIC_LRINT:
480 case TargetOpcode::G_INTRINSIC_LLRINT:
506 if (CallerAttrs.
hasRetAttr(Attribute::ZExt) ||
518 if (
MI.getOpcode() == TargetOpcode::G_BZERO)
525 if (!VReg.
isVirtual() || VReg != Next->getOperand(1).getReg())
528 Register PReg = Next->getOperand(0).getReg();
536 if (Ret->getNumImplicitOperands() != 1)
539 if (!Ret->getOperand(0).isReg() || PReg != Ret->getOperand(0).getReg())
563 Info.OrigRet = Result;
566 (Result.Ty->isVoidTy() ||
571 std::copy(Args.begin(), Args.end(), std::back_inserter(
Info.OrigArgs));
572 if (!CLI.lowerCall(MIRBuilder,
Info))
575 if (
MI &&
Info.LoweredTailCall) {
576 assert(
Info.IsTailCall &&
"Lowered tail call when it wasn't a tail call?");
587 "Expected instr following MI to be return or debug inst?");
591 }
while (
MI->getNextNode());
621 Args.push_back({MO.getReg(), OpType, 0});
623 {
MI.getOperand(0).
getReg(), OpType, 0}, Args,
634 for (
unsigned i = 0; i <
MI.getNumOperands() - 1; ++i) {
638 LLT OpLLT =
MRI.getType(Reg);
639 Type *OpTy =
nullptr;
644 Args.push_back({Reg, OpTy, 0});
650 unsigned Opc =
MI.getOpcode();
652 case TargetOpcode::G_BZERO:
653 RTLibcall = RTLIB::BZERO;
655 case TargetOpcode::G_MEMCPY:
656 RTLibcall = RTLIB::MEMCPY;
657 Args[0].Flags[0].setReturned();
659 case TargetOpcode::G_MEMMOVE:
660 RTLibcall = RTLIB::MEMMOVE;
661 Args[0].Flags[0].setReturned();
663 case TargetOpcode::G_MEMSET:
664 RTLibcall = RTLIB::MEMSET;
665 Args[0].Flags[0].setReturned();
670 const char *
Name = TLI.getLibcallName(RTLibcall);
680 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
684 MI.getOperand(
MI.getNumOperands() - 1).getImm() &&
687 std::copy(Args.begin(), Args.end(), std::back_inserter(
Info.OrigArgs));
688 if (!CLI.lowerCall(MIRBuilder,
Info))
691 if (
Info.LoweredTailCall) {
692 assert(
Info.IsTailCall &&
"Lowered tail call when it wasn't a tail call?");
703 "Expected instr following MI to be return or debug inst?");
707 }
while (
MI.getNextNode());
717 unsigned Opc =
MI.getOpcode();
718 auto &AtomicMI = cast<GMemOperation>(
MI);
719 auto &MMO = AtomicMI.getMMO();
720 auto Ordering = MMO.getMergedOrdering();
721 LLT MemType = MMO.getMemoryType();
724 return RTLIB::UNKNOWN_LIBCALL;
726#define LCALLS(A, B) \
727 { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
729 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
731 case TargetOpcode::G_ATOMIC_CMPXCHG:
732 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
734 return getOutlineAtomicHelper(LC, Ordering, MemSize);
736 case TargetOpcode::G_ATOMICRMW_XCHG: {
738 return getOutlineAtomicHelper(LC, Ordering, MemSize);
740 case TargetOpcode::G_ATOMICRMW_ADD:
741 case TargetOpcode::G_ATOMICRMW_SUB: {
743 return getOutlineAtomicHelper(LC, Ordering, MemSize);
745 case TargetOpcode::G_ATOMICRMW_AND: {
747 return getOutlineAtomicHelper(LC, Ordering, MemSize);
749 case TargetOpcode::G_ATOMICRMW_OR: {
751 return getOutlineAtomicHelper(LC, Ordering, MemSize);
753 case TargetOpcode::G_ATOMICRMW_XOR: {
755 return getOutlineAtomicHelper(LC, Ordering, MemSize);
758 return RTLIB::UNKNOWN_LIBCALL;
771 unsigned Opc =
MI.getOpcode();
773 case TargetOpcode::G_ATOMIC_CMPXCHG:
774 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
777 auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
778 MI.getFirst4RegLLTs();
781 if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
782 std::tie(Ret, RetLLT,
Success, SuccessLLT, Mem, MemLLT, Cmp, CmpLLT, New,
783 NewLLT) =
MI.getFirst5RegLLTs();
793 case TargetOpcode::G_ATOMICRMW_XCHG:
794 case TargetOpcode::G_ATOMICRMW_ADD:
795 case TargetOpcode::G_ATOMICRMW_SUB:
796 case TargetOpcode::G_ATOMICRMW_AND:
797 case TargetOpcode::G_ATOMICRMW_OR:
798 case TargetOpcode::G_ATOMICRMW_XOR: {
799 auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] =
MI.getFirst3RegLLTs();
802 if (Opc == TargetOpcode::G_ATOMICRMW_AND)
806 else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
821 const char *
Name = TLI.getLibcallName(RTLibcall);
831 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
835 std::copy(Args.begin(), Args.end(), std::back_inserter(
Info.OrigArgs));
836 if (!CLI.lowerCall(MIRBuilder,
Info))
848 case TargetOpcode::G_FPEXT:
850 case TargetOpcode::G_FPTRUNC:
852 case TargetOpcode::G_FPTOSI:
854 case TargetOpcode::G_FPTOUI:
856 case TargetOpcode::G_SITOFP:
858 case TargetOpcode::G_UITOFP:
870 {{
MI.getOperand(1).
getReg(), FromType, 0}}, LocObserver, &
MI);
876 switch (
MI.getOpcode()) {
877 case TargetOpcode::G_GET_FPENV:
878 RTLibcall = RTLIB::FEGETENV;
880 case TargetOpcode::G_SET_FPENV:
881 case TargetOpcode::G_RESET_FPENV:
882 RTLibcall = RTLIB::FESETENV;
884 case TargetOpcode::G_GET_FPMODE:
885 RTLibcall = RTLIB::FEGETMODE;
887 case TargetOpcode::G_SET_FPMODE:
888 case TargetOpcode::G_RESET_FPMODE:
889 RTLibcall = RTLIB::FESETMODE;
918 auto &Ctx = MF.getFunction().getContext();
929 unsigned TempAddrSpace =
DL.getAllocaAddrSpace();
936 LocObserver,
nullptr);
958 auto &Ctx = MF.getFunction().getContext();
974 unsigned TempAddrSpace =
DL.getAllocaAddrSpace();
980 LocObserver,
nullptr);
997 unsigned AddrSpace =
DL.getDefaultGlobalsAddressSpace();
999 unsigned PtrSize =
DL.getPointerSizeInBits(AddrSpace);
1016 switch (
MI.getOpcode()) {
1019 case TargetOpcode::G_MUL:
1020 case TargetOpcode::G_SDIV:
1021 case TargetOpcode::G_UDIV:
1022 case TargetOpcode::G_SREM:
1023 case TargetOpcode::G_UREM:
1024 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
1033 case TargetOpcode::G_FADD:
1034 case TargetOpcode::G_FSUB:
1035 case TargetOpcode::G_FMUL:
1036 case TargetOpcode::G_FDIV:
1037 case TargetOpcode::G_FMA:
1038 case TargetOpcode::G_FPOW:
1039 case TargetOpcode::G_FREM:
1040 case TargetOpcode::G_FCOS:
1041 case TargetOpcode::G_FSIN:
1042 case TargetOpcode::G_FTAN:
1043 case TargetOpcode::G_FLOG10:
1044 case TargetOpcode::G_FLOG:
1045 case TargetOpcode::G_FLOG2:
1046 case TargetOpcode::G_FLDEXP:
1047 case TargetOpcode::G_FEXP:
1048 case TargetOpcode::G_FEXP2:
1049 case TargetOpcode::G_FEXP10:
1050 case TargetOpcode::G_FCEIL:
1051 case TargetOpcode::G_FFLOOR:
1052 case TargetOpcode::G_FMINNUM:
1053 case TargetOpcode::G_FMAXNUM:
1054 case TargetOpcode::G_FSQRT:
1055 case TargetOpcode::G_FRINT:
1056 case TargetOpcode::G_FNEARBYINT:
1057 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
1062 LLVM_DEBUG(
dbgs() <<
"No libcall available for type " << LLTy <<
".\n");
1070 case TargetOpcode::G_INTRINSIC_LRINT:
1071 case TargetOpcode::G_INTRINSIC_LLRINT: {
1078 LLVM_DEBUG(
dbgs() <<
"No libcall available for type " << LLTy <<
".\n");
1084 {{
MI.getOperand(1).
getReg(), HLTy, 0}}, LocObserver, &
MI);
1087 MI.eraseFromParent();
1090 case TargetOpcode::G_FPOWI: {
1097 LLVM_DEBUG(
dbgs() <<
"No libcall available for type " << LLTy <<
".\n");
1101 std::initializer_list<CallLowering::ArgInfo> Args = {
1102 {
MI.getOperand(1).getReg(), HLTy, 0},
1103 {
MI.getOperand(2).getReg(), ITy, 1}};
1106 Args, LocObserver, &
MI);
1111 case TargetOpcode::G_FPEXT:
1112 case TargetOpcode::G_FPTRUNC: {
1115 if (!FromTy || !ToTy)
1123 case TargetOpcode::G_FPTOSI:
1124 case TargetOpcode::G_FPTOUI: {
1129 if ((ToSize != 32 && ToSize != 64 && ToSize != 128) || !FromTy)
1137 case TargetOpcode::G_SITOFP:
1138 case TargetOpcode::G_UITOFP: {
1142 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
1153 case TargetOpcode::G_ATOMICRMW_XCHG:
1154 case TargetOpcode::G_ATOMICRMW_ADD:
1155 case TargetOpcode::G_ATOMICRMW_SUB:
1156 case TargetOpcode::G_ATOMICRMW_AND:
1157 case TargetOpcode::G_ATOMICRMW_OR:
1158 case TargetOpcode::G_ATOMICRMW_XOR:
1159 case TargetOpcode::G_ATOMIC_CMPXCHG:
1160 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1166 case TargetOpcode::G_BZERO:
1167 case TargetOpcode::G_MEMCPY:
1168 case TargetOpcode::G_MEMMOVE:
1169 case TargetOpcode::G_MEMSET: {
1174 MI.eraseFromParent();
1177 case TargetOpcode::G_GET_FPENV:
1178 case TargetOpcode::G_GET_FPMODE: {
1184 case TargetOpcode::G_SET_FPENV:
1185 case TargetOpcode::G_SET_FPMODE: {
1191 case TargetOpcode::G_RESET_FPENV:
1192 case TargetOpcode::G_RESET_FPMODE: {
1201 MI.eraseFromParent();
1211 switch (
MI.getOpcode()) {
1214 case TargetOpcode::G_IMPLICIT_DEF: {
1224 if (SizeOp0 % NarrowSize != 0) {
1225 LLT ImplicitTy = NarrowTy;
1232 MI.eraseFromParent();
1236 int NumParts = SizeOp0 / NarrowSize;
1239 for (
int i = 0; i < NumParts; ++i)
1246 MI.eraseFromParent();
1249 case TargetOpcode::G_CONSTANT: {
1251 const APInt &Val =
MI.getOperand(1).getCImm()->getValue();
1254 int NumParts = TotalSize / NarrowSize;
1257 for (
int I = 0;
I != NumParts; ++
I) {
1258 unsigned Offset =
I * NarrowSize;
1265 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
1267 if (LeftoverBits != 0) {
1271 Val.
lshr(NumParts * NarrowSize).
trunc(LeftoverBits));
1275 insertParts(
MI.getOperand(0).getReg(),
1276 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
1278 MI.eraseFromParent();
1281 case TargetOpcode::G_SEXT:
1282 case TargetOpcode::G_ZEXT:
1283 case TargetOpcode::G_ANYEXT:
1285 case TargetOpcode::G_TRUNC: {
1291 LLVM_DEBUG(
dbgs() <<
"Can't narrow trunc to type " << NarrowTy <<
"\n");
1297 MI.eraseFromParent();
1300 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
1301 case TargetOpcode::G_FREEZE: {
1312 for (
unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
1319 MI.eraseFromParent();
1322 case TargetOpcode::G_ADD:
1323 case TargetOpcode::G_SUB:
1324 case TargetOpcode::G_SADDO:
1325 case TargetOpcode::G_SSUBO:
1326 case TargetOpcode::G_SADDE:
1327 case TargetOpcode::G_SSUBE:
1328 case TargetOpcode::G_UADDO:
1329 case TargetOpcode::G_USUBO:
1330 case TargetOpcode::G_UADDE:
1331 case TargetOpcode::G_USUBE:
1333 case TargetOpcode::G_MUL:
1334 case TargetOpcode::G_UMULH:
1336 case TargetOpcode::G_EXTRACT:
1338 case TargetOpcode::G_INSERT:
1340 case TargetOpcode::G_LOAD: {
1341 auto &LoadMI = cast<GLoad>(
MI);
1342 Register DstReg = LoadMI.getDstReg();
1347 if (8 * LoadMI.getMemSize().getValue() != DstTy.
getSizeInBits()) {
1351 LoadMI.eraseFromParent();
1357 case TargetOpcode::G_ZEXTLOAD:
1358 case TargetOpcode::G_SEXTLOAD: {
1359 auto &LoadMI = cast<GExtLoad>(
MI);
1360 Register DstReg = LoadMI.getDstReg();
1361 Register PtrReg = LoadMI.getPointerReg();
1364 auto &MMO = LoadMI.getMMO();
1367 if (MemSize == NarrowSize) {
1369 }
else if (MemSize < NarrowSize) {
1371 }
else if (MemSize > NarrowSize) {
1376 if (isa<GZExtLoad>(LoadMI))
1381 LoadMI.eraseFromParent();
1384 case TargetOpcode::G_STORE: {
1385 auto &StoreMI = cast<GStore>(
MI);
1387 Register SrcReg = StoreMI.getValueReg();
1392 int NumParts = SizeOp0 / NarrowSize;
1394 unsigned LeftoverBits = SrcTy.
getSizeInBits() - HandledSize;
1395 if (SrcTy.
isVector() && LeftoverBits != 0)
1398 if (8 * StoreMI.getMemSize().getValue() != SrcTy.
getSizeInBits()) {
1402 StoreMI.eraseFromParent();
1408 case TargetOpcode::G_SELECT:
1410 case TargetOpcode::G_AND:
1411 case TargetOpcode::G_OR:
1412 case TargetOpcode::G_XOR: {
1424 case TargetOpcode::G_SHL:
1425 case TargetOpcode::G_LSHR:
1426 case TargetOpcode::G_ASHR:
1428 case TargetOpcode::G_CTLZ:
1429 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1430 case TargetOpcode::G_CTTZ:
1431 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1432 case TargetOpcode::G_CTPOP:
1434 switch (
MI.getOpcode()) {
1435 case TargetOpcode::G_CTLZ:
1436 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1438 case TargetOpcode::G_CTTZ:
1439 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1441 case TargetOpcode::G_CTPOP:
1451 case TargetOpcode::G_INTTOPTR:
1459 case TargetOpcode::G_PTRTOINT:
1467 case TargetOpcode::G_PHI: {
1470 if (SizeOp0 % NarrowSize != 0)
1473 unsigned NumParts = SizeOp0 / NarrowSize;
1477 for (
unsigned i = 1; i <
MI.getNumOperands(); i += 2) {
1485 for (
unsigned i = 0; i < NumParts; ++i) {
1489 for (
unsigned j = 1; j <
MI.getNumOperands(); j += 2)
1490 MIB.
addUse(SrcRegs[j / 2][i]).
add(
MI.getOperand(j + 1));
1495 MI.eraseFromParent();
1498 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1499 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1503 int OpIdx =
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1509 case TargetOpcode::G_ICMP: {
1528 if (!
extractParts(
MI.getOperand(3).getReg(), SrcTy, NarrowTy, Unused,
1529 RHSPartRegs, RHSLeftoverRegs,
MIRBuilder, MRI))
1542 for (
auto LHSAndRHS :
zip(LHSPartRegs, RHSPartRegs)) {
1543 auto LHS = std::get<0>(LHSAndRHS);
1544 auto RHS = std::get<1>(LHSAndRHS);
1552 for (
auto LHSAndRHS :
zip(LHSLeftoverRegs, RHSLeftoverRegs)) {
1553 auto LHS = std::get<0>(LHSAndRHS);
1554 auto RHS = std::get<1>(LHSAndRHS);
1556 LLT GCDTy = extractGCDType(WidenedXors, NarrowTy, LeftoverTy,
Xor);
1557 buildLCMMergePieces(LeftoverTy, NarrowTy, GCDTy, WidenedXors,
1558 TargetOpcode::G_ZEXT);
1565 assert(Xors.
size() >= 2 &&
"Should have gotten at least two Xors?");
1567 for (
unsigned I = 2, E = Xors.
size();
I < E; ++
I)
1572 assert(LHSPartRegs.
size() == 2 &&
"Expected exactly 2 LHS part regs?");
1573 assert(RHSPartRegs.
size() == 2 &&
"Expected exactly 2 RHS part regs?");
1585 MI.eraseFromParent();
1588 case TargetOpcode::G_FCMP:
1597 case TargetOpcode::G_SEXT_INREG: {
1601 int64_t SizeInBits =
MI.getOperand(2).getImm();
1611 MO1.
setReg(TruncMIB.getReg(0));
1626 if (SizeOp0 % NarrowSize != 0)
1628 int NumParts = SizeOp0 / NarrowSize;
1636 for (
int i = 0; i < NumParts; ++i) {
1652 for (
int i = 0; i < NumParts; ++i) {
1655 PartialExtensionReg = DstRegs.
back();
1657 assert(PartialExtensionReg &&
1658 "Expected to visit partial extension before full");
1659 if (FullExtensionReg) {
1666 FullExtensionReg = DstRegs.
back();
1671 TargetOpcode::G_SEXT_INREG, {NarrowTy},
1674 PartialExtensionReg = DstRegs.
back();
1681 MI.eraseFromParent();
1684 case TargetOpcode::G_BSWAP:
1685 case TargetOpcode::G_BITREVERSE: {
1686 if (SizeOp0 % NarrowSize != 0)
1691 unsigned NumParts = SizeOp0 / NarrowSize;
1692 extractParts(
MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs,
1695 for (
unsigned i = 0; i < NumParts; ++i) {
1697 {SrcRegs[NumParts - 1 - i]});
1704 MI.eraseFromParent();
1707 case TargetOpcode::G_PTR_ADD:
1708 case TargetOpcode::G_PTRMASK: {
1716 case TargetOpcode::G_FPTOUI:
1717 case TargetOpcode::G_FPTOSI:
1719 case TargetOpcode::G_FPEXT:
1726 case TargetOpcode::G_FLDEXP:
1727 case TargetOpcode::G_STRICT_FLDEXP:
1729 case TargetOpcode::G_VSCALE: {
1740 MI.eraseFromParent();
1768 unsigned OpIdx,
unsigned ExtOpcode) {
1771 MO.
setReg(ExtB.getReg(0));
1778 MO.
setReg(ExtB.getReg(0));
1782 unsigned OpIdx,
unsigned TruncOpcode) {
1791 unsigned OpIdx,
unsigned ExtOpcode) {
1830LegalizerHelper::widenScalarMergeValues(
MachineInstr &
MI,
unsigned TypeIdx,
1835 auto [DstReg, DstTy, Src1Reg, Src1Ty] =
MI.getFirst2RegLLTs();
1836 if (DstTy.isVector())
1843 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
1845 unsigned NumOps =
MI.getNumOperands();
1846 unsigned NumSrc =
MI.getNumOperands() - 1;
1847 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1849 if (WideSize >= DstSize) {
1853 for (
unsigned I = 2;
I != NumOps; ++
I) {
1854 const unsigned Offset = (
I - 1) * PartSize;
1861 Register NextResult =
I + 1 == NumOps && WideTy == DstTy ? DstReg :
1867 ResultReg = NextResult;
1870 if (WideSize > DstSize)
1872 else if (DstTy.isPointer())
1875 MI.eraseFromParent();
1900 const int GCD = std::gcd(SrcSize, WideSize);
1911 if (GCD == SrcSize) {
1915 for (
int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1921 if (
static_cast<int>(Unmerges.
size()) != NumMerge * WideSize) {
1923 for (
int I = Unmerges.
size();
I != NumMerge * WideSize; ++
I)
1927 const int PartsPerGCD = WideSize / GCD;
1931 for (
int I = 0;
I != NumMerge; ++
I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1946 MI.eraseFromParent();
1951LegalizerHelper::widenScalarUnmergeValues(
MachineInstr &
MI,
unsigned TypeIdx,
1956 int NumDst =
MI.getNumOperands() - 1;
1957 Register SrcReg =
MI.getOperand(NumDst).getReg();
1962 Register Dst0Reg =
MI.getOperand(0).getReg();
1972 dbgs() <<
"Not casting non-integral address space integer\n");
1993 for (
int I = 1;
I != NumDst; ++
I) {
1999 MI.eraseFromParent();
2010 LLVM_DEBUG(
dbgs() <<
"Widening pointer source types not implemented\n");
2035 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2040 if (PartsPerRemerge == 1) {
2043 for (
int I = 0;
I != NumUnmerge; ++
I) {
2046 for (
int J = 0; J != PartsPerUnmerge; ++J) {
2047 int Idx =
I * PartsPerUnmerge + J;
2049 MIB.addDef(
MI.getOperand(
Idx).getReg());
2056 MIB.addUse(Unmerge.getReg(
I));
2060 for (
int J = 0; J != NumUnmerge; ++J)
2061 extractGCDType(Parts, GCDTy, Unmerge.getReg(J));
2064 for (
int I = 0;
I != NumDst; ++
I) {
2065 for (
int J = 0; J < PartsPerRemerge; ++J) {
2066 const int Idx =
I * PartsPerRemerge + J;
2071 RemergeParts.
clear();
2075 MI.eraseFromParent();
2080LegalizerHelper::widenScalarExtract(
MachineInstr &
MI,
unsigned TypeIdx,
2082 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
2083 unsigned Offset =
MI.getOperand(2).getImm();
2086 if (SrcTy.
isVector() || DstTy.isVector())
2102 if (DstTy.isPointer())
2109 MI.eraseFromParent();
2114 LLT ShiftTy = SrcTy;
2123 MI.eraseFromParent();
2154LegalizerHelper::widenScalarInsert(
MachineInstr &
MI,
unsigned TypeIdx,
2156 if (TypeIdx != 0 || WideTy.
isVector())
2166LegalizerHelper::widenScalarAddSubOverflow(
MachineInstr &
MI,
unsigned TypeIdx,
2170 std::optional<Register> CarryIn;
2171 switch (
MI.getOpcode()) {
2174 case TargetOpcode::G_SADDO:
2175 Opcode = TargetOpcode::G_ADD;
2176 ExtOpcode = TargetOpcode::G_SEXT;
2178 case TargetOpcode::G_SSUBO:
2179 Opcode = TargetOpcode::G_SUB;
2180 ExtOpcode = TargetOpcode::G_SEXT;
2182 case TargetOpcode::G_UADDO:
2183 Opcode = TargetOpcode::G_ADD;
2184 ExtOpcode = TargetOpcode::G_ZEXT;
2186 case TargetOpcode::G_USUBO:
2187 Opcode = TargetOpcode::G_SUB;
2188 ExtOpcode = TargetOpcode::G_ZEXT;
2190 case TargetOpcode::G_SADDE:
2191 Opcode = TargetOpcode::G_UADDE;
2192 ExtOpcode = TargetOpcode::G_SEXT;
2193 CarryIn =
MI.getOperand(4).getReg();
2195 case TargetOpcode::G_SSUBE:
2196 Opcode = TargetOpcode::G_USUBE;
2197 ExtOpcode = TargetOpcode::G_SEXT;
2198 CarryIn =
MI.getOperand(4).getReg();
2200 case TargetOpcode::G_UADDE:
2201 Opcode = TargetOpcode::G_UADDE;
2202 ExtOpcode = TargetOpcode::G_ZEXT;
2203 CarryIn =
MI.getOperand(4).getReg();
2205 case TargetOpcode::G_USUBE:
2206 Opcode = TargetOpcode::G_USUBE;
2207 ExtOpcode = TargetOpcode::G_ZEXT;
2208 CarryIn =
MI.getOperand(4).getReg();
2229 LLT CarryOutTy = MRI.
getType(
MI.getOperand(1).getReg());
2232 {LHSExt, RHSExt, *CarryIn})
2244 MI.eraseFromParent();
2249LegalizerHelper::widenScalarAddSubShlSat(
MachineInstr &
MI,
unsigned TypeIdx,
2251 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SADDSAT ||
2252 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
2253 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
2254 bool IsShift =
MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
2255 MI.getOpcode() == TargetOpcode::G_USHLSAT;
2280 {ShiftL, ShiftR},
MI.getFlags());
2288 MI.eraseFromParent();
2293LegalizerHelper::widenScalarMulo(
MachineInstr &
MI,
unsigned TypeIdx,
2302 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SMULO;
2305 LLT OverflowTy = MRI.
getType(OriginalOverflow);
2312 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2321 WideMulCanOverflow ?
MI.getOpcode() : (
unsigned)TargetOpcode::G_MUL;
2324 if (WideMulCanOverflow)
2326 {LeftOperand, RightOperand});
2347 if (WideMulCanOverflow) {
2355 MI.eraseFromParent();
2361 switch (
MI.getOpcode()) {
2364 case TargetOpcode::G_ATOMICRMW_XCHG:
2365 case TargetOpcode::G_ATOMICRMW_ADD:
2366 case TargetOpcode::G_ATOMICRMW_SUB:
2367 case TargetOpcode::G_ATOMICRMW_AND:
2368 case TargetOpcode::G_ATOMICRMW_OR:
2369 case TargetOpcode::G_ATOMICRMW_XOR:
2370 case TargetOpcode::G_ATOMICRMW_MIN:
2371 case TargetOpcode::G_ATOMICRMW_MAX:
2372 case TargetOpcode::G_ATOMICRMW_UMIN:
2373 case TargetOpcode::G_ATOMICRMW_UMAX:
2374 assert(TypeIdx == 0 &&
"atomicrmw with second scalar type");
2380 case TargetOpcode::G_ATOMIC_CMPXCHG:
2381 assert(TypeIdx == 0 &&
"G_ATOMIC_CMPXCHG with second scalar type");
2388 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2398 "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type");
2403 case TargetOpcode::G_EXTRACT:
2404 return widenScalarExtract(
MI, TypeIdx, WideTy);
2405 case TargetOpcode::G_INSERT:
2406 return widenScalarInsert(
MI, TypeIdx, WideTy);
2407 case TargetOpcode::G_MERGE_VALUES:
2408 return widenScalarMergeValues(
MI, TypeIdx, WideTy);
2409 case TargetOpcode::G_UNMERGE_VALUES:
2410 return widenScalarUnmergeValues(
MI, TypeIdx, WideTy);
2411 case TargetOpcode::G_SADDO:
2412 case TargetOpcode::G_SSUBO:
2413 case TargetOpcode::G_UADDO:
2414 case TargetOpcode::G_USUBO:
2415 case TargetOpcode::G_SADDE:
2416 case TargetOpcode::G_SSUBE:
2417 case TargetOpcode::G_UADDE:
2418 case TargetOpcode::G_USUBE:
2419 return widenScalarAddSubOverflow(
MI, TypeIdx, WideTy);
2420 case TargetOpcode::G_UMULO:
2421 case TargetOpcode::G_SMULO:
2422 return widenScalarMulo(
MI, TypeIdx, WideTy);
2423 case TargetOpcode::G_SADDSAT:
2424 case TargetOpcode::G_SSUBSAT:
2425 case TargetOpcode::G_SSHLSAT:
2426 case TargetOpcode::G_UADDSAT:
2427 case TargetOpcode::G_USUBSAT:
2428 case TargetOpcode::G_USHLSAT:
2429 return widenScalarAddSubShlSat(
MI, TypeIdx, WideTy);
2430 case TargetOpcode::G_CTTZ:
2431 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2432 case TargetOpcode::G_CTLZ:
2433 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2434 case TargetOpcode::G_CTPOP: {
2445 unsigned ExtOpc =
MI.getOpcode() == TargetOpcode::G_CTTZ ||
2446 MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF
2447 ? TargetOpcode::G_ANYEXT
2448 : TargetOpcode::G_ZEXT;
2451 unsigned NewOpc =
MI.getOpcode();
2452 if (NewOpc == TargetOpcode::G_CTTZ) {
2461 NewOpc = TargetOpcode::G_CTTZ_ZERO_UNDEF;
2467 if (
MI.getOpcode() == TargetOpcode::G_CTLZ ||
2468 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
2476 MI.eraseFromParent();
2479 case TargetOpcode::G_BSWAP: {
2488 MI.getOperand(0).setReg(DstExt);
2501 case TargetOpcode::G_BITREVERSE: {
2510 MI.getOperand(0).setReg(DstExt);
2519 case TargetOpcode::G_FREEZE:
2520 case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
2527 case TargetOpcode::G_ABS:
2534 case TargetOpcode::G_ADD:
2535 case TargetOpcode::G_AND:
2536 case TargetOpcode::G_MUL:
2537 case TargetOpcode::G_OR:
2538 case TargetOpcode::G_XOR:
2539 case TargetOpcode::G_SUB:
2540 case TargetOpcode::G_SHUFFLE_VECTOR:
2551 case TargetOpcode::G_SBFX:
2552 case TargetOpcode::G_UBFX:
2566 case TargetOpcode::G_SHL:
2582 case TargetOpcode::G_ROTR:
2583 case TargetOpcode::G_ROTL:
2592 case TargetOpcode::G_SDIV:
2593 case TargetOpcode::G_SREM:
2594 case TargetOpcode::G_SMIN:
2595 case TargetOpcode::G_SMAX:
2603 case TargetOpcode::G_SDIVREM:
2612 case TargetOpcode::G_ASHR:
2613 case TargetOpcode::G_LSHR:
2617 unsigned CvtOp =
MI.getOpcode() == TargetOpcode::G_ASHR ?
2618 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2631 case TargetOpcode::G_UDIV:
2632 case TargetOpcode::G_UREM:
2633 case TargetOpcode::G_UMIN:
2634 case TargetOpcode::G_UMAX:
2642 case TargetOpcode::G_UDIVREM:
2651 case TargetOpcode::G_SELECT:
2668 case TargetOpcode::G_FPTOSI:
2669 case TargetOpcode::G_FPTOUI:
2670 case TargetOpcode::G_INTRINSIC_LRINT:
2671 case TargetOpcode::G_INTRINSIC_LLRINT:
2672 case TargetOpcode::G_IS_FPCLASS:
2682 case TargetOpcode::G_SITOFP:
2692 case TargetOpcode::G_UITOFP:
2702 case TargetOpcode::G_LOAD:
2703 case TargetOpcode::G_SEXTLOAD:
2704 case TargetOpcode::G_ZEXTLOAD:
2710 case TargetOpcode::G_STORE: {
2721 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
2727 case TargetOpcode::G_CONSTANT: {
2731 MRI.
getType(
MI.getOperand(0).getReg()));
2732 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
2733 ExtOpc == TargetOpcode::G_ANYEXT) &&
2736 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
2740 SrcMO.
setCImm(ConstantInt::get(Ctx, Val));
2746 case TargetOpcode::G_FCONSTANT: {
2754 MI.eraseFromParent();
2757 case TargetOpcode::G_IMPLICIT_DEF: {
2763 case TargetOpcode::G_BRCOND:
2769 case TargetOpcode::G_FCMP:
2780 case TargetOpcode::G_ICMP:
2786 MI.getOperand(1).getPredicate()))
2787 ? TargetOpcode::G_SEXT
2788 : TargetOpcode::G_ZEXT;
2795 case TargetOpcode::G_PTR_ADD:
2796 assert(TypeIdx == 1 &&
"unable to legalize pointer of G_PTR_ADD");
2802 case TargetOpcode::G_PHI: {
2803 assert(TypeIdx == 0 &&
"Expecting only Idx 0");
2806 for (
unsigned I = 1;
I <
MI.getNumOperands();
I += 2) {
2818 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2826 TargetOpcode::G_ANYEXT);
2841 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2877 case TargetOpcode::G_FADD:
2878 case TargetOpcode::G_FMUL:
2879 case TargetOpcode::G_FSUB:
2880 case TargetOpcode::G_FMA:
2881 case TargetOpcode::G_FMAD:
2882 case TargetOpcode::G_FNEG:
2883 case TargetOpcode::G_FABS:
2884 case TargetOpcode::G_FCANONICALIZE:
2885 case TargetOpcode::G_FMINNUM:
2886 case TargetOpcode::G_FMAXNUM:
2887 case TargetOpcode::G_FMINNUM_IEEE:
2888 case TargetOpcode::G_FMAXNUM_IEEE:
2889 case TargetOpcode::G_FMINIMUM:
2890 case TargetOpcode::G_FMAXIMUM:
2891 case TargetOpcode::G_FDIV:
2892 case TargetOpcode::G_FREM:
2893 case TargetOpcode::G_FCEIL:
2894 case TargetOpcode::G_FFLOOR:
2895 case TargetOpcode::G_FCOS:
2896 case TargetOpcode::G_FSIN:
2897 case TargetOpcode::G_FTAN:
2898 case TargetOpcode::G_FLOG10:
2899 case TargetOpcode::G_FLOG:
2900 case TargetOpcode::G_FLOG2:
2901 case TargetOpcode::G_FRINT:
2902 case TargetOpcode::G_FNEARBYINT:
2903 case TargetOpcode::G_FSQRT:
2904 case TargetOpcode::G_FEXP:
2905 case TargetOpcode::G_FEXP2:
2906 case TargetOpcode::G_FEXP10:
2907 case TargetOpcode::G_FPOW:
2908 case TargetOpcode::G_INTRINSIC_TRUNC:
2909 case TargetOpcode::G_INTRINSIC_ROUND:
2910 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2914 for (
unsigned I = 1, E =
MI.getNumOperands();
I != E; ++
I)
2920 case TargetOpcode::G_FPOWI:
2921 case TargetOpcode::G_FLDEXP:
2922 case TargetOpcode::G_STRICT_FLDEXP: {
2924 if (
MI.getOpcode() == TargetOpcode::G_STRICT_FLDEXP)
2945 case TargetOpcode::G_FFREXP: {
2958 case TargetOpcode::G_INTTOPTR:
2966 case TargetOpcode::G_PTRTOINT:
2974 case TargetOpcode::G_BUILD_VECTOR: {
2978 for (
int I = 1, E =
MI.getNumOperands();
I != E; ++
I)
2992 case TargetOpcode::G_SEXT_INREG:
3001 case TargetOpcode::G_PTRMASK: {
3009 case TargetOpcode::G_VECREDUCE_FADD:
3010 case TargetOpcode::G_VECREDUCE_FMUL:
3011 case TargetOpcode::G_VECREDUCE_FMIN:
3012 case TargetOpcode::G_VECREDUCE_FMAX:
3013 case TargetOpcode::G_VECREDUCE_FMINIMUM:
3014 case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
3028 case TargetOpcode::G_VSCALE: {
3035 SrcMO.
setCImm(ConstantInt::get(Ctx, Val));
3040 case TargetOpcode::G_SPLAT_VECTOR: {
3054 auto Unmerge =
B.buildUnmerge(Ty, Src);
3055 for (
int I = 0, E = Unmerge->getNumOperands() - 1;
I != E; ++
I)
3064 unsigned AddrSpace =
DL.getDefaultGlobalsAddressSpace();
3066 LLT DstLLT =
MRI.getType(DstReg);
3087 MI.eraseFromParent();
3098 MI.eraseFromParent();
3105 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
3110 if (DstTy.isVector()) {
3111 int NumDstElt = DstTy.getNumElements();
3115 LLT DstCastTy = DstEltTy;
3116 LLT SrcPartTy = SrcEltTy;
3120 if (NumSrcElt < NumDstElt) {
3130 SrcPartTy = SrcEltTy;
3131 }
else if (NumSrcElt > NumDstElt) {
3142 DstCastTy = DstEltTy;
3152 MI.eraseFromParent();
3156 if (DstTy.isVector()) {
3160 MI.eraseFromParent();
3176 unsigned NewEltSize,
3177 unsigned OldEltSize) {
3178 const unsigned Log2EltRatio =
Log2_32(NewEltSize / OldEltSize);
3179 LLT IdxTy =
B.getMRI()->getType(
Idx);
3182 auto OffsetMask =
B.buildConstant(
3184 auto OffsetIdx =
B.buildAnd(IdxTy,
Idx, OffsetMask);
3185 return B.buildShl(IdxTy, OffsetIdx,
3186 B.buildConstant(IdxTy,
Log2_32(OldEltSize))).getReg(0);
3201 auto [Dst, DstTy, SrcVec, SrcVecTy,
Idx, IdxTy] =
MI.getFirst3RegLLTs();
3205 unsigned OldNumElts = SrcVecTy.getNumElements();
3212 if (NewNumElts > OldNumElts) {
3223 if (NewNumElts % OldNumElts != 0)
3227 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
3236 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
3240 NewOps[
I] = Elt.getReg(0);
3245 MI.eraseFromParent();
3249 if (NewNumElts < OldNumElts) {
3250 if (NewEltSize % OldEltSize != 0)
3272 const unsigned Log2EltRatio =
Log2_32(NewEltSize / OldEltSize);
3291 MI.eraseFromParent();
3305 LLT TargetTy =
B.getMRI()->getType(TargetReg);
3306 LLT InsertTy =
B.getMRI()->getType(InsertReg);
3307 auto ZextVal =
B.buildZExt(TargetTy, InsertReg);
3308 auto ShiftedInsertVal =
B.buildShl(TargetTy, ZextVal, OffsetBits);
3311 auto EltMask =
B.buildConstant(
3315 auto ShiftedMask =
B.buildShl(TargetTy, EltMask, OffsetBits);
3316 auto InvShiftedMask =
B.buildNot(TargetTy, ShiftedMask);
3319 auto MaskedOldElt =
B.buildAnd(TargetTy, TargetReg, InvShiftedMask);
3323 return B.buildOr(TargetTy, MaskedOldElt, ShiftedInsertVal).getReg(0);
3337 auto [Dst, DstTy, SrcVec, SrcVecTy, Val, ValTy,
Idx, IdxTy] =
3338 MI.getFirst4RegLLTs();
3350 if (NewNumElts < OldNumElts) {
3351 if (NewEltSize % OldEltSize != 0)
3360 const unsigned Log2EltRatio =
Log2_32(NewEltSize / OldEltSize);
3380 CastTy, CastVec, InsertedElt, ScaledIdx).
getReg(0);
3384 MI.eraseFromParent();
3403 if (MemSizeInBits != MemStoreSizeInBits) {
3423 if (isa<GSExtLoad>(LoadMI)) {
3426 }
else if (isa<GZExtLoad>(LoadMI) || WideMemTy == LoadTy) {
3435 if (DstTy != LoadTy)
3461 uint64_t LargeSplitSize, SmallSplitSize;
3466 SmallSplitSize = MemSizeInBits - LargeSplitSize;
3476 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
3501 LargeSplitSize / 8);
3505 SmallPtr, *SmallMMO);
3510 if (AnyExtTy == DstTy)
3545 if (StoreWidth != StoreSizeInBits) {
3580 uint64_t LargeSplitSize, SmallSplitSize;
3583 LargeSplitSize = llvm::bit_floor<uint64_t>(MemTy.
getSizeInBits());
3590 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
3629 switch (
MI.getOpcode()) {
3630 case TargetOpcode::G_LOAD: {
3645 case TargetOpcode::G_STORE: {
3661 case TargetOpcode::G_SELECT: {
3667 dbgs() <<
"bitcast action not implemented for vector select\n");
3678 case TargetOpcode::G_AND:
3679 case TargetOpcode::G_OR:
3680 case TargetOpcode::G_XOR: {
3688 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3690 case TargetOpcode::G_INSERT_VECTOR_ELT:
3698void LegalizerHelper::changeOpcode(
MachineInstr &
MI,
unsigned NewOpcode) {
3706 using namespace TargetOpcode;
3708 switch(
MI.getOpcode()) {
3711 case TargetOpcode::G_FCONSTANT:
3713 case TargetOpcode::G_BITCAST:
3715 case TargetOpcode::G_SREM:
3716 case TargetOpcode::G_UREM: {
3720 {MI.getOperand(1), MI.getOperand(2)});
3724 MI.eraseFromParent();
3727 case TargetOpcode::G_SADDO:
3728 case TargetOpcode::G_SSUBO:
3730 case TargetOpcode::G_UMULH:
3731 case TargetOpcode::G_SMULH:
3733 case TargetOpcode::G_SMULO:
3734 case TargetOpcode::G_UMULO: {
3737 auto [Res, Overflow,
LHS,
RHS] =
MI.getFirst4Regs();
3740 unsigned Opcode =
MI.getOpcode() == TargetOpcode::G_SMULO
3741 ? TargetOpcode::G_SMULH
3742 : TargetOpcode::G_UMULH;
3746 MI.setDesc(
TII.get(TargetOpcode::G_MUL));
3747 MI.removeOperand(1);
3758 if (Opcode == TargetOpcode::G_SMULH) {
3767 case TargetOpcode::G_FNEG: {
3768 auto [Res, SubByReg] =
MI.getFirst2Regs();
3778 MI.eraseFromParent();
3781 case TargetOpcode::G_FSUB:
3782 case TargetOpcode::G_STRICT_FSUB: {
3783 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
3789 if (
MI.getOpcode() == TargetOpcode::G_STRICT_FSUB)
3794 MI.eraseFromParent();
3797 case TargetOpcode::G_FMAD:
3799 case TargetOpcode::G_FFLOOR:
3801 case TargetOpcode::G_INTRINSIC_ROUND:
3803 case TargetOpcode::G_FRINT: {
3806 changeOpcode(
MI, TargetOpcode::G_INTRINSIC_ROUNDEVEN);
3809 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
3810 auto [OldValRes, SuccessRes,
Addr, CmpVal, NewVal] =
MI.getFirst5Regs();
3813 **
MI.memoperands_begin());
3816 MI.eraseFromParent();
3819 case TargetOpcode::G_LOAD:
3820 case TargetOpcode::G_SEXTLOAD:
3821 case TargetOpcode::G_ZEXTLOAD:
3823 case TargetOpcode::G_STORE:
3825 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
3826 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
3827 case TargetOpcode::G_CTLZ:
3828 case TargetOpcode::G_CTTZ:
3829 case TargetOpcode::G_CTPOP:
3832 auto [Res, CarryOut,
LHS,
RHS] =
MI.getFirst4Regs();
3841 MI.eraseFromParent();
3845 auto [Res, CarryOut,
LHS,
RHS, CarryIn] =
MI.getFirst5Regs();
3871 MI.eraseFromParent();
3875 auto [Res, BorrowOut,
LHS,
RHS] =
MI.getFirst4Regs();
3880 MI.eraseFromParent();
3884 auto [Res, BorrowOut,
LHS,
RHS, BorrowIn] =
MI.getFirst5Regs();
3906 MI.eraseFromParent();
3931 case G_MERGE_VALUES:
3933 case G_UNMERGE_VALUES:
3935 case TargetOpcode::G_SEXT_INREG: {
3936 assert(
MI.getOperand(2).isImm() &&
"Expected immediate");
3937 int64_t SizeInBits =
MI.getOperand(2).getImm();
3939 auto [DstReg, SrcReg] =
MI.getFirst2Regs();
3946 MI.eraseFromParent();
3949 case G_EXTRACT_VECTOR_ELT:
3950 case G_INSERT_VECTOR_ELT:
3952 case G_SHUFFLE_VECTOR:
3954 case G_DYN_STACKALLOC:
3958 case G_STACKRESTORE:
3968 case G_READ_REGISTER:
3969 case G_WRITE_REGISTER:
4014 case G_MEMCPY_INLINE:
4015 return lowerMemcpyInline(
MI);
4046 unsigned AddrSpace =
DL.getAllocaAddrSpace();
4055 LLT IdxTy =
B.getMRI()->getType(IdxReg);
4067 return B.buildAnd(IdxTy, IdxReg,
B.buildConstant(IdxTy, Imm)).getReg(0);
4070 return B.buildUMin(IdxTy, IdxReg,
B.buildConstant(IdxTy, NElts - 1))
4081 "Converting bits to bytes lost precision");
4088 unsigned IndexSizeInBits =
DL.getIndexSize(AS) * 8;
4105 std::initializer_list<unsigned> NonVecOpIndices) {
4106 if (
MI.getNumMemOperands() != 0)
4109 LLT VecTy =
MRI.getType(
MI.getReg(0));
4114 for (
unsigned OpIdx = 1; OpIdx <
MI.getNumOperands(); ++OpIdx) {
4147 int NumParts, NumLeftover;
4148 std::tie(NumParts, NumLeftover) =
4151 assert(NumParts > 0 &&
"Error in getNarrowTypeBreakDown");
4152 for (
int i = 0; i < NumParts; ++i) {
4157 assert(NumLeftover == 1 &&
"expected exactly one leftover");
4166 for (
unsigned i = 0; i <
N; ++i) {
4169 else if (
Op.isImm())
4171 else if (
Op.isPredicate())
4193 std::initializer_list<unsigned> NonVecOpIndices) {
4195 "Non-compatible opcode or not specified non-vector operands");
4198 unsigned NumInputs =
MI.getNumOperands() -
MI.getNumDefs();
4199 unsigned NumDefs =
MI.getNumDefs();
4207 for (
unsigned i = 0; i < NumDefs; ++i) {
4216 for (
unsigned UseIdx = NumDefs, UseNo = 0; UseIdx <
MI.getNumOperands();
4217 ++UseIdx, ++UseNo) {
4220 MI.getOperand(UseIdx));
4225 for (
auto Reg : SplitPieces)
4230 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
4234 for (
unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
4236 for (
unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
4237 Defs.
push_back(OutputOpsPieces[DstNo][i]);
4240 for (
unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
4241 Uses.push_back(InputOpsPieces[InputNo][i]);
4244 for (
unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
4245 OutputRegs[DstNo].push_back(
I.getReg(DstNo));
4250 for (
unsigned i = 0; i < NumDefs; ++i)
4251 mergeMixedSubvectors(
MI.getReg(i), OutputRegs[i]);
4253 for (
unsigned i = 0; i < NumDefs; ++i)
4257 MI.eraseFromParent();
4266 unsigned NumInputs =
MI.getNumOperands() -
MI.getNumDefs();
4267 unsigned NumDefs =
MI.getNumDefs();
4276 for (
unsigned UseIdx = NumDefs, UseNo = 0; UseIdx <
MI.getNumOperands();
4277 UseIdx += 2, ++UseNo) {
4285 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
4287 for (
unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
4293 for (
unsigned j = 0; j < NumInputs / 2; ++j) {
4294 Phi.addUse(InputOpsPieces[j][i]);
4295 Phi.add(
MI.getOperand(1 + j * 2 + 1));
4305 mergeMixedSubvectors(
MI.getReg(0), OutputRegs);
4310 MI.eraseFromParent();
4318 const int NumDst =
MI.getNumOperands() - 1;
4319 const Register SrcReg =
MI.getOperand(NumDst).getReg();
4323 if (TypeIdx != 1 || NarrowTy == DstTy)
4349 const int PartsPerUnmerge = NumDst / NumUnmerge;
4351 for (
int I = 0;
I != NumUnmerge; ++
I) {
4354 for (
int J = 0; J != PartsPerUnmerge; ++J)
4355 MIB.
addDef(
MI.getOperand(
I * PartsPerUnmerge + J).getReg());
4356 MIB.
addUse(Unmerge.getReg(
I));
4359 MI.eraseFromParent();
4366 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
4370 assert(DstTy.isVector() && NarrowTy.
isVector() &&
"Expected vector types");
4372 if (NarrowTy == SrcTy)
4382 if ((DstTy.getSizeInBits() % NarrowTy.
getSizeInBits() != 0) ||
4396 for (
unsigned i = 1; i <
MI.getNumOperands(); ++i) {
4398 for (
unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
4404 unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
4405 for (
unsigned i = 0,
Offset = 0; i < NumNarrowTyPieces;
4406 ++i,
Offset += NumNarrowTyElts) {
4413 MI.eraseFromParent();
4417 assert(TypeIdx == 0 &&
"Bad type index");
4433 unsigned NumParts = DstTy.getNumElements() / NarrowTy.
getNumElements();
4436 for (
unsigned i = 0; i < NumParts; ++i) {
4438 for (
unsigned j = 0; j < NumElts; ++j)
4439 Sources.
push_back(
MI.getOperand(1 + i * NumElts + j).getReg());
4445 MI.eraseFromParent();
4453 auto [DstReg, SrcVec] =
MI.getFirst2Regs();
4455 bool IsInsert =
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
4457 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) &&
"not a vector type index");
4459 InsertVal =
MI.getOperand(2).getReg();
4474 IdxVal = MaybeCst->Value.getSExtValue();
4478 MI.eraseFromParent();
4483 LLT GCDTy = extractGCDType(VecParts, VecTy, NarrowVecTy, SrcVec);
4486 LLT LCMTy = buildLCMMergePieces(VecTy, NarrowVecTy, GCDTy, VecParts,
4487 TargetOpcode::G_ANYEXT);
4492 int64_t PartIdx = IdxVal / NewNumElts;
4501 PartTy, VecParts[PartIdx], InsertVal, NewIdx);
4502 VecParts[PartIdx] = InsertPart.getReg(0);
4506 buildWidenedRemergeToDst(DstReg, LCMTy, VecParts);
4511 MI.eraseFromParent();
4535 bool IsLoad = isa<GLoad>(LdStMI);
4547 int NumLeftover = -1;
4553 if (
extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
4555 NumParts = NarrowRegs.
size();
4556 NumLeftover = NarrowLeftoverRegs.
size();
4573 auto MMO = LdStMI.
getMMO();
4575 unsigned NumParts,
unsigned Offset) ->
unsigned {
4578 for (
unsigned Idx = 0, E = NumParts;
Idx != E &&
Offset < TotalSize;
4580 unsigned ByteOffset =
Offset / 8;
4590 ValRegs.push_back(Dst);
4602 unsigned HandledOffset =
4603 splitTypePieces(NarrowTy, NarrowRegs, NumParts,
Offset);
4607 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset);
4610 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
4611 LeftoverTy, NarrowLeftoverRegs);
4621 using namespace TargetOpcode;
4625 switch (
MI.getOpcode()) {
4626 case G_IMPLICIT_DEF:
4642 case G_FCANONICALIZE:
4659 case G_INTRINSIC_ROUND:
4660 case G_INTRINSIC_ROUNDEVEN:
4661 case G_INTRINSIC_TRUNC:
4681 case G_FMINNUM_IEEE:
4682 case G_FMAXNUM_IEEE:
4702 case G_CTLZ_ZERO_UNDEF:
4704 case G_CTTZ_ZERO_UNDEF:
4718 case G_ADDRSPACE_CAST:
4731 case G_STRICT_FLDEXP:
4745 case G_UNMERGE_VALUES:
4747 case G_BUILD_VECTOR:
4748 assert(TypeIdx == 0 &&
"not a vector type index");
4750 case G_CONCAT_VECTORS:
4754 case G_EXTRACT_VECTOR_ELT:
4755 case G_INSERT_VECTOR_ELT:
4764 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
4765 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
4767 case G_SHUFFLE_VECTOR:
4773 case G_INTRINSIC_FPTRUNC_ROUND:
4783 assert(
MI.getOpcode() == TargetOpcode::G_BITCAST &&
4784 "Not a bitcast operation");
4789 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
4797 if (extractGCDType(SrcVRegs, DstTy, SrcNarrowTy, SrcReg) != SrcNarrowTy)
4802 for (
unsigned i = 0; i < SrcVRegs.
size(); i++)
4807 MI.eraseFromParent();
4813 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
4817 auto [DstReg, DstTy, Src1Reg, Src1Ty, Src2Reg, Src2Ty] =
4818 MI.getFirst3RegLLTs();
4821 if (DstTy != Src1Ty)
4823 if (DstTy != Src2Ty)
4838 Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
4854 unsigned InputUsed[2] = {-1U, -1U};
4855 unsigned FirstMaskIdx =
High * NewElts;
4856 bool UseBuildVector =
false;
4857 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4859 int Idx = Mask[FirstMaskIdx + MaskOffset];
4864 if (Input >= std::size(Inputs)) {
4871 Idx -= Input * NewElts;
4875 for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) {
4876 if (InputUsed[OpNo] == Input) {
4879 }
else if (InputUsed[OpNo] == -1U) {
4881 InputUsed[OpNo] = Input;
4886 if (OpNo >= std::size(InputUsed)) {
4889 UseBuildVector =
true;
4897 if (UseBuildVector) {
4902 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4904 int Idx = Mask[FirstMaskIdx + MaskOffset];
4909 if (Input >= std::size(Inputs)) {
4916 Idx -= Input * NewElts;
4920 .buildExtractVectorElement(
4921 EltTy, Inputs[Input],
4928 }
else if (InputUsed[0] == -1U) {
4932 Register Op0 = Inputs[InputUsed[0]];
4936 : Inputs[InputUsed[1]];
4945 MI.eraseFromParent();
4951 auto &RdxMI = cast<GVecReduce>(
MI);
4958 auto [DstReg, DstTy, SrcReg, SrcTy] = RdxMI.getFirst2RegLLTs();
4964 unsigned ScalarOpc = RdxMI.getScalarOpcForReduction();
4967 const unsigned NumParts =
4973 if (DstTy != NarrowTy)
4979 unsigned NumPartsLeft = NumParts;
4980 while (NumPartsLeft > 1) {
4981 for (
unsigned Idx = 0;
Idx < NumPartsLeft - 1;
Idx += 2) {
4984 .buildInstr(ScalarOpc, {NarrowTy},
4985 {SplitSrcs[
Idx], SplitSrcs[
Idx + 1]})
4988 SplitSrcs = PartialResults;
4989 PartialResults.
clear();
4990 NumPartsLeft = SplitSrcs.
size();
4994 MI.eraseFromParent();
4999 for (
unsigned Idx = 1;
Idx < NumParts; ++
Idx)
5003 MI.eraseFromParent();
5007 for (
unsigned Part = 0; Part < NumParts; ++Part) {
5017 return tryNarrowPow2Reduction(
MI, SrcReg, SrcTy, NarrowTy, ScalarOpc);
5020 Register Acc = PartialReductions[0];
5021 for (
unsigned Part = 1; Part < NumParts; ++Part) {
5022 if (Part == NumParts - 1) {
5024 {Acc, PartialReductions[Part]});
5027 .
buildInstr(ScalarOpc, {DstTy}, {Acc, PartialReductions[Part]})
5031 MI.eraseFromParent();
5037 unsigned int TypeIdx,
5039 auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] =
5040 MI.getFirst3RegLLTs();
5041 if (!NarrowTy.
isScalar() || TypeIdx != 2 || DstTy != ScalarTy ||
5045 assert((
MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD ||
5046 MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) &&
5047 "Unexpected vecreduce opcode");
5048 unsigned ScalarOpc =
MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD
5049 ? TargetOpcode::G_FADD
5050 : TargetOpcode::G_FMUL;
5056 for (
unsigned i = 0; i < NumParts; i++)
5061 MI.eraseFromParent();
5068 unsigned ScalarOpc) {
5076 while (SplitSrcs.
size() > 1) {
5078 for (
unsigned Idx = 0;
Idx < SplitSrcs.
size()-1;
Idx += 2) {
5086 SplitSrcs = std::move(PartialRdxs);
5090 MI.getOperand(1).setReg(SplitSrcs[0]);
5097 const LLT HalfTy,
const LLT AmtTy) {
5105 MI.eraseFromParent();
5111 unsigned VTBits = 2 * NVTBits;
5114 if (
MI.getOpcode() == TargetOpcode::G_SHL) {
5115 if (Amt.
ugt(VTBits)) {
5117 }
else if (Amt.
ugt(NVTBits)) {
5121 }
else if (Amt == NVTBits) {
5132 }
else if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
5133 if (Amt.
ugt(VTBits)) {
5135 }
else if (Amt.
ugt(NVTBits)) {
5139 }
else if (Amt == NVTBits) {
5153 if (Amt.
ugt(VTBits)) {
5156 }
else if (Amt.
ugt(NVTBits)) {
5161 }
else if (Amt == NVTBits) {
5178 MI.eraseFromParent();
5202 if (DstEltSize % 2 != 0)
5208 const unsigned NewBitSize = DstEltSize / 2;
5234 switch (
MI.getOpcode()) {
5235 case TargetOpcode::G_SHL: {
5251 ResultRegs[0] =
Lo.getReg(0);
5252 ResultRegs[1] =
Hi.getReg(0);
5255 case TargetOpcode::G_LSHR:
5256 case TargetOpcode::G_ASHR: {
5266 if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
5280 ResultRegs[0] =
Lo.getReg(0);
5281 ResultRegs[1] =
Hi.getReg(0);
5289 MI.eraseFromParent();
5296 assert(TypeIdx == 0 &&
"Expecting only Idx 0");
5299 for (
unsigned I = 1, E =
MI.getNumOperands();
I != E;
I += 2) {
5314 assert(Ty.
isScalar() &&
"Expected scalar type to make neutral element for");
5319 "getNeutralElementForVecReduce called with invalid opcode!");
5320 case TargetOpcode::G_VECREDUCE_ADD:
5321 case TargetOpcode::G_VECREDUCE_OR:
5322 case TargetOpcode::G_VECREDUCE_XOR:
5323 case TargetOpcode::G_VECREDUCE_UMAX:
5325 case TargetOpcode::G_VECREDUCE_MUL:
5327 case TargetOpcode::G_VECREDUCE_AND:
5328 case TargetOpcode::G_VECREDUCE_UMIN:
5331 case TargetOpcode::G_VECREDUCE_SMAX:
5334 case TargetOpcode::G_VECREDUCE_SMIN:
5337 case TargetOpcode::G_VECREDUCE_FADD:
5339 case TargetOpcode::G_VECREDUCE_FMUL:
5341 case TargetOpcode::G_VECREDUCE_FMINIMUM:
5342 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
5343 assert(
false &&
"getNeutralElementForVecReduce unimplemented for "
5344 "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
5352 unsigned Opc =
MI.getOpcode();
5354 case TargetOpcode::G_IMPLICIT_DEF:
5355 case TargetOpcode::G_LOAD: {
5363 case TargetOpcode::G_STORE:
5370 case TargetOpcode::G_AND:
5371 case TargetOpcode::G_OR:
5372 case TargetOpcode::G_XOR:
5373 case TargetOpcode::G_ADD:
5374 case TargetOpcode::G_SUB:
5375 case TargetOpcode::G_MUL:
5376 case TargetOpcode::G_FADD:
5377 case TargetOpcode::G_FSUB:
5378 case TargetOpcode::G_FMUL:
5379 case TargetOpcode::G_FDIV:
5380 case TargetOpcode::G_FCOPYSIGN:
5381 case TargetOpcode::G_UADDSAT:
5382 case TargetOpcode::G_USUBSAT:
5383 case TargetOpcode::G_SADDSAT:
5384 case TargetOpcode::G_SSUBSAT:
5385 case TargetOpcode::G_SMIN:
5386 case TargetOpcode::G_SMAX:
5387 case TargetOpcode::G_UMIN:
5388 case TargetOpcode::G_UMAX:
5389 case TargetOpcode::G_FMINNUM:
5390 case TargetOpcode::G_FMAXNUM:
5391 case TargetOpcode::G_FMINNUM_IEEE:
5392 case TargetOpcode::G_FMAXNUM_IEEE:
5393 case TargetOpcode::G_FMINIMUM:
5394 case TargetOpcode::G_FMAXIMUM:
5395 case TargetOpcode::G_STRICT_FADD:
5396 case TargetOpcode::G_STRICT_FSUB:
5397 case TargetOpcode::G_STRICT_FMUL:
5398 case TargetOpcode::G_SHL:
5399 case TargetOpcode::G_ASHR:
5400 case TargetOpcode::G_LSHR: {
5408 case TargetOpcode::G_FMA:
5409 case TargetOpcode::G_STRICT_FMA:
5410 case TargetOpcode::G_FSHR:
5411 case TargetOpcode::G_FSHL: {
5420 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
5421 case TargetOpcode::G_EXTRACT:
5428 case TargetOpcode::G_INSERT:
5429 case TargetOpcode::G_INSERT_VECTOR_ELT:
5430 case TargetOpcode::G_FREEZE:
5431 case TargetOpcode::G_FNEG:
5432 case TargetOpcode::G_FABS:
5433 case TargetOpcode::G_FSQRT:
5434 case TargetOpcode::G_FCEIL:
5435 case TargetOpcode::G_FFLOOR:
5436 case TargetOpcode::G_FNEARBYINT:
5437 case TargetOpcode::G_FRINT:
5438 case TargetOpcode::G_INTRINSIC_ROUND:
5439 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
5440 case TargetOpcode::G_INTRINSIC_TRUNC:
5441 case TargetOpcode::G_BSWAP:
5442 case TargetOpcode::G_FCANONICALIZE:
5443 case TargetOpcode::G_SEXT_INREG:
5444 case TargetOpcode::G_ABS:
5452 case TargetOpcode::G_SELECT: {
5453 auto [DstReg, DstTy, CondReg, CondTy] =
MI.getFirst2RegLLTs();
5455 if (!CondTy.isScalar() ||
5463 MI.getOperand(1).setReg(ShufSplat.getReg(0));
5468 if (CondTy.isVector())
5478 case TargetOpcode::G_UNMERGE_VALUES:
5480 case TargetOpcode::G_PHI:
5482 case TargetOpcode::G_SHUFFLE_VECTOR:
5484 case TargetOpcode::G_BUILD_VECTOR: {
5486 for (
auto Op :
MI.uses()) {
5496 MI.eraseFromParent();
5499 case TargetOpcode::G_SEXT:
5500 case TargetOpcode::G_ZEXT:
5501 case TargetOpcode::G_ANYEXT:
5502 case TargetOpcode::G_TRUNC:
5503 case TargetOpcode::G_FPTRUNC:
5504 case TargetOpcode::G_FPEXT:
5505 case TargetOpcode::G_FPTOSI:
5506 case TargetOpcode::G_FPTOUI:
5507 case TargetOpcode::G_SITOFP:
5508 case TargetOpcode::G_UITOFP: {
5528 case TargetOpcode::G_ICMP:
5529 case TargetOpcode::G_FCMP: {
5543 case TargetOpcode::G_BITCAST: {
5564 case TargetOpcode::G_VECREDUCE_FADD:
5565 case TargetOpcode::G_VECREDUCE_FMUL:
5566 case TargetOpcode::G_VECREDUCE_ADD:
5567 case TargetOpcode::G_VECREDUCE_MUL:
5568 case TargetOpcode::G_VECREDUCE_AND:
5569 case TargetOpcode::G_VECREDUCE_OR:
5570 case TargetOpcode::G_VECREDUCE_XOR:
5571 case TargetOpcode::G_VECREDUCE_SMAX:
5572 case TargetOpcode::G_VECREDUCE_SMIN:
5573 case TargetOpcode::G_VECREDUCE_UMAX:
5574 case TargetOpcode::G_VECREDUCE_UMIN: {
5578 auto NeutralElement = getNeutralElementForVecReduce(
5586 NeutralElement,
Idx);
5590 MO.
setReg(NewVec.getReg(0));
5602 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
5604 unsigned MaskNumElts = Mask.size();
5608 if (MaskNumElts == SrcNumElts)
5611 if (MaskNumElts < SrcNumElts) {
5615 for (
unsigned I = MaskNumElts;
I < SrcNumElts; ++
I)
5621 MI.getOperand(1).getReg(),
5622 MI.getOperand(2).getReg(), NewMask);
5623 MI.eraseFromParent();
5628 unsigned PaddedMaskNumElts =
alignTo(MaskNumElts, SrcNumElts);
5629 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
5637 MOps1[0] =
MI.getOperand(1).getReg();
5638 MOps2[0] =
MI.getOperand(2).getReg();
5645 for (
unsigned I = 0;
I != MaskNumElts; ++
I) {
5647 if (
Idx >=
static_cast<int>(SrcNumElts))
5648 Idx += PaddedMaskNumElts - SrcNumElts;
5653 if (MaskNumElts != PaddedMaskNumElts) {
5658 for (
unsigned I = 0;
I < MaskNumElts; ++
I) {
5668 MI.eraseFromParent();
5674 unsigned int TypeIdx,
LLT MoreTy) {
5675 auto [DstTy, Src1Ty, Src2Ty] =
MI.getFirst3LLTs();
5677 unsigned NumElts = DstTy.getNumElements();
5680 if (DstTy.isVector() && Src1Ty.isVector() &&
5681 DstTy.getNumElements() != Src1Ty.getNumElements()) {
5689 if (DstTy != Src1Ty || DstTy != Src2Ty)
5697 for (
unsigned I = 0;
I != NumElts; ++
I) {
5699 if (
Idx <
static_cast<int>(NumElts))
5704 for (
unsigned I = NumElts;
I != WidenNumElts; ++
I)
5709 MI.getOperand(1).getReg(),
5710 MI.getOperand(2).getReg(), NewMask);
5711 MI.eraseFromParent();
5720 unsigned SrcParts = Src1Regs.
size();
5721 unsigned DstParts = DstRegs.
size();
5723 unsigned DstIdx = 0;
5725 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
5726 DstRegs[DstIdx] = FactorSum;
5728 unsigned CarrySumPrevDstIdx;
5731 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
5733 for (
unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
5734 i <= std::min(DstIdx, SrcParts - 1); ++i) {
5736 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
5740 for (
unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
5741 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
5743 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
5753 if (DstIdx != DstParts - 1) {
5755 B.buildUAddo(NarrowTy,
LLT::scalar(1), Factors[0], Factors[1]);
5756 FactorSum = Uaddo.
getReg(0);
5757 CarrySum =
B.buildZExt(NarrowTy, Uaddo.
getReg(1)).
getReg(0);
5758 for (
unsigned i = 2; i < Factors.
size(); ++i) {
5760 B.buildUAddo(NarrowTy,
LLT::scalar(1), FactorSum, Factors[i]);
5761 FactorSum = Uaddo.
getReg(0);
5763 CarrySum =
B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
5767 FactorSum =
B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
5768 for (
unsigned i = 2; i < Factors.
size(); ++i)
5769 FactorSum =
B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
5772 CarrySumPrevDstIdx = CarrySum;
5773 DstRegs[DstIdx] = FactorSum;
5790 unsigned Opcode =
MI.getOpcode();
5791 unsigned OpO, OpE, OpF;
5793 case TargetOpcode::G_SADDO:
5794 case TargetOpcode::G_SADDE:
5795 case TargetOpcode::G_UADDO:
5796 case TargetOpcode::G_UADDE:
5797 case TargetOpcode::G_ADD:
5798 OpO = TargetOpcode::G_UADDO;
5799 OpE = TargetOpcode::G_UADDE;
5800 OpF = TargetOpcode::G_UADDE;
5801 if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE)
5802 OpF = TargetOpcode::G_SADDE;
5804 case TargetOpcode::G_SSUBO:
5805 case TargetOpcode::G_SSUBE:
5806 case TargetOpcode::G_USUBO:
5807 case TargetOpcode::G_USUBE:
5808 case TargetOpcode::G_SUB:
5809 OpO = TargetOpcode::G_USUBO;
5810 OpE = TargetOpcode::G_USUBE;
5811 OpF = TargetOpcode::G_USUBE;
5812 if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE)
5813 OpF = TargetOpcode::G_SSUBE;
5820 unsigned NumDefs =
MI.getNumExplicitDefs();
5821 Register Src1 =
MI.getOperand(NumDefs).getReg();
5822 Register Src2 =
MI.getOperand(NumDefs + 1).getReg();
5825 CarryDst =
MI.getOperand(1).getReg();
5826 if (
MI.getNumOperands() == NumDefs + 3)
5827 CarryIn =
MI.getOperand(NumDefs + 2).getReg();
5830 LLT LeftoverTy, DummyTy;
5832 extractParts(Src1, RegTy, NarrowTy, LeftoverTy, Src1Regs, Src1Left,
5837 int NarrowParts = Src1Regs.
size();
5838 for (
int I = 0, E = Src1Left.
size();
I != E; ++
I) {
5844 for (
int i = 0, e = Src1Regs.
size(); i != e; ++i) {
5849 if (i == e - 1 && CarryDst)
5850 CarryOut = CarryDst;
5854 {Src1Regs[i], Src2Regs[i]});
5855 }
else if (i == e - 1) {
5857 {Src1Regs[i], Src2Regs[i], CarryIn});
5860 {Src1Regs[i], Src2Regs[i], CarryIn});
5866 insertParts(
MI.getOperand(0).getReg(), RegTy, NarrowTy,
5867 ArrayRef(DstRegs).take_front(NarrowParts), LeftoverTy,
5868 ArrayRef(DstRegs).drop_front(NarrowParts));
5870 MI.eraseFromParent();
5876 auto [DstReg, Src1, Src2] =
MI.getFirst3Regs();
5884 if (
Size % NarrowSize != 0)
5887 unsigned NumParts =
Size / NarrowSize;
5888 bool IsMulHigh =
MI.getOpcode() == TargetOpcode::G_UMULH;
5889 unsigned DstTmpParts = NumParts * (IsMulHigh ? 2 : 1);
5895 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
5900 MI.eraseFromParent();
5910 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_FPTOSI;
5924 IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT);
5940 if (SizeOp1 % NarrowSize != 0)
5942 int NumParts = SizeOp1 / NarrowSize;
5946 extractParts(
MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs,
5950 uint64_t OpStart =
MI.getOperand(2).getImm();
5952 for (
int i = 0; i < NumParts; ++i) {
5953 unsigned SrcStart = i * NarrowSize;
5955 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
5958 }
else if (SrcStart == OpStart && NarrowTy == MRI.
getType(OpReg)) {
5966 int64_t ExtractOffset;
5968 if (OpStart < SrcStart) {
5970 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
5972 ExtractOffset = OpStart - SrcStart;
5973 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
5977 if (ExtractOffset != 0 || SegSize != NarrowSize) {
5989 else if (DstRegs.
size() > 1)
5993 MI.eraseFromParent();
6008 extractParts(
MI.getOperand(1).getReg(), RegTy, NarrowTy, LeftoverTy, SrcRegs,
6016 uint64_t OpStart =
MI.getOperand(3).getImm();
6018 for (
int I = 0, E = SrcRegs.
size();
I != E; ++
I) {
6019 unsigned DstStart =
I * NarrowSize;
6021 if (DstStart == OpStart && NarrowTy == MRI.
getType(OpReg)) {
6029 if (MRI.
getType(SrcRegs[
I]) == LeftoverTy) {
6035 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
6043 int64_t ExtractOffset, InsertOffset;
6045 if (OpStart < DstStart) {
6047 ExtractOffset = DstStart - OpStart;
6048 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
6050 InsertOffset = OpStart - DstStart;
6053 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
6057 if (ExtractOffset != 0 || SegSize != OpSize) {
6077 MI.eraseFromParent();
6087 assert(
MI.getNumOperands() == 3 && TypeIdx == 0);
6093 if (!
extractParts(
MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
6094 Src0Regs, Src0LeftoverRegs,
MIRBuilder, MRI))
6098 if (!
extractParts(
MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
6099 Src1Regs, Src1LeftoverRegs,
MIRBuilder, MRI))
6102 for (
unsigned I = 0, E = Src1Regs.
size();
I != E; ++
I) {
6104 {Src0Regs[I], Src1Regs[I]});
6108 for (
unsigned I = 0, E = Src1LeftoverRegs.
size();
I != E; ++
I) {
6111 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
6112 DstLeftoverRegs.
push_back(Inst.getReg(0));
6115 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
6116 LeftoverTy, DstLeftoverRegs);
6118 MI.eraseFromParent();
6128 auto [DstReg, SrcReg] =
MI.getFirst2Regs();
6135 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
6136 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts,
MI.getOpcode());
6137 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
6139 MI.eraseFromParent();
6149 Register CondReg =
MI.getOperand(1).getReg();
6161 if (!
extractParts(
MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
6162 Src1Regs, Src1LeftoverRegs,
MIRBuilder, MRI))
6166 if (!
extractParts(
MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
6167 Src2Regs, Src2LeftoverRegs,
MIRBuilder, MRI))
6170 for (
unsigned I = 0, E = Src1Regs.
size();
I != E; ++
I) {
6172 CondReg, Src1Regs[
I], Src2Regs[
I]);
6176 for (
unsigned I = 0, E = Src1LeftoverRegs.
size();
I != E; ++
I) {
6178 LeftoverTy, CondReg, Src1LeftoverRegs[
I], Src2LeftoverRegs[
I]);
6182 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
6183 LeftoverTy, DstLeftoverRegs);
6185 MI.eraseFromParent();
6195 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6199 const bool IsUndef =
MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF;
6202 auto UnmergeSrc =
B.buildUnmerge(NarrowTy, SrcReg);
6204 auto C_0 =
B.buildConstant(NarrowTy, 0);
6206 UnmergeSrc.getReg(1), C_0);
6207 auto LoCTLZ = IsUndef ?
6208 B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) :
6209 B.buildCTLZ(DstTy, UnmergeSrc.getReg(0));
6210 auto C_NarrowSize =
B.buildConstant(DstTy, NarrowSize);
6211 auto HiIsZeroCTLZ =
B.buildAdd(DstTy, LoCTLZ, C_NarrowSize);
6212 auto HiCTLZ =
B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1));
6213 B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ);
6215 MI.eraseFromParent();
6228 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6232 const bool IsUndef =
MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF;
6235 auto UnmergeSrc =
B.buildUnmerge(NarrowTy, SrcReg);
6237 auto C_0 =
B.buildConstant(NarrowTy, 0);
6239 UnmergeSrc.getReg(0), C_0);
6240 auto HiCTTZ = IsUndef ?
6241 B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) :
6242 B.buildCTTZ(DstTy, UnmergeSrc.getReg(1));
6243 auto C_NarrowSize =
B.buildConstant(DstTy, NarrowSize);
6244 auto LoIsZeroCTTZ =
B.buildAdd(DstTy, HiCTTZ, C_NarrowSize);
6245 auto LoCTTZ =
B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0));
6246 B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ);
6248 MI.eraseFromParent();
6261 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6271 MI.eraseFromParent();
6291 auto MinExp =
B.buildConstant(ExpTy,
minIntN(ClampSize));
6292 auto ClampMin =
B.buildSMax(ExpTy, ExpReg, MinExp);
6293 auto MaxExp =
B.buildConstant(ExpTy,
maxIntN(ClampSize));
6294 auto Clamp =
B.buildSMin(ExpTy, ClampMin, MaxExp);
6296 auto Trunc =
B.buildTrunc(NarrowTy, Clamp);
6298 MI.getOperand(2).setReg(Trunc.getReg(0));
6305 unsigned Opc =
MI.getOpcode();
6314 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
6317 MI.setDesc(
TII.get(TargetOpcode::G_CTLZ));
6321 case TargetOpcode::G_CTLZ: {
6322 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6325 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
6333 MI.eraseFromParent();
6349 for (
unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
6353 Op = MIBOp.getReg(0);
6358 MI.eraseFromParent();
6361 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
6364 MI.setDesc(
TII.get(TargetOpcode::G_CTTZ));
6368 case TargetOpcode::G_CTTZ: {
6369 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6372 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
6381 MI.eraseFromParent();
6392 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
6393 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
6397 MI.eraseFromParent();
6401 MI.setDesc(
TII.get(TargetOpcode::G_CTPOP));
6402 MI.getOperand(1).setReg(MIBTmp.getReg(0));
6406 case TargetOpcode::G_CTPOP: {
6417 auto C_1 =
B.buildConstant(Ty, 1);
6418 auto B2Set1LoTo1Hi =
B.buildLShr(Ty, SrcReg, C_1);
6420 auto C_B2Mask1HiTo0 =
B.buildConstant(Ty, B2Mask1HiTo0);
6421 auto B2Count1Hi =
B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0);
6422 auto B2Count =
B.buildSub(Ty, SrcReg, B2Count1Hi);
6426 auto C_2 =
B.buildConstant(Ty, 2);
6427 auto B4Set2LoTo2Hi =
B.buildLShr(Ty, B2Count, C_2);
6429 auto C_B4Mask2HiTo0 =
B.buildConstant(Ty, B4Mask2HiTo0);
6430 auto B4HiB2Count =
B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0);
6431 auto B4LoB2Count =
B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0);
6432 auto B4Count =
B.buildAdd(Ty, B4HiB2Count, B4LoB2Count);
6439 auto C_4 =
B.buildConstant(Ty, 4);
6440 auto B8HiB4Count =
B.buildLShr(Ty, B4Count, C_4);
6441 auto B8CountDirty4Hi =
B.buildAdd(Ty, B8HiB4Count, B4Count);
6443 auto C_B8Mask4HiTo0 =
B.buildConstant(Ty, B8Mask4HiTo0);
6444 auto B8Count =
B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0);
6446 assert(
Size<=128 &&
"Scalar size is too large for CTPOP lower algorithm");
6452 auto C_SizeM8 =
B.buildConstant(Ty,
Size - 8);
6454 auto IsMulSupported = [
this](
const LLT Ty) {
6455 auto Action = LI.
getAction({TargetOpcode::G_MUL, {Ty}}).Action;
6458 if (IsMulSupported(Ty)) {
6459 auto ResTmp =
B.buildMul(Ty, B8Count, MulMask);
6460 B.buildLShr(
MI.getOperand(0).getReg(), ResTmp, C_SizeM8);
6462 auto ResTmp = B8Count;
6463 for (
unsigned Shift = 8; Shift <
Size; Shift *= 2) {
6464 auto ShiftC =
B.buildConstant(Ty, Shift);
6465 auto Shl =
B.buildShl(Ty, ResTmp, ShiftC);
6466 ResTmp =
B.buildAdd(Ty, ResTmp, Shl);
6468 B.buildLShr(
MI.getOperand(0).getReg(), ResTmp, C_SizeM8);
6470 MI.eraseFromParent();
6483 const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(
C);
6491 auto [Dst,
X,
Y, Z] =
MI.getFirst4Regs();
6500 const bool IsFSHL =
MI.getOpcode() == TargetOpcode::G_FSHL;
6501 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
6524 MI.eraseFromParent();
6530 auto [Dst,
X,
Y, Z] =
MI.getFirst4Regs();
6535 const bool IsFSHL =
MI.getOpcode() == TargetOpcode::G_FSHL;
6579 MI.eraseFromParent();
6593 bool IsFSHL =
MI.getOpcode() == TargetOpcode::G_FSHL;
6594 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
6598 return lowerFunnelShiftAsShifts(
MI);
6602 if (Result == UnableToLegalize)
6603 return lowerFunnelShiftAsShifts(
MI);
6608 auto [Dst, Src] =
MI.getFirst2Regs();
6622 if (SrcTyScalarSize * 2 < DstTyScalarSize) {
6636 {UnmergeSrc.getReg(0)});
6638 {UnmergeSrc.getReg(1)});
6643 MI.eraseFromParent();
6660 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC);
6664 LLT DstTy =
MRI.getType(DstReg);
6665 LLT SrcTy =
MRI.getType(SrcReg);
6685 for (
unsigned I = 0;
I < SplitSrcs.
size(); ++
I) {
6699 MI.eraseFromParent();
6708 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] =
MI.getFirst3RegLLTs();
6710 bool IsLeft =
MI.getOpcode() == TargetOpcode::G_ROTL;
6711 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
6714 MI.eraseFromParent();
6719 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] =
MI.getFirst3RegLLTs();
6721 unsigned EltSizeInBits = DstTy.getScalarSizeInBits();
6722 bool IsLeft =
MI.getOpcode() == TargetOpcode::G_ROTL;
6727 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
6730 return lowerRotateWithReverseRotate(
MI);
6733 unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
6734 unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
6735 bool IsFShLegal =
false;
6736 if ((IsFShLegal = LI.isLegalOrCustom({FShOpc, {DstTy, AmtTy}})) ||
6737 LI.isLegalOrCustom({RevFsh, {DstTy, AmtTy}})) {
6741 MI.eraseFromParent();
6746 return buildFunnelShift(FShOpc, Dst, Src, Amt);
6749 return buildFunnelShift(RevFsh, Dst, Src, Amt);
6754 unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR;
6755 unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL;
6756 auto BitWidthMinusOneC = MIRBuilder.
buildConstant(AmtTy, EltSizeInBits - 1);
6762 auto NegAmt = MIRBuilder.
buildSub(AmtTy, Zero, Amt);
6763 auto ShAmt = MIRBuilder.
buildAnd(AmtTy, Amt, BitWidthMinusOneC);
6765 auto RevAmt = MIRBuilder.
buildAnd(AmtTy, NegAmt, BitWidthMinusOneC);
6771 auto BitWidthC = MIRBuilder.
buildConstant(AmtTy, EltSizeInBits);
6772 auto ShAmt = MIRBuilder.
buildURem(AmtTy, Amt, BitWidthC);
6774 auto RevAmt = MIRBuilder.
buildSub(AmtTy, BitWidthMinusOneC, ShAmt);
6776 auto Inner = MIRBuilder.
buildInstr(RevShiftOpc, {DstTy}, {Src, One});
6780 MIRBuilder.
buildOr(Dst, ShVal, RevShiftVal);
6781 MI.eraseFromParent();
6789 auto [Dst, Src] =
MI.getFirst2Regs();
6839 MI.eraseFromParent();
6844 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6850 MI.eraseFromParent();
6869 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6879 MI.eraseFromParent();
6904 MI.eraseFromParent();
6912 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6916 if (SrcTy !=
S64 && SrcTy !=
S32)
6918 if (DstTy !=
S32 && DstTy !=
S64)
6947 MI.eraseFromParent();
6952 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
7011 MI.eraseFromParent();
7021 auto [Dst, Src] =
MI.getFirst2Regs();
7029 unsigned Flags =
MI.getFlags();
7032 MI.eraseFromParent();
7036 const unsigned ExpMask = 0x7ff;
7037 const unsigned ExpBiasf64 = 1023;
7038 const unsigned ExpBiasf16 = 15;
7127 MI.eraseFromParent();
7133 auto [DstTy, SrcTy] =
MI.getFirst2LLTs();
7144 auto [Dst, Src0, Src1] =
MI.getFirst3Regs();
7149 MI.eraseFromParent();
7155 case TargetOpcode::G_SMIN:
7157 case TargetOpcode::G_SMAX:
7159 case TargetOpcode::G_UMIN:
7161 case TargetOpcode::G_UMAX:
7169 auto [Dst, Src0, Src1] =
MI.getFirst3Regs();
7177 MI.eraseFromParent();
7183 auto [Dst, DstTy, Src0, Src0Ty, Src1, Src1Ty] =
MI.getFirst3RegLLTs();
7184 const int Src0Size = Src0Ty.getScalarSizeInBits();
7185 const int Src1Size = Src1Ty.getScalarSizeInBits();
7195 if (Src0Ty == Src1Ty) {
7197 }
else if (Src0Size > Src1Size) {
7212 unsigned Flags =
MI.getFlags();
7219 MI.eraseFromParent();
7225 unsigned NewOp =
MI.getOpcode() == TargetOpcode::G_FMINNUM ?
7226 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
7228 auto [Dst, Src0, Src1] =
MI.getFirst3Regs();
7248 MI.eraseFromParent();
7256 unsigned Flags =
MI.getFlags();
7261 MI.eraseFromParent();
7267 auto [DstReg,
X] =
MI.getFirst2Regs();
7268 const unsigned Flags =
MI.getFlags();
7295 MI.eraseFromParent();
7300 auto [DstReg, SrcReg] =
MI.getFirst2Regs();
7301 unsigned Flags =
MI.getFlags();
7313 SrcReg, Zero, Flags);
7315 SrcReg, Trunc, Flags);
7320 MI.eraseFromParent();
7326 const unsigned NumOps =
MI.getNumOperands();
7327 auto [DstReg, DstTy, Src0Reg, Src0Ty] =
MI.getFirst2RegLLTs();
7328 unsigned PartSize = Src0Ty.getSizeInBits();
7333 for (
unsigned I = 2;
I != NumOps; ++
I) {
7334 const unsigned Offset = (
I - 1) * PartSize;
7339 Register NextResult =
I + 1 == NumOps && WideTy == DstTy ? DstReg :
7345 ResultReg = NextResult;
7348 if (DstTy.isPointer()) {
7350 DstTy.getAddressSpace())) {
7358 MI.eraseFromParent();
7364 const unsigned NumDst =
MI.getNumOperands() - 1;
7365 Register SrcReg =
MI.getOperand(NumDst).getReg();
7366 Register Dst0Reg =
MI.getOperand(0).getReg();
7381 unsigned Offset = DstSize;
7382 for (
unsigned I = 1;
I != NumDst; ++
I,
Offset += DstSize) {
7388 MI.eraseFromParent();
7407 if (
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
7408 InsertVal =
MI.getOperand(2).getReg();
7422 SrcRegs[IdxVal] =
MI.getOperand(2).getReg();
7428 MI.eraseFromParent();
7433 LLVM_DEBUG(
dbgs() <<
"Can't handle non-byte element vectors yet\n");
7451 int64_t
Offset = IdxVal * EltBytes;
7470 MI.eraseFromParent();
7476 auto [DstReg, DstTy, Src0Reg, Src0Ty, Src1Reg, Src1Ty] =
7477 MI.getFirst3RegLLTs();
7485 for (
int Idx : Mask) {
7487 if (!Undef.isValid())
7493 if (Src0Ty.isScalar()) {
7496 int NumElts = Src0Ty.getNumElements();
7497 Register SrcVec =
Idx < NumElts ? Src0Reg : Src1Reg;
7498 int ExtractIdx =
Idx < NumElts ?
Idx :
Idx - NumElts;
7505 if (DstTy.isScalar())
7509 MI.eraseFromParent();
7526 if (Alignment >
Align(1)) {
7538 const auto &MF = *
MI.getMF();
7539 const auto &TFI = *MF.getSubtarget().getFrameLowering();
7544 Register AllocSize =
MI.getOperand(1).getReg();
7555 MI.eraseFromParent();
7566 MI.eraseFromParent();
7577 MI.eraseFromParent();
7583 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
7584 unsigned Offset =
MI.getOperand(2).getImm();
7589 unsigned DstSize = DstTy.getSizeInBits();
7591 if ((
Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) &&
7598 for (
unsigned Idx =
Offset / SrcEltSize;
7602 if (SubVectorElts.
size() == 1)
7607 MI.eraseFromParent();
7612 if (DstTy.isScalar() &&
7615 LLT SrcIntTy = SrcTy;
7629 MI.eraseFromParent();
7637 auto [Dst, Src, InsertSrc] =
MI.getFirst3Regs();
7649 if ((
Offset % EltSize == 0) && (InsertSize % EltSize == 0) &&
7662 for (
unsigned i = 0;
Idx < (
Offset + InsertSize) / EltSize;
7664 DstElts.
push_back(UnmergeInsertSrc.getReg(i));
7677 MI.eraseFromParent();
7691 LLVM_DEBUG(
dbgs() <<
"Not casting non-integral address space integer\n");
7695 LLT IntDstTy = DstTy;
7721 MI.eraseFromParent();
7727 auto [Dst0, Dst0Ty, Dst1, Dst1Ty,
LHS, LHSTy,
RHS, RHSTy] =
7728 MI.getFirst4RegLLTs();
7729 const bool IsAdd =
MI.getOpcode() == TargetOpcode::G_SADDO;
7732 LLT BoolTy = Dst1Ty;
7751 auto ResultLowerThanLHS =
7759 MI.eraseFromParent();
7766 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
7771 switch (
MI.getOpcode()) {
7774 case TargetOpcode::G_UADDSAT:
7777 BaseOp = TargetOpcode::G_ADD;
7779 case TargetOpcode::G_SADDSAT:
7782 BaseOp = TargetOpcode::G_ADD;
7784 case TargetOpcode::G_USUBSAT:
7787 BaseOp = TargetOpcode::G_SUB;
7789 case TargetOpcode::G_SSUBSAT:
7792 BaseOp = TargetOpcode::G_SUB;
7835 MI.eraseFromParent();
7841 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
7846 unsigned OverflowOp;
7847 switch (
MI.getOpcode()) {
7850 case TargetOpcode::G_UADDSAT:
7853 OverflowOp = TargetOpcode::G_UADDO;
7855 case TargetOpcode::G_SADDSAT:
7858 OverflowOp = TargetOpcode::G_SADDO;
7860 case TargetOpcode::G_USUBSAT:
7863 OverflowOp = TargetOpcode::G_USUBO;
7865 case TargetOpcode::G_SSUBSAT:
7868 OverflowOp = TargetOpcode::G_SSUBO;
7874 Register Tmp = OverflowRes.getReg(0);
7875 Register Ov = OverflowRes.getReg(1);
7901 MI.eraseFromParent();
7907 assert((
MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
7908 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&
7909 "Expected shlsat opcode!");
7910 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SSHLSAT;
7911 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
7933 MI.eraseFromParent();
7938 auto [Dst, Src] =
MI.getFirst2Regs();
7941 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
7950 for (
unsigned i = 1; i < SizeInBytes / 2; ++i) {
7952 APInt APMask(SizeInBytes * 8, 0xFF << (i * 8));
7964 Res.getInstr()->getOperand(0).setReg(Dst);
7966 MI.eraseFromParent();
7973 const LLT Ty = Dst.getLLTTy(*
B.getMRI());
7976 auto LHS =
B.buildLShr(Ty,
B.buildAnd(Ty, Src, MaskLoNTo0), C_N);
7977 auto RHS =
B.buildAnd(Ty,
B.buildShl(Ty, Src, C_N), MaskLoNTo0);
7978 return B.buildOr(Dst,
LHS,
RHS);
7983 auto [Dst, Src] =
MI.getFirst2Regs();
8011 for (
unsigned I = 0, J =
Size - 1;
I <
Size; ++
I, --J) {
8031 MI.eraseFromParent();
8039 bool IsRead =
MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
8040 int NameOpIdx = IsRead ? 1 : 0;
8041 int ValRegIndex = IsRead ? 0 : 1;
8043 Register ValReg =
MI.getOperand(ValRegIndex).getReg();
8045 const MDString *RegStr = cast<MDString>(
8046 cast<MDNode>(
MI.getOperand(NameOpIdx).getMetadata())->getOperand(0));
8057 MI.eraseFromParent();
8063 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SMULH;
8064 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
8073 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
8079 MI.eraseFromParent();
8085 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
8090 MI.eraseFromParent();
8095 MI.eraseFromParent();
8114 APInt ExpMask = Inf;
8132 LLT DstTyCopy = DstTy;
8147 Mask &= ~fcPosFinite;
8154 Mask &= ~fcNegFinite;
8165 Mask &= ~PartialCheck;
8174 else if (PartialCheck ==
fcZero)
8193 appendToRes(SubnormalRes);
8200 else if (PartialCheck ==
fcInf)
8213 if (PartialCheck ==
fcNan) {
8217 }
else if (PartialCheck ==
fcQNan) {
8227 Abs, InfWithQnanBitC);
8235 APInt ExpLSB = ExpMask & ~(ExpMask.
shl(1));
8238 APInt MaxExpMinusOne = ExpMask - ExpLSB;
8249 appendToRes(NormalRes);
8253 MI.eraseFromParent();
8259 auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] =
8260 MI.getFirst4RegLLTs();
8262 bool IsEltPtr = DstTy.isPointerOrPointerVector();
8271 if (MaskTy.isScalar()) {
8285 if (DstTy.isVector()) {
8288 MaskReg = ShufSplat.
getReg(0);
8293 }
else if (!DstTy.isVector()) {
8298 if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) {
8311 MI.eraseFromParent();
8317 unsigned Opcode =
MI.getOpcode();
8320 Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV
8321 : TargetOpcode::G_UDIV,
8322 {
MI.getOperand(0).
getReg()}, {
MI.getOperand(2),
MI.getOperand(3)});
8324 Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM
8325 : TargetOpcode::G_UREM,
8326 {
MI.getOperand(1).
getReg()}, {
MI.getOperand(2),
MI.getOperand(3)});
8327 MI.eraseFromParent();
8344 MI.eraseFromParent();
8359 MI.eraseFromParent();
8366 Register DestReg =
MI.getOperand(0).getReg();
8372 MI.eraseFromParent();
8399 Register ListPtr =
MI.getOperand(1).getReg();
8409 const Align A(
MI.getOperand(2).getImm());
8416 VAList = AndDst.
getReg(0);
8434 Align EltAlignment =
DL.getABITypeAlign(Ty);
8439 MI.eraseFromParent();
8454 unsigned Limit,
const MemOp &
Op,
8455 unsigned DstAS,
unsigned SrcAS,
8458 if (
Op.isMemcpyWithFixedDstAlign() &&
Op.getSrcAlign() <
Op.getDstAlign())
8468 if (
Op.isFixedDstAlign())
8476 unsigned NumMemOps = 0;
8480 while (TySize >
Size) {
8489 assert(NewTySize > 0 &&
"Could not find appropriate type");
8496 if (NumMemOps &&
Op.allowOverlap() && NewTySize <
Size &&
8498 VT, DstAS,
Op.isFixedDstAlign() ?
Op.getDstAlign() :
Align(1),
8508 if (++NumMemOps > Limit)
8511 MemOps.push_back(Ty);
8523 if (!Ty.
isVector() && ValVRegAndVal) {
8524 APInt Scalar = ValVRegAndVal->Value.trunc(8);
8532 if (ValVRegAndVal && ValVRegAndVal->Value == 0) {
8555 auto &MF = *
MI.getParent()->getParent();
8556 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8557 auto &
DL = MF.getDataLayout();
8560 assert(KnownLen != 0 &&
"Have a zero length memset length!");
8562 bool DstAlignCanChange =
false;
8568 DstAlignCanChange =
true;
8571 std::vector<LLT> MemOps;
8573 const auto &DstMMO = **
MI.memoperands_begin();
8577 bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
8585 MF.getFunction().getAttributes(), TLI))
8588 if (DstAlignCanChange) {
8591 Align NewAlign =
DL.getABITypeAlign(IRTy);
8592 if (NewAlign > Alignment) {
8593 Alignment = NewAlign;
8603 LLT LargestTy = MemOps[0];
8604 for (
unsigned i = 1; i < MemOps.size(); i++)
8606 LargestTy = MemOps[i];
8619 unsigned DstOff = 0;
8620 unsigned Size = KnownLen;
8621 for (
unsigned I = 0;
I < MemOps.size();
I++) {
8624 if (TySize >
Size) {
8627 assert(
I == MemOps.size() - 1 &&
I != 0);
8628 DstOff -= TySize -
Size;
8639 Value = MIB.buildTrunc(Ty, MemSetValue).getReg(0);
8646 auto *StoreMMO = MF.getMachineMemOperand(&DstMMO, DstOff, Ty);
8652 Ptr = MIB.buildPtrAdd(PtrTy, Dst,
Offset).getReg(0);
8655 MIB.buildStore(
Value,
Ptr, *StoreMMO);
8660 MI.eraseFromParent();
8666 assert(
MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE);
8668 auto [Dst, Src, Len] =
MI.getFirst3Regs();
8670 const auto *MMOIt =
MI.memoperands_begin();
8672 bool IsVolatile =
MemOp->isVolatile();
8678 "inline memcpy with dynamic size is not yet supported");
8679 uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue();
8680 if (KnownLen == 0) {
8681 MI.eraseFromParent();
8685 const auto &DstMMO = **
MI.memoperands_begin();
8686 const auto &SrcMMO = **std::next(
MI.memoperands_begin());
8687 Align DstAlign = DstMMO.getBaseAlign();
8688 Align SrcAlign = SrcMMO.getBaseAlign();
8690 return lowerMemcpyInline(
MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
8697 Align SrcAlign,
bool IsVolatile) {
8698 assert(
MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE);
8699 return lowerMemcpy(
MI, Dst, Src, KnownLen,
8700 std::numeric_limits<uint64_t>::max(), DstAlign, SrcAlign,
8707 Align SrcAlign,
bool IsVolatile) {
8708 auto &MF = *
MI.getParent()->getParent();
8709 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8710 auto &
DL = MF.getDataLayout();
8713 assert(KnownLen != 0 &&
"Have a zero length memcpy length!");
8715 bool DstAlignCanChange =
false;
8717 Align Alignment = std::min(DstAlign, SrcAlign);
8721 DstAlignCanChange =
true;
8727 std::vector<LLT> MemOps;
8729 const auto &DstMMO = **
MI.memoperands_begin();
8730 const auto &SrcMMO = **std::next(
MI.memoperands_begin());
8736 MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
8739 MF.getFunction().getAttributes(), TLI))
8742 if (DstAlignCanChange) {
8745 Align NewAlign =
DL.getABITypeAlign(IRTy);
8750 if (!
TRI->hasStackRealignment(MF))
8751 while (NewAlign > Alignment &&
DL.exceedsNaturalStackAlignment(NewAlign))
8754 if (NewAlign > Alignment) {
8755 Alignment = NewAlign;
8763 LLVM_DEBUG(
dbgs() <<
"Inlining memcpy: " <<
MI <<
" into loads & stores\n");
8771 unsigned CurrOffset = 0;
8772 unsigned Size = KnownLen;
8773 for (
auto CopyTy : MemOps) {
8776 if (CopyTy.getSizeInBytes() >
Size)
8777 CurrOffset -= CopyTy.getSizeInBytes() -
Size;
8781 MF.getMachineMemOperand(&SrcMMO, CurrOffset, CopyTy.getSizeInBytes());
8783 MF.getMachineMemOperand(&DstMMO, CurrOffset, CopyTy.getSizeInBytes());
8788 if (CurrOffset != 0) {
8792 LoadPtr = MIB.buildPtrAdd(SrcTy, Src,
Offset).getReg(0);
8794 auto LdVal = MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO);
8798 if (CurrOffset != 0) {
8800 StorePtr = MIB.buildPtrAdd(DstTy, Dst,
Offset).getReg(0);
8802 MIB.buildStore(LdVal, StorePtr, *StoreMMO);
8803 CurrOffset += CopyTy.getSizeInBytes();
8804 Size -= CopyTy.getSizeInBytes();
8807 MI.eraseFromParent();
8815 auto &MF = *
MI.getParent()->getParent();
8816 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8817 auto &
DL = MF.getDataLayout();
8820 assert(KnownLen != 0 &&
"Have a zero length memmove length!");
8822 bool DstAlignCanChange =
false;
8825 Align Alignment = std::min(DstAlign, SrcAlign);
8829 DstAlignCanChange =
true;
8832 std::vector<LLT> MemOps;
8834 const auto &DstMMO = **
MI.memoperands_begin();
8835 const auto &SrcMMO = **std::next(
MI.memoperands_begin());
8844 MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
8847 MF.getFunction().getAttributes(), TLI))
8850 if (DstAlignCanChange) {
8853 Align NewAlign =
DL.getABITypeAlign(IRTy);
8858 if (!
TRI->hasStackRealignment(MF))
8859 while (NewAlign > Alignment &&
DL.exceedsNaturalStackAlignment(NewAlign))
8862 if (NewAlign > Alignment) {
8863 Alignment = NewAlign;
8871 LLVM_DEBUG(
dbgs() <<
"Inlining memmove: " <<
MI <<
" into loads & stores\n");
8877 unsigned CurrOffset = 0;
8879 for (
auto CopyTy : MemOps) {
8882 MF.getMachineMemOperand(&SrcMMO, CurrOffset, CopyTy.getSizeInBytes());
8886 if (CurrOffset != 0) {
8890 LoadPtr = MIB.buildPtrAdd(SrcTy, Src,
Offset).getReg(0);
8892 LoadVals.
push_back(MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO).getReg(0));
8893 CurrOffset += CopyTy.getSizeInBytes();
8897 for (
unsigned I = 0;
I < MemOps.size(); ++
I) {
8898 LLT CopyTy = MemOps[
I];
8901 MF.getMachineMemOperand(&DstMMO, CurrOffset, CopyTy.
getSizeInBytes());
8904 if (CurrOffset != 0) {
8908 StorePtr = MIB.buildPtrAdd(DstTy, Dst,
Offset).getReg(0);
8910 MIB.buildStore(LoadVals[
I], StorePtr, *StoreMMO);
8913 MI.eraseFromParent();
8919 const unsigned Opc =
MI.getOpcode();
8922 assert((Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMMOVE ||
8923 Opc == TargetOpcode::G_MEMSET) &&
8924 "Expected memcpy like instruction");
8926 auto MMOIt =
MI.memoperands_begin();
8931 auto [Dst, Src, Len] =
MI.getFirst3Regs();
8933 if (Opc != TargetOpcode::G_MEMSET) {
8934 assert(MMOIt !=
MI.memoperands_end() &&
"Expected a second MMO on MI");
8936 SrcAlign =
MemOp->getBaseAlign();
8943 uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue();
8945 if (KnownLen == 0) {
8946 MI.eraseFromParent();
8950 bool IsVolatile =
MemOp->isVolatile();
8951 if (Opc == TargetOpcode::G_MEMCPY_INLINE)
8952 return lowerMemcpyInline(
MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
8959 if (MaxLen && KnownLen > MaxLen)
8962 if (Opc == TargetOpcode::G_MEMCPY) {
8963 auto &MF = *
MI.getParent()->getParent();
8964 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8967 return lowerMemcpy(
MI, Dst, Src, KnownLen, Limit, DstAlign, SrcAlign,
8970 if (Opc == TargetOpcode::G_MEMMOVE)
8971 return lowerMemmove(
MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile);
8972 if (Opc == TargetOpcode::G_MEMSET)
8973 return lowerMemset(
MI, Dst, Src, KnownLen, DstAlign, IsVolatile);
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file describes how to lower LLVM calls to machine code calls.
#define GISEL_VECREDUCE_CASES_NONSEQ
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Rewrite Partial Register Uses
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
#define RTLIBCASE_INT(LibcallPrefix)
static bool findGISelOptimalMemOpLowering(std::vector< LLT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, const TargetLowering &TLI)
static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI)
static Register buildBitFieldInsert(MachineIRBuilder &B, Register TargetReg, Register InsertReg, Register OffsetBits)
Emit code to insert InsertReg into TargetRet at OffsetBits in TargetReg, while preserving other bits ...
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size)
static void broadcastSrcOp(SmallVectorImpl< SrcOp > &Ops, unsigned N, MachineOperand &Op)
Operand Op is used on N sub-instructions.
static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result, MachineInstr &MI, const TargetInstrInfo &TII, MachineRegisterInfo &MRI)
True if an instruction is in tail position in its caller.
static LegalizerHelper::LegalizeResult simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType, LostDebugLocObserver &LocObserver)
static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B, Register Idx, unsigned NewEltSize, unsigned OldEltSize)
Figure out the bit offset into a register when coercing a vector index for the wide element type.
static void makeDstOps(SmallVectorImpl< DstOp > &DstOps, LLT Ty, unsigned NumElts)
Fill DstOps with DstOps that have same number of elements combined as the Ty.
static bool shouldLowerMemFuncForSize(const MachineFunction &MF)
static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, MachineInstrBuilder Src, const APInt &Mask)
static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal, MachineIRBuilder &MIRBuilder)
static void getUnmergePieces(SmallVectorImpl< Register > &Pieces, MachineIRBuilder &B, Register Src, LLT Ty)
static CmpInst::Predicate minMaxToCompare(unsigned Opc)
static LegalizerHelper::LegalizeResult createAtomicLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI)
static RTLIB::Libcall getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI)
static std::pair< int, int > getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy)
Try to break down OrigTy into NarrowTy sized pieces.
static bool hasSameNumEltsOnAllVectorOperands(GenericMachineInstr &MI, MachineRegisterInfo &MRI, std::initializer_list< unsigned > NonVecOpIndices)
Check that all vector operands have same number of elements.
static LegalizerHelper::LegalizeResult conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, Type *FromType, LostDebugLocObserver &LocObserver)
static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg, LLT VecTy)
static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, Type *FromType)
static void getUnmergeResults(SmallVectorImpl< Register > &Regs, const MachineInstr &MI)
Append the result registers of G_UNMERGE_VALUES MI to Regs.
static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI, Register Reg, unsigned BW)
#define RTLIBCASE(LibcallPrefix)
static Type * getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty)
Interface for Targets to specify which operations they can successfully select and how the others sho...
Tracks DebugLocs between checkpoints and verifies that they are transferred.
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
APInt bitcastToAPInt() const
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
void negate()
Negate this APInt in place.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
APInt sext(unsigned width) const
Sign extend to a new width.
APInt shl(unsigned shiftAmt) const
Left-shift function.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
static APInt getBitsSetWithWrap(unsigned numBits, unsigned loBit, unsigned hiBit)
Wrap version of getBitsSet.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
bool hasAttributes() const
Return true if the builder has IR-level attributes.
AttrBuilder & removeAttribute(Attribute::AttrKind Val)
Remove an attribute from the builder.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasRetAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the return value.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ ICMP_ULT
unsigned less than
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
const APInt & getValue() const
Return the constant as an APInt value reference.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
bool isNonIntegralAddressSpace(unsigned AddrSpace) const
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Represents any generic load, including sign/zero extending variants.
Register getDstReg() const
Get the definition register of the loaded value.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
LocationSize getMemSize() const
Returns the size in bytes of the memory access.
bool isAtomic() const
Returns true if the attached MachineMemOperand has the atomic flag set.
Register getValueReg() const
Get the stored value register.
A base class for all GenericMachineInstrs.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
bool isTailCall(const MachineInstr &MI) const override
bool isEquality() const
Return true if this predicate is either EQ or NE.
Predicate getUnsignedPredicate() const
For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
constexpr bool isPointerVector() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr bool isScalable() const
Returns true if the LLT is a scalable vector.
constexpr bool isByteSized() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr LLT getScalarType() const
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
static constexpr LLT scalarOrVector(ElementCount EC, LLT ScalarTy)
This is an important class for using LLVM in a threaded context.
LegalizeResult lowerShlSat(MachineInstr &MI)
LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI)
LegalizeResult equalizeVectorShuffleLengths(MachineInstr &MI)
Equalize source and destination vector sizes of G_SHUFFLE_VECTOR.
LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT CastTy)
Perform Bitcast legalize action on G_INSERT_VECTOR_ELT.
LegalizeResult lowerSITOFP(MachineInstr &MI)
LegalizeResult lowerDynStackAlloc(MachineInstr &MI)
LegalizeResult lowerBitCount(MachineInstr &MI)
LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty)
LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI)
LegalizeResult lowerIntrinsicRound(MachineInstr &MI)
void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, unsigned ExtOpcode)
Legalize a single operand OpIdx of the machine instruction MI as a Use by extending the operand's typ...
LegalizeResult moreElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
LegalizeResult lowerSMULH_UMULH(MachineInstr &MI)
LegalizeResult lowerLoad(GAnyLoad &MI)
LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult lowerAbsToAddXor(MachineInstr &MI)
void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Def by performing it with addition...
LegalizeResult lowerFConstant(MachineInstr &MI)
LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerBitreverse(MachineInstr &MI)
LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI)
Lower a vector extract or insert by writing the vector to a stack temporary and reloading the element...
LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
Legalize a vector instruction by increasing the number of vector elements involved and ignoring the a...
LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI)
LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI)
LegalizeResult lowerEXT(MachineInstr &MI)
LegalizeResult lowerStore(GStore &MI)
LegalizeResult lowerAbsToCNeg(MachineInstr &MI)
LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI)
MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, MachinePointerInfo &PtrInfo)
Create a stack temporary based on the size in bytes and the alignment.
void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Use by truncating the operand's ty...
LegalizeResult fewerElementsVectorPhi(GenericMachineInstr &MI, unsigned NumElts)
LegalizeResult lowerFPTOUI(MachineInstr &MI)
LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize an instruction by reducing the width of the underlying scalar type.
LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerUnmergeValues(MachineInstr &MI)
LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by replacing the value type.
LegalizeResult lowerBitcast(MachineInstr &MI)
LegalizeResult lowerMinMax(MachineInstr &MI)
LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI)
LegalizeResult lowerInsert(MachineInstr &MI)
LegalizeResult lowerReadWriteRegister(MachineInstr &MI)
LegalizeResult lowerExtract(MachineInstr &MI)
LegalizeResult fewerElementsBitcast(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, LLT HalfTy, LLT ShiftAmtTy)
LegalizeResult lowerISFPCLASS(MachineInstr &MI)
LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI)
LegalizeResult lowerFPOWI(MachineInstr &MI)
LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerVectorReduction(MachineInstr &MI)
LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult fewerElementsVectorMultiEltType(GenericMachineInstr &MI, unsigned NumElts, std::initializer_list< unsigned > NonVecOpIndices={})
Handles most opcodes.
LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerVAArg(MachineInstr &MI)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
@ AlreadyLegal
Instruction was already legal and no change was made to the MachineFunction.
@ UnableToLegalize
Some kind of error has occurred and we could not legalize this instruction.
LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI)
LegalizeResult lowerFCopySign(MachineInstr &MI)
LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, MachineIRBuilder &B)
LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI)
LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by splitting it into simpler parts, hopefully understood by the target.
LegalizeResult lowerFunnelShift(MachineInstr &MI)
LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize a vector instruction by splitting into multiple components, each acting on the same scalar t...
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a def by inserting a G_BITCAST from ...
LegalizeResult lowerFPTRUNC(MachineInstr &MI)
LegalizeResult lowerFMad(MachineInstr &MI)
LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy)
Legalize an instruction by performing the operation on a wider scalar type (for example a 16-bit addi...
LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI)
LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerFFloor(MachineInstr &MI)
LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult fewerElementsVectorSeqReductions(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Register getDynStackAllocTargetPtr(Register SPReg, Register AllocSize, Align Alignment, LLT PtrTy)
LegalizeResult lowerFPTOSI(MachineInstr &MI)
LegalizeResult lowerUITOFP(MachineInstr &MI)
LegalizeResult lowerShuffleVector(MachineInstr &MI)
LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult lowerMergeValues(MachineInstr &MI)
LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Use by producing a vector with und...
LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT CastTy)
Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT.
LegalizeResult lowerRotate(MachineInstr &MI)
LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0)
Register coerceToScalar(Register Val)
Cast the given value to an LLT::scalar with an equivalent size.
LegalizeResult lowerDIVREM(MachineInstr &MI)
LegalizeResult lowerSelect(MachineInstr &MI)
LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a use by inserting a G_BITCAST to Ca...
void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, unsigned ExtOpcode)
LegalizeResult libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver)
Legalize an instruction by emiting a runtime library call instead.
LegalizeResult lowerStackRestore(MachineInstr &MI)
LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult lowerStackSave(MachineInstr &MI)
LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LegalizeResult lowerTRUNC(MachineInstr &MI)
LegalizeResult lowerBswap(MachineInstr &MI)
Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index)
Get a pointer to vector element Index located in memory for a vector of type VecTy starting at a base...
LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Align getStackTemporaryAlignment(LLT Type, Align MinAlign=Align()) const
Return the alignment to use for a stack temporary object with the given type.
LegalizeResult lowerConstant(MachineInstr &MI)
void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx=0, unsigned TruncOpcode=TargetOpcode::G_TRUNC)
Legalize a single operand OpIdx of the machine instruction MI as a Def by extending the operand's typ...
LegalizeResult legalizeInstrStep(MachineInstr &MI, LostDebugLocObserver &LocObserver)
Replace MI by a sequence of legal instructions that can implement the same operation.
virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const
Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while widening a constant of type Small...
bool isLegalOrCustom(const LegalityQuery &Query) const
virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const
Called for instructions with the Custom LegalizationAction.
virtual bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
TypeSize getValue() const
void checkpoint(bool CheckDebugLocs=true)
Call this to indicate that it's a good point to assess whether locations have been lost.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
StringRef getString() const
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
iterator getFirstTerminatorForward()
Finds the first terminator in a block by scanning forward.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
Helper class to build MachineInstr.
MachineInstrBuilder buildFSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FSUB Op0, Op1.
MachineInstrBuilder buildFPTOSI(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_FPTOSI Src0.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildConstantPool(const DstOp &Res, unsigned Idx)
Build and insert Res = G_CONSTANT_POOL Idx.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FABS Op0.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildZExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and inserts Res = G_AND Op, LowBitsSet(ImmOp) Since there is no G_ZEXT_INREG like G_SEXT_INREG,...
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ... = G_EXTRACT Src, Idx0.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
const TargetInstrInfo & getTII()
MachineInstrBuilder buildURem(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_UREM Op0, Op1.
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFPow(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_FPOW Src0, Src1.
MachineInstrBuilder buildAnyExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Res = COPY Op depending on the differing sizes of Res and Op.
MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op.
MachineInstrBuilder buildIntrinsicTrunc(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_INTRINSIC_TRUNC Src0.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildShuffleSplat(const DstOp &Res, const SrcOp &Src)
Build and insert a vector splat of a scalar Src using a G_INSERT_VECTOR_ELT and G_SHUFFLE_VECTOR idio...
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ZEXT Op.
MachineInstrBuilder buildConcatVectors(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_CONCAT_VECTORS Op0, ...
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildCTLZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ_ZERO_UNDEF Op0, Src0.
MachineInstrBuilder buildVScale(const DstOp &Res, unsigned MinElts)
Build and insert Res = G_VSCALE MinElts.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildIntToPtr(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_INTTOPTR instruction.
unsigned getBoolExtOp(bool IsVec, bool IsFP) const
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildNeg(const DstOp &Dst, const SrcOp &Src0)
Build and insert integer negation Zero = G_CONSTANT 0 Res = G_SUB Zero, Op0.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMAX Op0, Op1.
MachineInstrBuilder buildAssertZExt(const DstOp &Res, const SrcOp &Op, unsigned Size)
Build and insert Res = G_ASSERT_ZEXT Op, Size.
MachineInstrBuilder buildStrictFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_STRICT_FADD Op0, Op1.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildCTTZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTTZ_ZERO_UNDEF Op0, Src0.
virtual MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val)
Build and insert Res = G_FCONSTANT Val.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildUITOFP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_UITOFP Src0.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildSITOFP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_SITOFP Src0.
MachineInstrBuilder buildPadVectorWithUndefElements(const DstOp &Res, const SrcOp &Op0)
Build and insert a, b, ..., x = G_UNMERGE_VALUES Op0 Res = G_BUILD_VECTOR a, b, .....
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMIN Op0, Op1.
MachineInstrBuilder buildInsert(const DstOp &Res, const SrcOp &Src, const SrcOp &Op, unsigned Index)
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_TRUNC Op.
MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_FCOPYSIGN Op0, Op1.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FNEG Op0.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineInstrBuilder buildDeleteTrailingVectorElements(const DstOp &Res, const SrcOp &Op0)
Build and insert a, b, ..., x, y, z = G_UNMERGE_VALUES Op0 Res = G_BUILD_VECTOR a,...
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
MachineInstrBuilder buildAtomicCmpXchg(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, MMO.
MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef< int > Mask)
Build and insert Res = G_SHUFFLE_VECTOR Src1, Src2, Mask.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
const DataLayout & getDataLayout() const
MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = <opcode> Addr, MMO.
MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_XOR Op0, Op1.
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMIN Op0, Op1.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
MachineInstrBuilder buildFCanonicalize(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_FCANONICALIZE Src0.
MachineInstrBuilder buildSExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and insert Res = G_SEXT_INREG Op, ImmOp.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isReturn(QueryType Type=AnyInBundle) const
bool isDebugInstr() const
unsigned getNumOperands() const
Retuns the total number of operands.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
void setType(LLT NewTy)
Reset the tracked memory type.
LLT getMemoryType() const
Return the memory type of the memory reference.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
LocationSize getSizeInBits() const
Return the size in bits of the memory reference.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
const ConstantInt * getCImm() const
void setReg(Register Reg)
Change the register this operand corresponds to.
void setCImm(const ConstantInt *CI)
Register getReg() const
getReg - Returns the register number.
const ConstantFP * getFPImm() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Register cloneVirtualRegister(Register VReg, StringRef Name="")
Create and return a new virtual register in the function with the same attributes as the given regist...
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
virtual LLT getOptimalMemOpLLT(const MemOp &Op, const AttributeList &) const
LLT returning variant.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
const Triple & getTargetTriple() const
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const CallLowering * getCallLowering() const
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getHalfTy(LLVMContext &C)
static Type * getDoubleTy(LLVMContext &C)
static Type * getX86_FP80Ty(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Type * getVoidTy(LLVMContext &C)
static Type * getFP128Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static Type * getFloatTy(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ FewerElements
The (vector) operation should be implemented by splitting it into sub-vectors where the operation is ...
@ Libcall
The operation should be implemented as a call to some kind of runtime support library.
@ WidenScalar
The operation should be implemented in terms of a wider scalar base-type.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
@ NarrowScalar
The operation should be synthesized from multiple instructions acting on a narrower scalar base-type.
@ Custom
The target wants to do something special with this combination of operand and type.
@ MoreElements
The (vector) operation should be implemented by widening the input vector and ignoring the lanes adde...
ConstantMatch< APInt > m_ICst(APInt &Cst)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
int64_t maxIntN(int64_t N)
Gets the maximum value for a N-bit signed integer.
Type * getTypeForLLT(LLT Ty, LLVMContext &C)
Get the type back from LLT.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg, std::function< bool(const Constant *ConstVal)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant G_B...
LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, MachineInstr &MI, LostDebugLocObserver &LocObserver)
Create a libcall to memcpy et al.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
LLVM_READNONE LLT getLCMType(LLT OrigTy, LLT TargetTy)
Return the least common multiple type of OrigTy and TargetTy, by changing the number of vector elemen...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LegalizerHelper::LegalizeResult createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, const CallLowering::ArgInfo &Result, ArrayRef< CallLowering::ArgInfo > Args, CallingConv::ID CC, LostDebugLocObserver &LocObserver, MachineInstr *MI=nullptr)
Helper function that creates a libcall to the given Name using the given calling convention CC.
void extractParts(Register Reg, LLT Ty, int NumParts, SmallVectorImpl< Register > &VRegs, MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
Helper function to split a wide generic register into bitwise blocks with the given Type (which impli...
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
int64_t minIntN(int64_t N)
Gets the minimum value for a N-bit signed integer.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI)
Returns true if Val can be assumed to never be a signaling NaN.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
unsigned Log2(Align A)
Returns the log2 of the alignment.
LLVM_READNONE LLT getGCDType(LLT OrigTy, LLT TargetTy)
Return a type where the total size is the greatest common divisor of OrigTy and TargetTy.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
void extractVectorParts(Register Reg, unsigned NumElts, SmallVectorImpl< Register > &VRegs, MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
Version which handles irregular sub-vector splits.
static const fltSemantics & IEEEsingle() LLVM_READNONE
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
LegalizeAction Action
The action to take or the final answer.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)