28#include "llvm/IR/IntrinsicsAArch64.h"
31#include <initializer_list>
33#define DEBUG_TYPE "aarch64-legalinfo"
36using namespace LegalizeActions;
37using namespace LegalizeMutations;
38using namespace LegalityPredicates;
39using namespace MIPatternMatch;
43 using namespace TargetOpcode;
67 std::initializer_list<LLT> PackedVectorAllTypeList = {
73 std::initializer_list<LLT> ScalarAndPtrTypesList = {s8, s16, s32, s64, p0};
77 const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
80 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
87 const bool HasFP16 = ST.hasFullFP16();
88 const LLT &MinFPScalar = HasFP16 ? s16 : s32;
90 const bool HasCSSC = ST.hasCSSC();
91 const bool HasRCPC3 = ST.hasRCPC3();
92 const bool HasSVE = ST.hasSVE();
95 {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
96 .legalFor({p0, s8, s16, s32, s64})
97 .legalFor({v16s8, v8s16, v4s32, v2s64, v2p0, v8s8, v4s16, v2s32, v4s8,
99 .widenScalarToNextPow2(0)
112 .legalFor(PackedVectorAllTypeList)
124 .
legalFor({s32, s64, v4s16, v8s16, v2s32, v4s32, v2s64})
126 .clampScalar(0, s32, s64)
127 .clampNumElements(0, v4s16, v8s16)
128 .clampNumElements(0, v2s32, v4s32)
129 .clampNumElements(0, v2s64, v2s64)
130 .moreElementsToNextPow2(0);
133 .legalFor({s32, s64, v2s32, v2s64, v4s32, v4s16, v8s16, v16s8, v8s8})
134 .legalFor(HasSVE, {nxv16s8, nxv8s16, nxv4s32, nxv2s64})
135 .widenScalarToNextPow2(0)
143 return Query.
Types[0].getNumElements() <= 2;
148 return Query.
Types[0].getNumElements() <= 4;
153 return Query.
Types[0].getNumElements() <= 16;
160 .
legalFor({s32, s64, v2s32, v2s64, v4s32, v4s16, v8s16, v16s8, v8s8})
161 .widenScalarToNextPow2(0)
169 return Query.
Types[0].getNumElements() <= 2;
174 return Query.
Types[0].getNumElements() <= 4;
179 return Query.
Types[0].getNumElements() <= 16;
187 const auto &SrcTy = Query.
Types[0];
188 const auto &AmtTy = Query.
Types[1];
189 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
190 AmtTy.getSizeInBits() == 32;
204 .widenScalarToNextPow2(0)
216 .
legalFor({{p0, s64}, {v2p0, v2s64}})
217 .clampScalarOrElt(1, s64, s64)
223 .legalFor({s32, s64})
225 .clampScalar(0, s32, s64)
230 .lowerFor({s8, s16, s32, s64, v2s64, v4s32, v2s32})
233 .minScalarOrElt(0, s32)
234 .clampNumElements(0, v2s32, v4s32)
235 .clampNumElements(0, v2s64, v2s64)
239 .widenScalarToNextPow2(0, 32)
244 .legalFor({s64, v8s16, v16s8, v4s32})
248 .legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
249 .legalFor(HasCSSC, {s32, s64})
250 .minScalar(HasCSSC, 0, s32)
260 {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
261 .legalFor({{s32, s32}, {s64, s32}})
262 .clampScalar(0, s32, s64)
267 {G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT, G_FMAXNUM, G_FMINNUM,
268 G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
269 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
270 .legalFor({s32, s64, v2s32, v4s32, v2s64})
271 .legalFor(HasFP16, {s16, v4s16, v8s16})
281 .legalFor({s32, s64, v2s32, v4s32, v2s64})
282 .legalFor(HasFP16, {s16, v4s16, v8s16})
297 .legalFor({{s64, MinFPScalar}, {s64, s32}, {s64, s64}})
298 .libcallFor({{s64, s128}})
299 .minScalarOrElt(1, MinFPScalar);
302 G_FLOG10, G_FTAN, G_FEXP, G_FEXP2, G_FEXP10,
303 G_FACOS, G_FASIN, G_FATAN, G_FATAN2, G_FCOSH,
309 .libcallFor({s32, s64, s128});
313 .
libcallFor({{s32, s32}, {s64, s32}, {s128, s32}});
337 for (
unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
340 if (
Op == G_SEXTLOAD)
345 .legalForTypesWithMemDesc({{s32, p0, s8, 8},
353 {v2s32, p0, s64, 8}})
354 .widenScalarToNextPow2(0)
355 .clampScalar(0, s32, s64)
358 .unsupportedIfMemSizeNotPow2()
370 return HasRCPC3 && Query.
Types[0] == s128 &&
374 return Query.
Types[0] == s128 &&
377 .legalForTypesWithMemDesc({{s8, p0, s8, 8},
384 {v16s8, p0, s128, 8},
386 {v8s16, p0, s128, 8},
388 {v4s32, p0, s128, 8},
389 {v2s64, p0, s128, 8}})
391 .legalForTypesWithMemDesc(
392 {{s32, p0, s8, 8}, {s32, p0, s16, 8}, {s64, p0, s32, 8}})
393 .legalForTypesWithMemDesc({
395 {nxv16s8, p0, nxv16s8, 8},
396 {nxv8s16, p0, nxv8s16, 8},
397 {nxv4s32, p0, nxv4s32, 8},
398 {nxv2s64, p0, nxv2s64, 8},
400 .widenScalarToNextPow2(0, 8)
411 return Query.
Types[0].isScalar() &&
413 Query.
Types[0].getSizeInBits() > 32;
422 .customIf(IsPtrVecPred)
428 return HasRCPC3 && Query.
Types[0] == s128 &&
432 return Query.
Types[0] == s128 &&
435 .legalForTypesWithMemDesc(
436 {{s8, p0, s8, 8}, {s16, p0, s8, 8},
439 {s16, p0, s16, 8}, {s32, p0, s16, 8},
441 {s32, p0, s8, 8}, {s32, p0, s16, 8}, {s32, p0, s32, 8},
442 {s64, p0, s64, 8}, {s64, p0, s32, 8},
443 {p0, p0, s64, 8}, {s128, p0, s128, 8}, {v16s8, p0, s128, 8},
444 {v8s8, p0, s64, 8}, {v4s16, p0, s64, 8}, {v8s16, p0, s128, 8},
445 {v2s32, p0, s64, 8}, {v4s32, p0, s128, 8}, {v2s64, p0, s128, 8}})
446 .legalForTypesWithMemDesc({
451 {nxv16s8, p0, nxv16s8, 8},
452 {nxv8s16, p0, nxv8s16, 8},
453 {nxv4s32, p0, nxv4s32, 8},
454 {nxv2s64, p0, nxv2s64, 8},
456 .clampScalar(0, s8, s64)
459 return Query.
Types[0].isScalar() &&
463 .clampMaxNumElements(0, s8, 16)
475 .customIf(IsPtrVecPred)
492 {p0, v16s8, v16s8, 8},
493 {p0, v4s16, v4s16, 8},
494 {p0, v8s16, v8s16, 8},
495 {p0, v2s32, v2s32, 8},
496 {p0, v4s32, v4s32, 8},
497 {p0, v2s64, v2s64, 8},
503 auto IndexedLoadBasicPred = [=](
const LegalityQuery &Query) {
531 return MemTy == s8 || MemTy == s16;
533 return MemTy == s8 || MemTy == s16 || MemTy == s32;
541 .widenScalarToNextPow2(0)
545 .legalFor(HasFP16, {s16})
546 .clampScalar(0, MinFPScalar, s128);
550 .
legalFor({{s32, s32}, {s32, s64}, {s32, p0}})
552 .clampScalar(1, s32, s64)
553 .clampScalar(0, s32, s32)
555 .minScalarEltSameAsIf(
568 return Query.
Types[1].isPointerVector();
572 .clampNumElements(1, v8s8, v16s8)
573 .clampNumElements(1, v4s16, v8s16)
574 .clampNumElements(1, v2s32, v4s32)
575 .clampNumElements(1, v2s64, v2s64)
576 .clampNumElements(1, v2p0, v2p0)
585 .legalFor(HasFP16, {{s32, s16}, {v4s16, v4s16}, {v8s16, v8s16}})
587 .clampScalar(0, s32, s32)
588 .minScalarOrElt(1, MinFPScalar)
590 .minScalarEltSameAsIf(
598 .clampNumElements(1, v4s16, v8s16)
599 .clampNumElements(1, v2s32, v4s32)
600 .clampMaxNumElements(1, s64, 2)
601 .moreElementsToNextPow2(1)
602 .libcallFor({{s32, s128}});
606 unsigned DstSize = Query.
Types[0].getSizeInBits();
609 if (Query.
Types[0].isVector())
612 if (DstSize < 8 || DstSize >= 128 || !
isPowerOf2_32(DstSize))
627 .legalIf(ExtLegalFunc)
628 .
legalFor({{v2s64, v2s32}, {v4s32, v4s16}, {v8s16, v8s8}})
629 .clampScalar(0, s64, s64)
636 return (Query.
Types[0].getScalarSizeInBits() >
637 Query.
Types[1].getScalarSizeInBits() * 2) &&
638 Query.
Types[0].isVector() &&
639 (Query.
Types[1].getScalarSizeInBits() == 8 ||
640 Query.
Types[1].getScalarSizeInBits() == 16);
642 .clampMinNumElements(1, s8, 8)
646 .
legalFor({{v2s32, v2s64}, {v4s16, v4s32}, {v8s8, v8s16}})
648 .clampMaxNumElements(0, s8, 8)
649 .clampMaxNumElements(0, s16, 4)
650 .clampMaxNumElements(0, s32, 2)
660 .clampMinNumElements(0, s8, 8)
661 .clampMinNumElements(0, s16, 4)
666 .legalFor(PackedVectorAllTypeList)
677 {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
678 .libcallFor({{s16, s128}, {s32, s128}, {s64, s128}})
679 .clampNumElements(0, v4s16, v4s16)
685 {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
686 .libcallFor({{s128, s64}, {s128, s32}, {s128, s16}})
687 .clampNumElements(0, v4s32, v4s32)
693 .legalFor({{s32, s32},
701 {{s32, s16}, {s64, s16}, {v4s16, v4s16}, {v8s16, v8s16}})
708 return Query.
Types[1] == s16 && Query.
Types[0].getSizeInBits() > 64;
712 .widenScalarOrEltToNextPow2OrMinSize(0)
714 .widenScalarOrEltToNextPow2OrMinSize(1, HasFP16 ? 16 : 32)
717 return Query.
Types[0].getScalarSizeInBits() <= 64 &&
718 Query.
Types[0].getScalarSizeInBits() >
719 Query.
Types[1].getScalarSizeInBits();
724 return Query.
Types[1].getScalarSizeInBits() <= 64 &&
725 Query.
Types[0].getScalarSizeInBits() <
726 Query.
Types[1].getScalarSizeInBits();
729 .clampNumElements(0, v4s16, v8s16)
730 .clampNumElements(0, v2s32, v4s32)
731 .clampMaxNumElements(0, s64, 2)
733 {{s32, s128}, {s64, s128}, {s128, s128}, {s128, s32}, {s128, s64}});
736 .legalFor({{s32, s32},
744 {{s32, s16}, {s64, s16}, {v4s16, v4s16}, {v8s16, v8s16}})
752 return Query.
Types[1] == s16 && Query.
Types[0].getSizeInBits() > 64;
757 .widenScalarToNextPow2(0, 32)
759 .widenScalarOrEltToNextPow2OrMinSize(1, HasFP16 ? 16 : 32)
762 unsigned ITySize = Query.
Types[0].getScalarSizeInBits();
763 return (ITySize == 16 || ITySize == 32 || ITySize == 64) &&
764 ITySize > Query.
Types[1].getScalarSizeInBits();
769 unsigned FTySize = Query.
Types[1].getScalarSizeInBits();
770 return (FTySize == 16 || FTySize == 32 || FTySize == 64) &&
771 Query.
Types[0].getScalarSizeInBits() < FTySize;
775 .clampNumElements(0, v4s16, v8s16)
776 .clampNumElements(0, v2s32, v4s32)
777 .clampMaxNumElements(0, s64, 2);
780 .legalFor({{s32, s32},
788 {{s16, s32}, {s16, s64}, {v4s16, v4s16}, {v8s16, v8s16}})
797 return Query.
Types[1].getScalarSizeInBits() <= 64 &&
798 Query.
Types[0].getScalarSizeInBits() <
799 Query.
Types[1].getScalarSizeInBits();
804 return Query.
Types[0].getScalarSizeInBits() <= 64 &&
805 Query.
Types[0].getScalarSizeInBits() >
806 Query.
Types[1].getScalarSizeInBits();
809 .clampNumElements(0, v4s16, v8s16)
822 .clampScalar(0, s32, s32);
826 .
legalFor({{s32, s32}, {s64, s32}, {p0, s32}})
827 .widenScalarToNextPow2(0)
846 .
legalFor({{s64, p0}, {v2s64, v2p0}})
847 .widenScalarToNextPow2(0, 64)
853 return Query.
Types[0].getSizeInBits() != Query.
Types[1].getSizeInBits();
855 .legalFor({{p0, s64}, {v2p0, v2s64}})
856 .clampMaxNumElements(1, s64, 2);
863 .legalForCartesianProduct({s64, v8s8, v4s16, v2s32})
864 .legalForCartesianProduct({s128, v16s8, v8s16, v4s32, v2s64, v2p0})
873 return Query.
Types[0].isVector() != Query.
Types[1].isVector();
876 .clampNumElements(0, v8s8, v16s8)
877 .clampNumElements(0, v4s16, v8s16)
878 .clampNumElements(0, v2s32, v4s32)
887 .clampScalar(0, s8, s64)
894 bool UseOutlineAtomics = ST.outlineAtomics() && !ST.hasLSE();
897 .
legalFor(!UseOutlineAtomics, {{s32, p0}, {s64, p0}})
898 .customFor(!UseOutlineAtomics, {{s128, p0}})
899 .libcallFor(UseOutlineAtomics,
900 {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}, {s128, p0}})
901 .clampScalar(0, s32, s64);
904 G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR,
906 .legalFor(!UseOutlineAtomics, {{s32, p0}, {s64, p0}})
907 .libcallFor(UseOutlineAtomics,
908 {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
909 .clampScalar(0, s32, s64);
914 {G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
921 for (
unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
922 unsigned BigTyIdx =
Op == G_MERGE_VALUES ? 0 : 1;
923 unsigned LitTyIdx =
Op == G_MERGE_VALUES ? 1 : 0;
930 switch (Q.
Types[BigTyIdx].getSizeInBits()) {
938 switch (Q.
Types[LitTyIdx].getSizeInBits()) {
952 .
legalFor(HasSVE, {{s16, nxv16s8, s64},
955 {s64, nxv2s64, s64}})
957 const LLT &EltTy = Query.
Types[1].getElementType();
958 if (Query.
Types[1].isScalableVector())
960 return Query.
Types[0] != EltTy;
965 return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 ||
966 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 ||
967 VecTy == v8s8 || VecTy == v16s8 || VecTy == v2p0;
973 return Query.
Types[1].isFixedVector() &&
974 Query.
Types[1].getNumElements() <= 2;
979 return Query.
Types[1].isFixedVector() &&
980 Query.
Types[1].getNumElements() <= 4;
985 return Query.
Types[1].isFixedVector() &&
986 Query.
Types[1].getNumElements() <= 8;
991 return Query.
Types[1].isFixedVector() &&
992 Query.
Types[1].getNumElements() <= 16;
995 .minScalarOrElt(0, s8)
1005 typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64, v2p0}))
1006 .
legalFor(HasSVE, {{nxv16s8, s32, s64},
1007 {nxv8s16, s32, s64},
1008 {nxv4s32, s32, s64},
1009 {nxv2s64, s64, s64}})
1011 .widenVectorEltsToVectorMinSize(0, 64)
1012 .clampNumElements(0, v8s8, v16s8)
1013 .clampNumElements(0, v4s16, v8s16)
1014 .clampNumElements(0, v2s32, v4s32)
1015 .clampMaxNumElements(0, s64, 2)
1016 .clampMaxNumElements(0, p0, 2);
1027 .clampNumElements(0, v4s32, v4s32)
1038 {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
1040 .widenScalarToNextPow2(1, 32)
1041 .clampScalar(1, s32, s64)
1042 .scalarSameSizeAs(0, 1);
1048 .widenScalarToNextPow2(0, 32)
1060 .customFor(!HasCSSC, {s32, s64});
1071 {v2s64, v2s32, v4s32, v4s16, v16s8, v8s8, v8s16}, DstTy);
1077 return !Query.
Types[0].isVector() || !Query.
Types[1].isVector();
1081 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
1082 Query.
Types[0].getNumElements() >
1083 Query.
Types[1].getNumElements();
1089 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
1090 Query.
Types[0].getNumElements() <
1091 Query.
Types[1].getNumElements();
1094 .widenScalarOrEltToNextPow2OrMinSize(0, 8)
1095 .clampNumElements(0, v8s8, v16s8)
1096 .clampNumElements(0, v4s16, v8s16)
1097 .clampNumElements(0, v4s32, v4s32)
1098 .clampNumElements(0, v2s64, v2s64)
1107 .
legalFor({{v4s32, v2s32}, {v8s16, v4s16}, {v16s8, v8s8}})
1110 return Query.
Types[0].getSizeInBits() <= 128 &&
1111 Query.
Types[1].getSizeInBits() <= 64;
1138 .customForCartesianProduct({p0}, {s8}, {s64})
1142 .legalForCartesianProduct({p0}, {p0}, {s64})
1157 .legalFor(PackedVectorAllTypeList)
1165 [=](
const LegalityQuery &Query) {
return std::make_pair(0, v4s16); })
1168 [=](
const LegalityQuery &Query) {
return std::make_pair(0, v2s32); })
1169 .clampNumElements(0, v8s8, v16s8)
1180 .
legalFor({{s32, v2s32}, {s32, v4s32}, {s64, v2s64}})
1181 .legalFor(HasFP16, {{s16, v4s16}, {s16, v8s16}})
1182 .minScalarOrElt(0, MinFPScalar)
1213 .clampMaxNumElements(1, s64, 2)
1220 G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
1221 .legalFor({{s32, v4s32}, {s32, v2s32}, {s64, v2s64}})
1222 .legalFor(HasFP16, {{s16, v4s16}, {s16, v8s16}})
1223 .minScalarOrElt(0, MinFPScalar)
1237 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX})
1238 .legalFor({{s8, v8s8},
1246 return Query.
Types[1].isVector() &&
1247 Query.
Types[1].getElementType() != s8 &&
1248 Query.
Types[1].getNumElements() & 1;
1251 .clampMaxNumElements(1, s64, 2)
1259 {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
1275 return std::make_pair(1, SrcTy.
divide(2));
1284 .customFor({{s32, s32}, {s32, s64}, {s64, s64}})
1288 .
legalFor({{s32, s64}, {s64, s64}})
1290 return Q.
Types[0].isScalar() && Q.
Types[1].getScalarSizeInBits() < 64;
1296 .customFor({{s32, s32}, {s64, s64}});
1300 .
legalFor(HasCSSC, {{s32, s32}, {s64, s64}})
1301 .legalFor({{v8s8, v8s8}, {v16s8, v16s8}})
1302 .customFor(!HasCSSC, {{s32, s32}, {s64, s64}})
1303 .customFor({{s128, s128},
1309 .clampScalar(0, s32, s128)
1315 .legalFor({v2s64, v2s32, v4s32, v4s16, v8s16, v8s8, v16s8})
1316 .legalFor(HasSVE, {nxv2s64, nxv4s32, nxv8s16, nxv16s8})
1317 .clampNumElements(0, v8s8, v16s8)
1328 .legalFor({{s64, s32}, {s64, s64}});
1344 G_GET_FPMODE, G_SET_FPMODE, G_RESET_FPMODE})
1354 .
legalFor({{v8s8, v16s8}, {v4s16, v8s16}, {v2s32, v4s32}})
1360 .
legalFor(HasSVE, {{nxv4s32, s32}, {nxv2s64, s64}});
1363 verify(*ST.getInstrInfo());
1372 switch (
MI.getOpcode()) {
1376 case TargetOpcode::G_VAARG:
1377 return legalizeVaArg(
MI,
MRI, MIRBuilder);
1378 case TargetOpcode::G_LOAD:
1379 case TargetOpcode::G_STORE:
1380 return legalizeLoadStore(
MI,
MRI, MIRBuilder, Observer);
1381 case TargetOpcode::G_SHL:
1382 case TargetOpcode::G_ASHR:
1383 case TargetOpcode::G_LSHR:
1384 return legalizeShlAshrLshr(
MI,
MRI, MIRBuilder, Observer);
1385 case TargetOpcode::G_GLOBAL_VALUE:
1386 return legalizeSmallCMGlobalValue(
MI,
MRI, MIRBuilder, Observer);
1387 case TargetOpcode::G_SBFX:
1388 case TargetOpcode::G_UBFX:
1389 return legalizeBitfieldExtract(
MI,
MRI, Helper);
1390 case TargetOpcode::G_FSHL:
1391 case TargetOpcode::G_FSHR:
1392 return legalizeFunnelShift(
MI,
MRI, MIRBuilder, Observer, Helper);
1393 case TargetOpcode::G_ROTR:
1394 return legalizeRotate(
MI,
MRI, Helper);
1395 case TargetOpcode::G_CTPOP:
1396 return legalizeCTPOP(
MI,
MRI, Helper);
1397 case TargetOpcode::G_ATOMIC_CMPXCHG:
1398 return legalizeAtomicCmpxchg128(
MI,
MRI, Helper);
1399 case TargetOpcode::G_CTTZ:
1400 return legalizeCTTZ(
MI, Helper);
1401 case TargetOpcode::G_BZERO:
1402 case TargetOpcode::G_MEMCPY:
1403 case TargetOpcode::G_MEMMOVE:
1404 case TargetOpcode::G_MEMSET:
1405 return legalizeMemOps(
MI, Helper);
1406 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1407 return legalizeExtractVectorElt(
MI,
MRI, Helper);
1408 case TargetOpcode::G_DYN_STACKALLOC:
1409 return legalizeDynStackAlloc(
MI, Helper);
1410 case TargetOpcode::G_PREFETCH:
1411 return legalizePrefetch(
MI, Helper);
1412 case TargetOpcode::G_ABS:
1414 case TargetOpcode::G_ICMP:
1415 return legalizeICMP(
MI,
MRI, MIRBuilder);
1416 case TargetOpcode::G_BITCAST:
1417 return legalizeBitcast(
MI, Helper);
1425 assert(
MI.getOpcode() == TargetOpcode::G_BITCAST &&
"Unexpected opcode");
1426 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
1429 if (!DstTy.isScalar() || !SrcTy.isVector() ||
1434 MI.eraseFromParent();
1443 assert(
MI.getOpcode() == TargetOpcode::G_FSHL ||
1444 MI.getOpcode() == TargetOpcode::G_FSHR);
1448 Register ShiftNo =
MI.getOperand(3).getReg();
1449 LLT ShiftTy =
MRI.getType(ShiftNo);
1454 LLT OperationTy =
MRI.getType(
MI.getOperand(0).getReg());
1458 if (!VRegAndVal || VRegAndVal->Value.urem(
BitWidth) == 0)
1464 Amount =
MI.getOpcode() == TargetOpcode::G_FSHL ?
BitWidth - Amount : Amount;
1468 if (ShiftTy.
getSizeInBits() == 64 &&
MI.getOpcode() == TargetOpcode::G_FSHR &&
1475 if (
MI.getOpcode() == TargetOpcode::G_FSHR) {
1477 MI.getOperand(3).setReg(Cast64.getReg(0));
1482 else if (
MI.getOpcode() == TargetOpcode::G_FSHL) {
1484 {
MI.getOperand(1).
getReg(),
MI.getOperand(2).getReg(),
1486 MI.eraseFromParent();
1495 Register SrcReg1 =
MI.getOperand(2).getReg();
1496 Register SrcReg2 =
MI.getOperand(3).getReg();
1497 LLT DstTy =
MRI.getType(DstReg);
1498 LLT SrcTy =
MRI.getType(SrcReg1);
1515 MIRBuilder.
buildNot(DstReg, CmpReg);
1517 MI.eraseFromParent();
1527 LLT AmtTy =
MRI.getType(AmtReg);
1533 MI.getOperand(2).setReg(NewAmt.getReg(0));
1538bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
1541 assert(
MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
1546 auto &GlobalOp =
MI.getOperand(1);
1548 if (GlobalOp.isSymbol())
1550 const auto* GV = GlobalOp.getGlobal();
1551 if (GV->isThreadLocal())
1560 auto Offset = GlobalOp.getOffset();
1565 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1582 "Should not have folded in an offset for a tagged global!");
1584 .addGlobalAddress(GV, 0x100000000,
1587 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1591 .addGlobalAddress(GV,
Offset,
1593 MI.eraseFromParent();
1599 auto LowerBinOp = [&
MI](
unsigned Opcode) {
1602 {
MI.getOperand(2),
MI.getOperand(3)});
1603 MI.eraseFromParent();
1608 switch (IntrinsicID) {
1609 case Intrinsic::vacopy: {
1611 unsigned VaListSize =
1623 VaListSize,
Align(PtrSize)));
1627 VaListSize,
Align(PtrSize)));
1628 MI.eraseFromParent();
1631 case Intrinsic::get_dynamic_area_offset: {
1634 MI.eraseFromParent();
1637 case Intrinsic::aarch64_mops_memset_tag: {
1638 assert(
MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1642 auto &
Value =
MI.getOperand(3);
1644 Value.setReg(ExtValueReg);
1647 case Intrinsic::aarch64_prefetch: {
1649 auto &AddrVal =
MI.getOperand(1);
1651 int64_t IsWrite =
MI.getOperand(2).getImm();
1652 int64_t
Target =
MI.getOperand(3).getImm();
1653 int64_t IsStream =
MI.getOperand(4).getImm();
1654 int64_t IsData =
MI.getOperand(5).getImm();
1656 unsigned PrfOp = (IsWrite << 4) |
1662 MI.eraseFromParent();
1665 case Intrinsic::aarch64_neon_uaddv:
1666 case Intrinsic::aarch64_neon_saddv:
1667 case Intrinsic::aarch64_neon_umaxv:
1668 case Intrinsic::aarch64_neon_smaxv:
1669 case Intrinsic::aarch64_neon_uminv:
1670 case Intrinsic::aarch64_neon_sminv: {
1673 bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
1674 IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
1675 IntrinsicID == Intrinsic::aarch64_neon_sminv;
1677 auto OldDst =
MI.getOperand(0).getReg();
1678 auto OldDstTy =
MRI.getType(OldDst);
1679 LLT NewDstTy =
MRI.getType(
MI.getOperand(2).getReg()).getElementType();
1680 if (OldDstTy == NewDstTy)
1683 auto NewDst =
MRI.createGenericVirtualRegister(NewDstTy);
1686 MI.getOperand(0).setReg(NewDst);
1690 MIB.
buildExtOrTrunc(IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
1695 case Intrinsic::aarch64_neon_uaddlp:
1696 case Intrinsic::aarch64_neon_saddlp: {
1699 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlp
1701 : AArch64::G_SADDLP;
1703 MI.eraseFromParent();
1707 case Intrinsic::aarch64_neon_uaddlv:
1708 case Intrinsic::aarch64_neon_saddlv: {
1712 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlv
1714 : AArch64::G_SADDLV;
1717 LLT DstTy =
MRI.getType(DstReg);
1741 MI.eraseFromParent();
1745 case Intrinsic::aarch64_neon_smax:
1746 return LowerBinOp(TargetOpcode::G_SMAX);
1747 case Intrinsic::aarch64_neon_smin:
1748 return LowerBinOp(TargetOpcode::G_SMIN);
1749 case Intrinsic::aarch64_neon_umax:
1750 return LowerBinOp(TargetOpcode::G_UMAX);
1751 case Intrinsic::aarch64_neon_umin:
1752 return LowerBinOp(TargetOpcode::G_UMIN);
1753 case Intrinsic::aarch64_neon_fmax:
1754 return LowerBinOp(TargetOpcode::G_FMAXIMUM);
1755 case Intrinsic::aarch64_neon_fmin:
1756 return LowerBinOp(TargetOpcode::G_FMINIMUM);
1757 case Intrinsic::aarch64_neon_fmaxnm:
1758 return LowerBinOp(TargetOpcode::G_FMAXNUM);
1759 case Intrinsic::aarch64_neon_fminnm:
1760 return LowerBinOp(TargetOpcode::G_FMINNUM);
1761 case Intrinsic::aarch64_neon_smull:
1762 return LowerBinOp(AArch64::G_SMULL);
1763 case Intrinsic::aarch64_neon_umull:
1764 return LowerBinOp(AArch64::G_UMULL);
1765 case Intrinsic::aarch64_neon_abs: {
1768 MIB.
buildInstr(TargetOpcode::G_ABS, {
MI.getOperand(0)}, {
MI.getOperand(2)});
1769 MI.eraseFromParent();
1773 case Intrinsic::vector_reverse:
1781bool AArch64LegalizerInfo::legalizeShlAshrLshr(
1784 assert(
MI.getOpcode() == TargetOpcode::G_ASHR ||
1785 MI.getOpcode() == TargetOpcode::G_LSHR ||
1786 MI.getOpcode() == TargetOpcode::G_SHL);
1799 MI.getOperand(2).setReg(ExtCst.getReg(0));
1812 isShiftedInt<7, 3>(NewOffset)) {
1820bool AArch64LegalizerInfo::legalizeLoadStore(
1823 assert(
MI.getOpcode() == TargetOpcode::G_STORE ||
1824 MI.getOpcode() == TargetOpcode::G_LOAD);
1835 const LLT ValTy =
MRI.getType(ValReg);
1840 bool IsLoad =
MI.getOpcode() == TargetOpcode::G_LOAD;
1844 ST->hasLSE2() && ST->hasRCPC3() && (IsLoadAcquire || IsStoreRelease);
1850 Opcode = IsLoad ? AArch64::LDIAPPX : AArch64::STILPX;
1856 assert(ST->hasLSE2() &&
"ldp/stp not single copy atomic without +lse2");
1858 Opcode = IsLoad ? AArch64::LDPXi : AArch64::STPXi;
1863 NewI = MIRBuilder.
buildInstr(Opcode, {s64, s64}, {});
1869 Opcode, {}, {
Split->getOperand(0),
Split->getOperand(1)});
1873 NewI.
addUse(
MI.getOperand(1).getReg());
1884 *
MRI.getTargetRegisterInfo(),
1886 MI.eraseFromParent();
1892 LLVM_DEBUG(
dbgs() <<
"Tried to do custom legalization on wrong load/store");
1898 auto &MMO = **
MI.memoperands_begin();
1901 if (
MI.getOpcode() == TargetOpcode::G_STORE) {
1905 auto NewLoad = MIRBuilder.
buildLoad(NewTy,
MI.getOperand(1), MMO);
1908 MI.eraseFromParent();
1916 Align Alignment(
MI.getOperand(2).getImm());
1918 Register ListPtr =
MI.getOperand(1).getReg();
1920 LLT PtrTy =
MRI.getType(ListPtr);
1931 if (Alignment > PtrAlign) {
1935 auto ListTmp = MIRBuilder.
buildPtrAdd(PtrTy,
List, AlignMinus1.getReg(0));
1940 LLT ValTy =
MRI.getType(Dst);
1945 ValTy, std::max(Alignment, PtrAlign)));
1956 MI.eraseFromParent();
1960bool AArch64LegalizerInfo::legalizeBitfieldExtract(
1994 LLT Ty =
MRI.getType(Val);
1998 "Expected src and dst to have the same type!");
2006 auto Add = MIRBuilder.
buildAdd(s64, CTPOP1, CTPOP2);
2009 MI.eraseFromParent();
2013 if (!ST->hasNEON() ||
2014 MI.getMF()->getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) {
2026 assert((
Size == 32 ||
Size == 64 ||
Size == 128) &&
"Expected only 32, 64, or 128 bit scalars!");
2050 Sum = MIRBuilder.
buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones,
CTPOP});
2052 Sum = MIRBuilder.
buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones,
CTPOP});
2058 MI.eraseFromParent();
2066 Opc = Intrinsic::aarch64_neon_uaddlv;
2069 Opc = Intrinsic::aarch64_neon_uaddlp;
2072 Opc = Intrinsic::aarch64_neon_uaddlp;
2076 Opc = Intrinsic::aarch64_neon_uaddlp;
2081 Opc = Intrinsic::aarch64_neon_uaddlp;
2084 Opc = Intrinsic::aarch64_neon_uaddlp;
2090 for (
LLT HTy : HAddTys) {
2100 MI.eraseFromParent();
2104bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
2108 auto Addr =
MI.getOperand(1).getReg();
2109 auto DesiredI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(2));
2110 auto NewI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(3));
2111 auto DstLo =
MRI.createGenericVirtualRegister(s64);
2112 auto DstHi =
MRI.createGenericVirtualRegister(s64);
2125 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
2129 Opcode = AArch64::CASPAX;
2132 Opcode = AArch64::CASPLX;
2136 Opcode = AArch64::CASPALX;
2139 Opcode = AArch64::CASPX;
2144 auto CASDst =
MRI.createGenericVirtualRegister(s128);
2145 auto CASDesired =
MRI.createGenericVirtualRegister(s128);
2146 auto CASNew =
MRI.createGenericVirtualRegister(s128);
2147 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASDesired}, {})
2148 .addUse(DesiredI->getOperand(0).getReg())
2150 .
addUse(DesiredI->getOperand(1).getReg())
2151 .
addImm(AArch64::subo64);
2152 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASNew}, {})
2156 .
addImm(AArch64::subo64);
2158 CAS = MIRBuilder.
buildInstr(Opcode, {CASDst}, {CASDesired, CASNew,
Addr});
2166 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
2170 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
2173 Opcode = AArch64::CMP_SWAP_128_RELEASE;
2177 Opcode = AArch64::CMP_SWAP_128;
2180 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
2184 auto Scratch =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
2185 CAS = MIRBuilder.
buildInstr(Opcode, {DstLo, DstHi, Scratch},
2186 {
Addr, DesiredI->getOperand(0),
2187 DesiredI->getOperand(1), NewI->
getOperand(0),
2193 *
MRI.getTargetRegisterInfo(),
2197 MI.eraseFromParent();
2205 LLT Ty =
MRI.getType(
MI.getOperand(1).getReg());
2207 MIRBuilder.
buildCTLZ(
MI.getOperand(0).getReg(), BitReverse);
2208 MI.eraseFromParent();
2217 if (
MI.getOpcode() == TargetOpcode::G_MEMSET) {
2220 auto &
Value =
MI.getOperand(1);
2223 Value.setReg(ExtValueReg);
2230bool AArch64LegalizerInfo::legalizeExtractVectorElt(
2244bool AArch64LegalizerInfo::legalizeDynStackAlloc(
2260 Register AllocSize =
MI.getOperand(1).getReg();
2264 "Unexpected type for dynamic alloca");
2266 "Unexpected type for dynamic alloca");
2268 LLT PtrTy =
MRI.getType(Dst);
2274 MIRBuilder.
buildInstr(AArch64::PROBED_STACKALLOC_DYN, {}, {SPTmp});
2275 MRI.setRegClass(NewMI.getReg(0), &AArch64::GPR64commonRegClass);
2276 MIRBuilder.
setInsertPt(*NewMI->getParent(), NewMI);
2279 MI.eraseFromParent();
2286 auto &AddrVal =
MI.getOperand(0);
2288 int64_t IsWrite =
MI.getOperand(1).getImm();
2289 int64_t Locality =
MI.getOperand(2).getImm();
2290 int64_t
IsData =
MI.getOperand(3).getImm();
2292 bool IsStream = Locality == 0;
2293 if (Locality != 0) {
2294 assert(Locality <= 3 &&
"Prefetch locality out-of-range");
2298 Locality = 3 - Locality;
2301 unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
2304 MI.eraseFromParent();
unsigned const MachineRegisterInfo * MRI
static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset, MachineRegisterInfo &MRI)
This file declares the targeting of the Machinelegalizer class for AArch64.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Interface for Targets to specify which operations they can successfully select and how the others sho...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static constexpr Register SPReg
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
AArch64LegalizerInfo(const AArch64Subtarget &ST)
bool isTargetWindows() const
const AArch64InstrInfo * getInstrInfo() const override
bool isTargetDarwin() const
bool isTargetILP32() const
const AArch64TargetLowering * getTargetLowering() const override
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
const RegisterBankInfo * getRegBankInfo() const override
Class for arbitrary precision integers.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
int64_t getSExtValue() const
Get sign extended value.
StringRef getValueAsString() const
Return the attribute's value as a string.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
This class represents an Operation in the Expression.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
constexpr bool isScalableVector() const
Returns true if the LLT is a scalable vector.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT scalable_vector(unsigned MinNumElements, unsigned ScalarSizeInBits)
Get a low-level scalable vector of some number of elements and element width.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
constexpr bool isPointerVector() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr LLT divide(int Factor) const
Return a type that is Factor times smaller.
void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & widenScalarOrEltToNextPow2OrMinSize(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar or vector element type to the next power of two that is at least MinSize.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & maxScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned SmallTypeIdx)
Conditionally narrow the scalar or elt to match the size of another.
LegalizeRuleSet & unsupported()
The instruction is unsupported.
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
LegalizeRuleSet & bitcastIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
The specified type index is coerced if predicate is true.
LegalizeRuleSet & libcallFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & maxScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at most as wide as Ty.
LegalizeRuleSet & minScalarOrElt(unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
LegalizeRuleSet & clampMinNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MinElements)
Limit the number of elements in EltTy vectors to at least MinElements.
LegalizeRuleSet & widenVectorEltsToVectorMinSize(unsigned TypeIdx, unsigned VectorSize)
Ensure the vector size is at least as wide as VectorSize by promoting the element.
LegalizeRuleSet & lowerIfMemSizeNotPow2()
Lower a memory operation if the memory size, rounded to bytes, is not a power of 2.
LegalizeRuleSet & minScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned LargeTypeIdx)
Conditionally widen the scalar or elt to match the size of another.
LegalizeRuleSet & customForCartesianProduct(std::initializer_list< LLT > Types)
LegalizeRuleSet & lowerIfMemSizeNotByteSizePow2()
Lower a memory operation if the memory access size is not a round power of 2 byte size.
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
LegalizeRuleSet & narrowScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Narrow the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & moreElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Add more elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & lowerFor(std::initializer_list< LLT > Types)
The instruction is lowered when type index 0 is any type in the given list.
LegalizeRuleSet & scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx)
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx)
Widen the scalar to match the size of another.
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
LegalizeRuleSet & minScalarOrEltIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & legalForTypesWithMemDesc(std::initializer_list< LegalityPredicates::TypePairAndMemDesc > TypesAndMemDesc)
The instruction is legal when type indexes 0 and 1 along with the memory size and minimum alignment i...
LegalizeRuleSet & widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar or vector element type to the next power of two that is at least MinSize.
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeResult lowerDynStackAlloc(MachineInstr &MI)
LegalizeResult lowerBitCount(MachineInstr &MI)
LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI)
Lower a vector extract or insert by writing the vector to a stack temporary and reloading the element...
LegalizeResult lowerAbsToCNeg(MachineInstr &MI)
const TargetLowering & getTargetLowering() const
LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI)
MachineInstrBuilder createStackStoreLoad(const DstOp &Res, const SrcOp &Val)
Create a store of Val to a stack temporary and return a load as the same type as Res.
@ Legalized
Instruction has been legalized and the MachineFunction changed.
@ UnableToLegalize
Some kind of error has occurred and we could not legalize this instruction.
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
Register getDynStackAllocTargetPtr(Register SPReg, Register AllocSize, Align Alignment, LLT PtrTy)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
const LegacyLegalizerInfo & getLegacyLegalizerInfo() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ... = G_EXTRACT Src, Idx0.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ZEXT Op.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildBitReverse(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITREVERSE Src.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op)
Build and insert Res = ExtOpc, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes of...
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_TRUNC Op.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Wrapper class representing virtual and physical registers.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
const TargetMachine & getTargetMachine() const
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
Primary interface to the complete machine description for the target machine.
Target - Wrapper for Target specific information.
LLVM Value Representation.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar or a vector with an element type that's wider than the ...
LegalityPredicate isPointerVector(unsigned TypeIdx)
True iff the specified type index is a vector of pointers (with any address space).
LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, AtomicOrdering Ordering)
True iff the specified MMO index has at an atomic ordering of at Ordering or stronger.
Predicate any(Predicate P0, Predicate P1)
True iff P0 or P1 are true.
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar that's wider than the given size.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min=0)
Add more elements to the type for the given type index to the next power of.
LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx)
Change the scalar size or element size to have the same scalar size as type index FromIndex.
operand_type_match m_Reg()
ConstantMatch< APInt > m_ICst(APInt &Cst)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
This is an optimization pass for GlobalISel generic memory operations.
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
AtomicOrdering
Atomic ordering for LLVM's memory model.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
unsigned Log2(Align A)
Returns the log2 of the alignment.
std::function< bool(const LegalityQuery &)> LegalityPredicate
This struct is a compact representation of a valid (non-zero power of two) alignment.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
This class contains a discriminated union of information about pointers in memory operands,...