30#include "llvm/IR/IntrinsicsAArch64.h"
33#include <initializer_list>
35#define DEBUG_TYPE "aarch64-legalinfo"
38using namespace LegalizeActions;
39using namespace LegalizeMutations;
40using namespace LegalityPredicates;
41using namespace MIPatternMatch;
45 using namespace TargetOpcode;
69 std::initializer_list<LLT> PackedVectorAllTypeList = {
75 std::initializer_list<LLT> ScalarAndPtrTypesList = {s8, s16, s32, s64, p0};
79 const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
82 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
89 const bool HasFP16 = ST.hasFullFP16();
90 const LLT &MinFPScalar = HasFP16 ? s16 : s32;
92 const bool HasCSSC = ST.hasCSSC();
93 const bool HasRCPC3 = ST.hasRCPC3();
96 {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
97 .legalFor({p0, s8, s16, s32, s64})
98 .legalFor(PackedVectorAllTypeList)
110 .legalFor(PackedVectorAllTypeList)
121 .
legalFor({s32, s64, v4s16, v8s16, v2s32, v4s32, v2s64})
123 .clampScalar(0, s32, s64)
124 .clampNumElements(0, v4s16, v8s16)
125 .clampNumElements(0, v2s32, v4s32)
126 .clampNumElements(0, v2s64, v2s64)
127 .moreElementsToNextPow2(0);
130 .legalFor({s32, s64, v2s32, v2s64, v4s32, v4s16, v8s16, v16s8, v8s8})
131 .widenScalarToNextPow2(0)
139 return Query.
Types[0].getNumElements() <= 2;
144 return Query.
Types[0].getNumElements() <= 4;
149 return Query.
Types[0].getNumElements() <= 16;
156 const auto &SrcTy = Query.
Types[0];
157 const auto &AmtTy = Query.
Types[1];
158 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
159 AmtTy.getSizeInBits() == 32;
173 .widenScalarToNextPow2(0)
184 .
legalFor({{p0, s64}, {v2p0, v2s64}})
185 .clampScalarOrElt(1, s64, s64)
191 .legalFor({s32, s64})
193 .clampScalar(0, s32, s64)
198 .lowerFor({s8, s16, s32, s64, v2s64, v4s32, v2s32})
200 .clampScalarOrElt(0, s32, s64)
201 .clampNumElements(0, v2s32, v4s32)
202 .clampNumElements(0, v2s64, v2s64)
203 .moreElementsToNextPow2(0);
207 .widenScalarToNextPow2(0, 32)
212 .legalFor({s64, v8s16, v16s8, v4s32})
216 {G_SMIN, G_SMAX, G_UMIN, G_UMAX});
219 .
legalFor({s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
226 .
legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32});
237 {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
238 .legalFor({{s32, s32}, {s64, s32}})
239 .clampScalar(0, s32, s64)
244 G_FABS, G_FSQRT, G_FMAXNUM, G_FMINNUM,
245 G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR,
246 G_FRINT, G_FNEARBYINT, G_INTRINSIC_TRUNC,
247 G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
248 .legalFor({MinFPScalar, s32, s64, v2s32, v4s32, v2s64})
250 const auto &Ty = Query.
Types[0];
251 return (Ty == v8s16 || Ty == v4s16) && HasFP16;
254 .minScalarOrElt(0, MinFPScalar)
266 .legalFor({{s64, MinFPScalar}, {s64, s32}, {s64, s64}})
267 .libcallFor({{s64, s128}})
268 .minScalarOrElt(1, MinFPScalar);
271 {G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, G_FLOG10, G_FTAN, G_FEXP,
272 G_FEXP2, G_FEXP10, G_FACOS, G_FASIN, G_FATAN, G_FCOSH, G_FSINH, G_FTANH})
277 .libcallFor({s32, s64});
305 for (
unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
308 if (
Op == G_SEXTLOAD)
313 .legalForTypesWithMemDesc({{s32, p0, s8, 8},
321 {v2s32, p0, s64, 8}})
322 .widenScalarToNextPow2(0)
323 .clampScalar(0, s32, s64)
326 .unsupportedIfMemSizeNotPow2()
340 LoadActions.legalForTypesWithMemDesc({
342 {nxv16s8, p0, nxv16s8, 8},
343 {nxv8s16, p0, nxv8s16, 8},
344 {nxv4s32, p0, nxv4s32, 8},
345 {nxv2s64, p0, nxv2s64, 8},
351 StoreActions.legalForTypesWithMemDesc({
353 {nxv16s8, p0, nxv16s8, 8},
354 {nxv8s16, p0, nxv8s16, 8},
355 {nxv4s32, p0, nxv4s32, 8},
356 {nxv2s64, p0, nxv2s64, 8},
362 return HasRCPC3 && Query.
Types[0] == s128 &&
366 return Query.
Types[0] == s128 &&
369 .legalForTypesWithMemDesc({{s8, p0, s8, 8},
376 {v16s8, p0, s128, 8},
378 {v8s16, p0, s128, 8},
380 {v4s32, p0, s128, 8},
381 {v2s64, p0, s128, 8}})
383 .legalForTypesWithMemDesc(
384 {{s32, p0, s8, 8}, {s32, p0, s16, 8}, {s64, p0, s32, 8}})
385 .widenScalarToNextPow2(0, 8)
386 .clampMaxNumElements(0, s8, 16)
387 .clampMaxNumElements(0, s16, 8)
388 .clampMaxNumElements(0, s32, 4)
389 .clampMaxNumElements(0, s64, 2)
390 .clampMaxNumElements(0, p0, 2)
391 .lowerIfMemSizeNotByteSizePow2()
392 .clampScalar(0, s8, s64)
396 return Query.
Types[0].isScalar() &&
398 Query.
Types[0].getSizeInBits() > 32;
407 .customIf(IsPtrVecPred)
408 .scalarizeIf(
typeInSet(0, {v2s16, v2s8}), 0);
412 return HasRCPC3 && Query.
Types[0] == s128 &&
416 return Query.
Types[0] == s128 &&
419 .legalForTypesWithMemDesc(
420 {{s8, p0, s8, 8}, {s16, p0, s8, 8},
423 {s16, p0, s16, 8}, {s32, p0, s16, 8},
425 {s32, p0, s8, 8}, {s32, p0, s16, 8}, {s32, p0, s32, 8},
426 {s64, p0, s64, 8}, {s64, p0, s32, 8},
427 {p0, p0, s64, 8}, {s128, p0, s128, 8}, {v16s8, p0, s128, 8},
428 {v8s8, p0, s64, 8}, {v4s16, p0, s64, 8}, {v8s16, p0, s128, 8},
429 {v2s32, p0, s64, 8}, {v4s32, p0, s128, 8}, {v2s64, p0, s128, 8}})
430 .clampScalar(0, s8, s64)
432 return Query.
Types[0].isScalar() &&
436 .clampMaxNumElements(0, s8, 16)
437 .clampMaxNumElements(0, s16, 8)
438 .clampMaxNumElements(0, s32, 4)
439 .clampMaxNumElements(0, s64, 2)
440 .clampMaxNumElements(0, p0, 2)
441 .lowerIfMemSizeNotPow2()
448 .customIf(IsPtrVecPred)
449 .scalarizeIf(
typeInSet(0, {v2s16, v2s8}), 0);
464 {p0, v16s8, v16s8, 8},
465 {p0, v4s16, v4s16, 8},
466 {p0, v8s16, v8s16, 8},
467 {p0, v2s32, v2s32, 8},
468 {p0, v4s32, v4s32, 8},
469 {p0, v2s64, v2s64, 8},
475 auto IndexedLoadBasicPred = [=](
const LegalityQuery &Query) {
503 return MemTy == s8 || MemTy == s16;
505 return MemTy == s8 || MemTy == s16 || MemTy == s32;
513 .widenScalarToNextPow2(0)
517 const auto &Ty = Query.
Types[0];
518 if (HasFP16 && Ty == s16)
520 return Ty == s32 || Ty == s64 || Ty == s128;
522 .clampScalar(0, MinFPScalar, s128);
526 .
legalFor({{s32, s32}, {s32, s64}, {s32, p0}})
528 .clampScalar(1, s32, s64)
529 .clampScalar(0, s32, s32)
530 .minScalarEltSameAsIf(
545 .clampNumElements(1, v8s8, v16s8)
546 .clampNumElements(1, v4s16, v8s16)
547 .clampNumElements(1, v2s32, v4s32)
548 .clampNumElements(1, v2s64, v2s64)
559 const auto &Ty = Query.
Types[1];
560 return (Ty == v8s16 || Ty == v4s16) && Ty == Query.
Types[0] && HasFP16;
563 .clampScalar(0, s32, s32)
564 .minScalarOrElt(1, MinFPScalar)
566 .minScalarEltSameAsIf(
574 .clampNumElements(1, v4s16, v8s16)
575 .clampNumElements(1, v2s32, v4s32)
576 .clampMaxNumElements(1, s64, 2)
577 .moreElementsToNextPow2(1)
578 .libcallFor({{s32, s128}});
582 unsigned DstSize = Query.
Types[0].getSizeInBits();
585 if (Query.
Types[0].isVector())
588 if (DstSize < 8 || DstSize >= 128 || !
isPowerOf2_32(DstSize))
603 .legalIf(ExtLegalFunc)
604 .
legalFor({{v2s64, v2s32}, {v4s32, v4s16}, {v8s16, v8s8}})
605 .clampScalar(0, s64, s64)
612 return (Query.
Types[0].getScalarSizeInBits() >
613 Query.
Types[1].getScalarSizeInBits() * 2) &&
614 Query.
Types[0].isVector() &&
615 (Query.
Types[1].getScalarSizeInBits() == 8 ||
616 Query.
Types[1].getScalarSizeInBits() == 16);
618 .clampMinNumElements(1, s8, 8)
622 .
legalFor({{v2s32, v2s64}, {v4s16, v4s32}, {v8s8, v8s16}})
624 .clampMaxNumElements(0, s8, 8)
625 .clampMaxNumElements(0, s16, 4)
626 .clampMaxNumElements(0, s32, 2)
636 .clampMinNumElements(0, s8, 8)
637 .clampMinNumElements(0, s16, 4)
642 .legalFor(PackedVectorAllTypeList)
653 {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
654 .libcallFor({{s16, s128}, {s32, s128}, {s64, s128}})
655 .clampNumElements(0, v4s16, v4s16)
661 {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
662 .libcallFor({{s128, s64}, {s128, s32}, {s128, s16}})
663 .clampNumElements(0, v4s32, v4s32)
669 .legalFor({{s32, s32},
678 (Query.
Types[1] == s16 || Query.
Types[1] == v4s16 ||
679 Query.
Types[1] == v8s16) &&
680 (Query.
Types[0] == s32 || Query.
Types[0] == s64 ||
681 Query.
Types[0] == v4s16 || Query.
Types[0] == v8s16);
689 return Query.
Types[1] == s16 && Query.
Types[0].getSizeInBits() > 64;
693 .widenScalarOrEltToNextPow2OrMinSize(0)
695 .widenScalarOrEltToNextPow2OrMinSize(1, HasFP16 ? 16 : 32)
698 return Query.
Types[0].getScalarSizeInBits() <= 64 &&
699 Query.
Types[0].getScalarSizeInBits() >
700 Query.
Types[1].getScalarSizeInBits();
705 return Query.
Types[1].getScalarSizeInBits() <= 64 &&
706 Query.
Types[0].getScalarSizeInBits() <
707 Query.
Types[1].getScalarSizeInBits();
710 .clampNumElements(0, v4s16, v8s16)
711 .clampNumElements(0, v2s32, v4s32)
712 .clampMaxNumElements(0, s64, 2)
714 {{s32, s128}, {s64, s128}, {s128, s128}, {s128, s32}, {s128, s64}});
717 .legalFor({{s32, s32},
726 (Query.
Types[0] == s16 || Query.
Types[0] == v4s16 ||
727 Query.
Types[0] == v8s16) &&
728 (Query.
Types[1] == s32 || Query.
Types[1] == s64 ||
729 Query.
Types[1] == v4s16 || Query.
Types[1] == v8s16);
734 .widenScalarOrEltToNextPow2OrMinSize(1)
736 .widenScalarOrEltToNextPow2OrMinSize(0, HasFP16 ? 16 : 32)
739 return Query.
Types[1].getScalarSizeInBits() <= 64 &&
740 Query.
Types[0].getScalarSizeInBits() <
741 Query.
Types[1].getScalarSizeInBits();
746 return Query.
Types[0].getScalarSizeInBits() <= 64 &&
747 Query.
Types[0].getScalarSizeInBits() >
748 Query.
Types[1].getScalarSizeInBits();
751 .clampNumElements(0, v4s16, v8s16)
752 .clampNumElements(0, v2s32, v4s32)
753 .clampMaxNumElements(0, s64, 2)
754 .libcallFor({{s16, s128},
764 .clampScalar(0, s32, s32);
768 .
legalFor({{s32, s32}, {s64, s32}, {p0, s32}})
769 .widenScalarToNextPow2(0)
787 .
legalFor({{s64, p0}, {v2s64, v2p0}})
788 .widenScalarToNextPow2(0, 64)
793 return Query.
Types[0].getSizeInBits() != Query.
Types[1].getSizeInBits();
795 .legalFor({{p0, s64}, {v2p0, v2s64}});
802 .legalForCartesianProduct({s64, v8s8, v4s16, v2s32})
803 .legalForCartesianProduct({s128, v16s8, v8s16, v4s32, v2s64, v2p0})
805 return Query.
Types[0].isVector() != Query.
Types[1].isVector();
808 .clampNumElements(0, v8s8, v16s8)
809 .clampNumElements(0, v4s16, v8s16)
810 .clampNumElements(0, v2s32, v4s32)
819 .clampScalar(0, s8, s64)
827 return ST.outlineAtomics() && !ST.hasLSE();
835 return Query.
Types[0].getSizeInBits() == 128 &&
836 !UseOutlineAtomics(Query);
843 G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR,
854 {G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
861 for (
unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
862 unsigned BigTyIdx =
Op == G_MERGE_VALUES ? 0 : 1;
863 unsigned LitTyIdx =
Op == G_MERGE_VALUES ? 1 : 0;
870 switch (Q.
Types[BigTyIdx].getSizeInBits()) {
878 switch (Q.
Types[LitTyIdx].getSizeInBits()) {
892 const LLT &EltTy = Query.
Types[1].getElementType();
893 return Query.
Types[0] != EltTy;
898 return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 ||
899 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 ||
900 VecTy == v8s8 || VecTy == v16s8 || VecTy == v2p0;
906 return Query.
Types[1].getNumElements() <= 2;
911 return Query.
Types[1].getNumElements() <= 4;
916 return Query.
Types[1].getNumElements() <= 8;
921 return Query.
Types[1].getNumElements() <= 16;
924 .minScalarOrElt(0, s8)
934 typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64, v2p0}))
952 .clampNumElements(0, v4s32, v4s32)
963 {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
965 .widenScalarToNextPow2(1, 32)
966 .clampScalar(1, s32, s64)
967 .scalarSameSizeAs(0, 1);
973 .widenScalarToNextPow2(0, 32)
984 return (HasCSSC &&
typeInSet(0, {s32, s64})(Query));
987 return (!HasCSSC &&
typeInSet(0, {s32, s64})(Query));
999 {v2s64, v2p0, v2s32, v4s32, v4s16, v16s8, v8s8, v8s16}, DstTy);
1004 return !Query.
Types[1].isVector();
1008 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
1009 Query.
Types[0].getNumElements() >
1010 Query.
Types[1].getNumElements();
1016 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
1017 Query.
Types[0].getNumElements() <
1018 Query.
Types[1].getNumElements();
1021 .widenScalarOrEltToNextPow2OrMinSize(0, 8)
1022 .clampNumElements(0, v8s8, v16s8)
1023 .clampNumElements(0, v4s16, v8s16)
1024 .clampNumElements(0, v4s32, v4s32)
1025 .clampNumElements(0, v2s64, v2s64);
1028 .
legalFor({{v4s32, v2s32}, {v8s16, v4s16}, {v16s8, v8s8}})
1031 return Query.
Types[0].getSizeInBits() <= 128 &&
1032 Query.
Types[1].getSizeInBits() <= 64;
1059 .customForCartesianProduct({p0}, {s8}, {s64})
1063 .legalForCartesianProduct({p0}, {p0}, {s64})
1079 .legalFor({s32, s64});
1080 ABSActions.legalFor(PackedVectorAllTypeList)
1088 [=](
const LegalityQuery &Query) {
return std::make_pair(0, v4s16); })
1091 [=](
const LegalityQuery &Query) {
return std::make_pair(0, v2s32); })
1092 .clampNumElements(0, v8s8, v16s8)
1093 .clampNumElements(0, v4s16, v8s16)
1094 .clampNumElements(0, v2s32, v4s32)
1095 .clampNumElements(0, v2s64, v2s64)
1096 .moreElementsToNextPow2(0)
1103 .
legalFor({{s32, v2s32}, {s32, v4s32}, {s64, v2s64}})
1105 const auto &Ty = Query.
Types[1];
1106 return (Ty == v4s16 || Ty == v8s16) && HasFP16;
1108 .minScalarOrElt(0, MinFPScalar)
1139 .clampMaxNumElements(1, s64, 2)
1146 G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
1147 .legalFor({{s32, v4s32}, {s32, v2s32}, {s64, v2s64}})
1149 const auto &Ty = Query.
Types[1];
1150 return Query.
Types[0] == s16 && (Ty == v8s16 || Ty == v4s16) && HasFP16;
1152 .minScalarOrElt(0, MinFPScalar)
1166 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX})
1167 .legalFor({{s8, v8s8},
1175 return Query.
Types[1].isVector() &&
1176 Query.
Types[1].getElementType() != s8 &&
1177 Query.
Types[1].getNumElements() & 1;
1180 .clampMaxNumElements(1, s64, 2)
1188 {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
1204 return std::make_pair(1, SrcTy.
divide(2));
1213 .customFor({{s32, s32}, {s32, s64}, {s64, s64}})
1217 .
legalFor({{s32, s64}, {s64, s64}})
1219 return Q.
Types[0].isScalar() && Q.
Types[1].getScalarSizeInBits() < 64;
1225 .customFor({{s32, s32}, {s64, s64}});
1231 .legalFor({{s32, s32},
1235 .customFor({{s128, s128},
1243 .legalFor({{v8s8, v8s8},
1245 .customFor({{s32, s32},
1254 .clampScalar(0, s32, s128)
1255 .widenScalarToNextPow2(0)
1256 .minScalarEltSameAsIf(always, 1, 0)
1257 .maxScalarEltSameAsIf(always, 1, 0);
1260 .legalFor({v2s64, v2s32, v4s32, v4s16, v8s16, v8s8, v16s8})
1261 .clampNumElements(0, v8s8, v16s8)
1271 .legalFor({{s64, s32}, {s64, s64}});
1287 G_GET_FPMODE, G_SET_FPMODE, G_RESET_FPMODE})
1297 verify(*ST.getInstrInfo());
1306 switch (
MI.getOpcode()) {
1310 case TargetOpcode::G_VAARG:
1311 return legalizeVaArg(
MI,
MRI, MIRBuilder);
1312 case TargetOpcode::G_LOAD:
1313 case TargetOpcode::G_STORE:
1314 return legalizeLoadStore(
MI,
MRI, MIRBuilder, Observer);
1315 case TargetOpcode::G_SHL:
1316 case TargetOpcode::G_ASHR:
1317 case TargetOpcode::G_LSHR:
1318 return legalizeShlAshrLshr(
MI,
MRI, MIRBuilder, Observer);
1319 case TargetOpcode::G_GLOBAL_VALUE:
1320 return legalizeSmallCMGlobalValue(
MI,
MRI, MIRBuilder, Observer);
1321 case TargetOpcode::G_SBFX:
1322 case TargetOpcode::G_UBFX:
1323 return legalizeBitfieldExtract(
MI,
MRI, Helper);
1324 case TargetOpcode::G_FSHL:
1325 case TargetOpcode::G_FSHR:
1326 return legalizeFunnelShift(
MI,
MRI, MIRBuilder, Observer, Helper);
1327 case TargetOpcode::G_ROTR:
1328 return legalizeRotate(
MI,
MRI, Helper);
1329 case TargetOpcode::G_CTPOP:
1330 return legalizeCTPOP(
MI,
MRI, Helper);
1331 case TargetOpcode::G_ATOMIC_CMPXCHG:
1332 return legalizeAtomicCmpxchg128(
MI,
MRI, Helper);
1333 case TargetOpcode::G_CTTZ:
1334 return legalizeCTTZ(
MI, Helper);
1335 case TargetOpcode::G_BZERO:
1336 case TargetOpcode::G_MEMCPY:
1337 case TargetOpcode::G_MEMMOVE:
1338 case TargetOpcode::G_MEMSET:
1339 return legalizeMemOps(
MI, Helper);
1340 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1341 return legalizeExtractVectorElt(
MI,
MRI, Helper);
1342 case TargetOpcode::G_DYN_STACKALLOC:
1343 return legalizeDynStackAlloc(
MI, Helper);
1344 case TargetOpcode::G_PREFETCH:
1345 return legalizePrefetch(
MI, Helper);
1346 case TargetOpcode::G_ABS:
1348 case TargetOpcode::G_ICMP:
1349 return legalizeICMP(
MI,
MRI, MIRBuilder);
1360 assert(
MI.getOpcode() == TargetOpcode::G_FSHL ||
1361 MI.getOpcode() == TargetOpcode::G_FSHR);
1365 Register ShiftNo =
MI.getOperand(3).getReg();
1366 LLT ShiftTy =
MRI.getType(ShiftNo);
1371 LLT OperationTy =
MRI.getType(
MI.getOperand(0).getReg());
1375 if (!VRegAndVal || VRegAndVal->Value.urem(
BitWidth) == 0)
1381 Amount =
MI.getOpcode() == TargetOpcode::G_FSHL ?
BitWidth - Amount : Amount;
1385 if (ShiftTy.
getSizeInBits() == 64 &&
MI.getOpcode() == TargetOpcode::G_FSHR &&
1392 if (
MI.getOpcode() == TargetOpcode::G_FSHR) {
1394 MI.getOperand(3).setReg(Cast64.getReg(0));
1399 else if (
MI.getOpcode() == TargetOpcode::G_FSHL) {
1401 {
MI.getOperand(1).
getReg(),
MI.getOperand(2).getReg(),
1403 MI.eraseFromParent();
1412 Register SrcReg1 =
MI.getOperand(2).getReg();
1413 Register SrcReg2 =
MI.getOperand(3).getReg();
1414 LLT DstTy =
MRI.getType(DstReg);
1415 LLT SrcTy =
MRI.getType(SrcReg1);
1432 MIRBuilder.
buildNot(DstReg, CmpReg);
1434 MI.eraseFromParent();
1444 LLT AmtTy =
MRI.getType(AmtReg);
1450 MI.getOperand(2).setReg(NewAmt.getReg(0));
1455bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
1458 assert(
MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
1463 auto &GlobalOp =
MI.getOperand(1);
1465 if (GlobalOp.isSymbol())
1467 const auto* GV = GlobalOp.getGlobal();
1468 if (GV->isThreadLocal())
1477 auto Offset = GlobalOp.getOffset();
1482 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1499 "Should not have folded in an offset for a tagged global!");
1501 .addGlobalAddress(GV, 0x100000000,
1504 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1508 .addGlobalAddress(GV,
Offset,
1510 MI.eraseFromParent();
1517 switch (IntrinsicID) {
1518 case Intrinsic::vacopy: {
1520 unsigned VaListSize =
1532 VaListSize,
Align(PtrSize)));
1536 VaListSize,
Align(PtrSize)));
1537 MI.eraseFromParent();
1540 case Intrinsic::get_dynamic_area_offset: {
1543 MI.eraseFromParent();
1546 case Intrinsic::aarch64_mops_memset_tag: {
1547 assert(
MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1551 auto &
Value =
MI.getOperand(3);
1553 Value.setReg(ExtValueReg);
1556 case Intrinsic::aarch64_prefetch: {
1558 auto &AddrVal =
MI.getOperand(1);
1560 int64_t IsWrite =
MI.getOperand(2).getImm();
1561 int64_t
Target =
MI.getOperand(3).getImm();
1562 int64_t IsStream =
MI.getOperand(4).getImm();
1563 int64_t IsData =
MI.getOperand(5).getImm();
1565 unsigned PrfOp = (IsWrite << 4) |
1571 MI.eraseFromParent();
1574 case Intrinsic::aarch64_neon_uaddv:
1575 case Intrinsic::aarch64_neon_saddv:
1576 case Intrinsic::aarch64_neon_umaxv:
1577 case Intrinsic::aarch64_neon_smaxv:
1578 case Intrinsic::aarch64_neon_uminv:
1579 case Intrinsic::aarch64_neon_sminv: {
1582 bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
1583 IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
1584 IntrinsicID == Intrinsic::aarch64_neon_sminv;
1586 auto OldDst =
MI.getOperand(0).getReg();
1587 auto OldDstTy =
MRI.getType(OldDst);
1588 LLT NewDstTy =
MRI.getType(
MI.getOperand(2).getReg()).getElementType();
1589 if (OldDstTy == NewDstTy)
1592 auto NewDst =
MRI.createGenericVirtualRegister(NewDstTy);
1595 MI.getOperand(0).setReg(NewDst);
1599 MIB.
buildExtOrTrunc(IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
1604 case Intrinsic::aarch64_neon_uaddlp:
1605 case Intrinsic::aarch64_neon_saddlp: {
1608 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlp
1610 : AArch64::G_SADDLP;
1612 MI.eraseFromParent();
1616 case Intrinsic::aarch64_neon_uaddlv:
1617 case Intrinsic::aarch64_neon_saddlv: {
1621 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlv
1623 : AArch64::G_SADDLV;
1626 LLT DstTy =
MRI.getType(DstReg);
1650 MI.eraseFromParent();
1654 case Intrinsic::aarch64_neon_smax:
1655 case Intrinsic::aarch64_neon_smin:
1656 case Intrinsic::aarch64_neon_umax:
1657 case Intrinsic::aarch64_neon_umin:
1658 case Intrinsic::aarch64_neon_fmax:
1659 case Intrinsic::aarch64_neon_fmin:
1660 case Intrinsic::aarch64_neon_fmaxnm:
1661 case Intrinsic::aarch64_neon_fminnm: {
1663 if (IntrinsicID == Intrinsic::aarch64_neon_smax)
1665 else if (IntrinsicID == Intrinsic::aarch64_neon_smin)
1667 else if (IntrinsicID == Intrinsic::aarch64_neon_umax)
1669 else if (IntrinsicID == Intrinsic::aarch64_neon_umin)
1671 else if (IntrinsicID == Intrinsic::aarch64_neon_fmax)
1672 MIB.
buildInstr(TargetOpcode::G_FMAXIMUM, {
MI.getOperand(0)},
1673 {
MI.getOperand(2),
MI.getOperand(3)});
1674 else if (IntrinsicID == Intrinsic::aarch64_neon_fmin)
1675 MIB.
buildInstr(TargetOpcode::G_FMINIMUM, {
MI.getOperand(0)},
1676 {
MI.getOperand(2),
MI.getOperand(3)});
1677 else if (IntrinsicID == Intrinsic::aarch64_neon_fmaxnm)
1678 MIB.
buildInstr(TargetOpcode::G_FMAXNUM, {
MI.getOperand(0)},
1679 {
MI.getOperand(2),
MI.getOperand(3)});
1680 else if (IntrinsicID == Intrinsic::aarch64_neon_fminnm)
1681 MIB.
buildInstr(TargetOpcode::G_FMINNUM, {
MI.getOperand(0)},
1682 {
MI.getOperand(2),
MI.getOperand(3)});
1683 MI.eraseFromParent();
1686 case Intrinsic::vector_reverse:
1694bool AArch64LegalizerInfo::legalizeShlAshrLshr(
1697 assert(
MI.getOpcode() == TargetOpcode::G_ASHR ||
1698 MI.getOpcode() == TargetOpcode::G_LSHR ||
1699 MI.getOpcode() == TargetOpcode::G_SHL);
1712 MI.getOperand(2).setReg(ExtCst.getReg(0));
1725 isShiftedInt<7, 3>(NewOffset)) {
1733bool AArch64LegalizerInfo::legalizeLoadStore(
1736 assert(
MI.getOpcode() == TargetOpcode::G_STORE ||
1737 MI.getOpcode() == TargetOpcode::G_LOAD);
1748 const LLT ValTy =
MRI.getType(ValReg);
1753 bool IsLoad =
MI.getOpcode() == TargetOpcode::G_LOAD;
1757 ST->hasLSE2() && ST->hasRCPC3() && (IsLoadAcquire || IsStoreRelease);
1763 Opcode = IsLoad ? AArch64::LDIAPPX : AArch64::STILPX;
1769 assert(ST->hasLSE2() &&
"ldp/stp not single copy atomic without +lse2");
1771 Opcode = IsLoad ? AArch64::LDPXi : AArch64::STPXi;
1776 NewI = MIRBuilder.
buildInstr(Opcode, {s64, s64}, {});
1782 Opcode, {}, {
Split->getOperand(0),
Split->getOperand(1)});
1786 NewI.
addUse(
MI.getOperand(1).getReg());
1797 *
MRI.getTargetRegisterInfo(),
1799 MI.eraseFromParent();
1805 LLVM_DEBUG(
dbgs() <<
"Tried to do custom legalization on wrong load/store");
1811 auto &MMO = **
MI.memoperands_begin();
1814 if (
MI.getOpcode() == TargetOpcode::G_STORE) {
1818 auto NewLoad = MIRBuilder.
buildLoad(NewTy,
MI.getOperand(1), MMO);
1821 MI.eraseFromParent();
1829 Align Alignment(
MI.getOperand(2).getImm());
1831 Register ListPtr =
MI.getOperand(1).getReg();
1833 LLT PtrTy =
MRI.getType(ListPtr);
1844 if (Alignment > PtrAlign) {
1848 auto ListTmp = MIRBuilder.
buildPtrAdd(PtrTy,
List, AlignMinus1.getReg(0));
1853 LLT ValTy =
MRI.getType(Dst);
1858 ValTy, std::max(Alignment, PtrAlign)));
1869 MI.eraseFromParent();
1873bool AArch64LegalizerInfo::legalizeBitfieldExtract(
1907 LLT Ty =
MRI.getType(Val);
1911 "Expected src and dst to have the same type!");
1919 auto Add = MIRBuilder.
buildAdd(s64, CTPOP1, CTPOP2);
1922 MI.eraseFromParent();
1926 if (!ST->hasNEON() ||
1927 MI.getMF()->getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) {
1939 assert((
Size == 32 ||
Size == 64 ||
Size == 128) &&
"Expected only 32, 64, or 128 bit scalars!");
1963 Sum = MIRBuilder.
buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones,
CTPOP});
1965 Sum = MIRBuilder.
buildInstr(AArch64::G_UDOT, {Dt}, {Zeros, Ones,
CTPOP});
1971 MI.eraseFromParent();
1979 Opc = Intrinsic::aarch64_neon_uaddlv;
1982 Opc = Intrinsic::aarch64_neon_uaddlp;
1985 Opc = Intrinsic::aarch64_neon_uaddlp;
1989 Opc = Intrinsic::aarch64_neon_uaddlp;
1994 Opc = Intrinsic::aarch64_neon_uaddlp;
1997 Opc = Intrinsic::aarch64_neon_uaddlp;
2003 for (
LLT HTy : HAddTys) {
2013 MI.eraseFromParent();
2017bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
2021 auto Addr =
MI.getOperand(1).getReg();
2022 auto DesiredI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(2));
2023 auto NewI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(3));
2024 auto DstLo =
MRI.createGenericVirtualRegister(s64);
2025 auto DstHi =
MRI.createGenericVirtualRegister(s64);
2038 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
2042 Opcode = AArch64::CASPAX;
2045 Opcode = AArch64::CASPLX;
2049 Opcode = AArch64::CASPALX;
2052 Opcode = AArch64::CASPX;
2057 auto CASDst =
MRI.createGenericVirtualRegister(s128);
2058 auto CASDesired =
MRI.createGenericVirtualRegister(s128);
2059 auto CASNew =
MRI.createGenericVirtualRegister(s128);
2060 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASDesired}, {})
2061 .addUse(DesiredI->getOperand(0).getReg())
2063 .
addUse(DesiredI->getOperand(1).getReg())
2064 .
addImm(AArch64::subo64);
2065 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASNew}, {})
2069 .
addImm(AArch64::subo64);
2071 CAS = MIRBuilder.
buildInstr(Opcode, {CASDst}, {CASDesired, CASNew,
Addr});
2079 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
2083 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
2086 Opcode = AArch64::CMP_SWAP_128_RELEASE;
2090 Opcode = AArch64::CMP_SWAP_128;
2093 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
2097 auto Scratch =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
2098 CAS = MIRBuilder.
buildInstr(Opcode, {DstLo, DstHi, Scratch},
2099 {
Addr, DesiredI->getOperand(0),
2100 DesiredI->getOperand(1), NewI->
getOperand(0),
2106 *
MRI.getTargetRegisterInfo(),
2110 MI.eraseFromParent();
2118 LLT Ty =
MRI.getType(
MI.getOperand(1).getReg());
2120 MIRBuilder.
buildCTLZ(
MI.getOperand(0).getReg(), BitReverse);
2121 MI.eraseFromParent();
2130 if (
MI.getOpcode() == TargetOpcode::G_MEMSET) {
2133 auto &
Value =
MI.getOperand(1);
2136 Value.setReg(ExtValueReg);
2143bool AArch64LegalizerInfo::legalizeExtractVectorElt(
2145 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
2154bool AArch64LegalizerInfo::legalizeDynStackAlloc(
2170 Register AllocSize =
MI.getOperand(1).getReg();
2174 "Unexpected type for dynamic alloca");
2176 "Unexpected type for dynamic alloca");
2178 LLT PtrTy =
MRI.getType(Dst);
2184 MIRBuilder.
buildInstr(AArch64::PROBED_STACKALLOC_DYN, {}, {SPTmp});
2185 MRI.setRegClass(NewMI.getReg(0), &AArch64::GPR64commonRegClass);
2186 MIRBuilder.
setInsertPt(*NewMI->getParent(), NewMI);
2189 MI.eraseFromParent();
2196 auto &AddrVal =
MI.getOperand(0);
2198 int64_t IsWrite =
MI.getOperand(1).getImm();
2199 int64_t Locality =
MI.getOperand(2).getImm();
2200 int64_t
IsData =
MI.getOperand(3).getImm();
2202 bool IsStream = Locality == 0;
2203 if (Locality != 0) {
2204 assert(Locality <= 3 &&
"Prefetch locality out-of-range");
2208 Locality = 3 - Locality;
2211 unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
2214 MI.eraseFromParent();
unsigned const MachineRegisterInfo * MRI
static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset, MachineRegisterInfo &MRI)
This file declares the targeting of the Machinelegalizer class for AArch64.
This file declares the targeting of the RegisterBankInfo class for AArch64.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Interface for Targets to specify which operations they can successfully select and how the others sho...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
AArch64LegalizerInfo(const AArch64Subtarget &ST)
bool isTargetWindows() const
const AArch64InstrInfo * getInstrInfo() const override
bool isTargetDarwin() const
bool isTargetILP32() const
const AArch64TargetLowering * getTargetLowering() const override
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
const RegisterBankInfo * getRegBankInfo() const override
Class for arbitrary precision integers.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
int64_t getSExtValue() const
Get sign extended value.
StringRef getValueAsString() const
Return the attribute's value as a string.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
This class represents an Operation in the Expression.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT scalable_vector(unsigned MinNumElements, unsigned ScalarSizeInBits)
Get a low-level scalable vector of some number of elements and element width.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
constexpr bool isPointerVector() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr LLT divide(int Factor) const
Return a type that is Factor times smaller.
void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & unsupported()
The instruction is unsupported.
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
LegalizeRuleSet & libcallFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & maxScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at most as wide as Ty.
LegalizeRuleSet & minScalarOrElt(unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
LegalizeRuleSet & clampMinNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MinElements)
Limit the number of elements in EltTy vectors to at least MinElements.
LegalizeRuleSet & widenVectorEltsToVectorMinSize(unsigned TypeIdx, unsigned VectorSize)
Ensure the vector size is at least as wide as VectorSize by promoting the element.
LegalizeRuleSet & minScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned LargeTypeIdx)
Conditionally widen the scalar or elt to match the size of another.
LegalizeRuleSet & customForCartesianProduct(std::initializer_list< LLT > Types)
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & moreElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Add more elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx)
Widen the scalar to match the size of another.
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
LegalizeRuleSet & minScalarOrEltIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & legalForTypesWithMemDesc(std::initializer_list< LegalityPredicates::TypePairAndMemDesc > TypesAndMemDesc)
The instruction is legal when type indexes 0 and 1 along with the memory size and minimum alignment i...
LegalizeRuleSet & libcallIf(LegalityPredicate Predicate)
Like legalIf, but for the Libcall action.
LegalizeRuleSet & widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar or vector element type to the next power of two that is at least MinSize.
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeResult lowerDynStackAlloc(MachineInstr &MI)
LegalizeResult lowerBitCount(MachineInstr &MI)
LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI)
Lower a vector extract or insert by writing the vector to a stack temporary and reloading the element...
LegalizeResult lowerAbsToCNeg(MachineInstr &MI)
const TargetLowering & getTargetLowering() const
LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
@ UnableToLegalize
Some kind of error has occurred and we could not legalize this instruction.
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
Register getDynStackAllocTargetPtr(Register SPReg, Register AllocSize, Align Alignment, LLT PtrTy)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
const LegacyLegalizerInfo & getLegacyLegalizerInfo() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ... = G_EXTRACT Src, Idx0.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ZEXT Op.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMAX Op0, Op1.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildBitReverse(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITREVERSE Src.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMIN Op0, Op1.
MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op)
Build and insert Res = ExtOpc, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes of...
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_TRUNC Op.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMIN Op0, Op1.
MachineInstrBuilder buildUMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMAX Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Wrapper class representing virtual and physical registers.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
const TargetMachine & getTargetMachine() const
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
Primary interface to the complete machine description for the target machine.
Target - Wrapper for Target specific information.
LLVM Value Representation.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar or a vector with an element type that's wider than the ...
LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, AtomicOrdering Ordering)
True iff the specified MMO index has at an atomic ordering of at Ordering or stronger.
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
Predicate predNot(Predicate P)
True iff P is false.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min=0)
Add more elements to the type for the given type index to the next power of.
LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx)
Change the scalar size or element size to have the same scalar size as type index FromIndex.
operand_type_match m_Reg()
ConstantMatch< APInt > m_ICst(APInt &Cst)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
This is an optimization pass for GlobalISel generic memory operations.
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
AtomicOrdering
Atomic ordering for LLVM's memory model.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
unsigned Log2(Align A)
Returns the log2 of the alignment.
std::function< bool(const LegalityQuery &)> LegalityPredicate
This struct is a compact representation of a valid (non-zero power of two) alignment.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
This class contains a discriminated union of information about pointers in memory operands,...