30#include "llvm/IR/IntrinsicsAArch64.h"
33#include <initializer_list>
35#define DEBUG_TYPE "aarch64-legalinfo"
38using namespace LegalizeActions;
39using namespace LegalizeMutations;
40using namespace LegalityPredicates;
41using namespace MIPatternMatch;
45 using namespace TargetOpcode;
63 std::initializer_list<LLT> PackedVectorAllTypeList = {
69 std::initializer_list<LLT> ScalarAndPtrTypesList = {s8, s16, s32, s64, p0};
76 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
83 const bool HasFP16 = ST.hasFullFP16();
84 const LLT &MinFPScalar = HasFP16 ? s16 : s32;
86 const bool HasCSSC = ST.hasCSSC();
87 const bool HasRCPC3 = ST.hasRCPC3();
90 {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
91 .legalFor({p0, s8, s16, s32, s64})
92 .legalFor(PackedVectorAllTypeList)
97 return Query.
Types[0].isVector() &&
98 (Query.
Types[0].getElementType() != s64 ||
99 Query.
Types[0].getNumElements() != 2);
102 LLT EltTy = Query.
Types[0].getElementType();
105 return std::make_pair(0, EltTy);
110 .legalFor(PackedVectorAllTypeList)
121 .
legalFor({s32, s64, v4s32, v2s32, v2s64})
122 .widenScalarToNextPow2(0)
126 .legalFor({s32, s64, v2s32, v2s64, v4s32, v4s16, v8s16, v16s8, v8s8})
127 .widenScalarToNextPow2(0)
135 return Query.
Types[0].getNumElements() <= 2;
140 return Query.
Types[0].getNumElements() <= 4;
145 return Query.
Types[0].getNumElements() <= 16;
152 const auto &SrcTy = Query.
Types[0];
153 const auto &AmtTy = Query.
Types[1];
154 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
155 AmtTy.getSizeInBits() == 32;
169 .widenScalarToNextPow2(0)
178 .
legalFor({{p0, s64}, {v2p0, v2s64}})
179 .clampScalar(1, s64, s64);
184 .legalFor({s32, s64})
186 .clampScalar(0, s32, s64)
191 .lowerFor({s8, s16, s32, s64, v2s64, v4s32, v2s32})
193 .clampScalarOrElt(0, s32, s64)
194 .clampNumElements(0, v2s32, v4s32)
195 .clampNumElements(0, v2s64, v2s64)
196 .moreElementsToNextPow2(0);
200 .widenScalarToNextPow2(0, 32)
205 .legalFor({s64, v8s16, v16s8, v4s32})
209 {G_SMIN, G_SMAX, G_UMIN, G_UMAX});
212 .
legalFor({s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
219 .
legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32});
230 {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
231 .legalFor({{s32, s32}, {s64, s32}})
232 .clampScalar(0, s32, s64)
237 G_FABS, G_FSQRT, G_FMAXNUM, G_FMINNUM,
238 G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR,
239 G_FRINT, G_FNEARBYINT, G_INTRINSIC_TRUNC,
240 G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
241 .legalFor({MinFPScalar, s32, s64, v2s32, v4s32, v2s64})
243 const auto &Ty = Query.
Types[0];
244 return (Ty == v8s16 || Ty == v4s16) && HasFP16;
247 .minScalarOrElt(0, MinFPScalar)
263 const auto &Ty = Query.
Types[0];
264 return Ty.isVector() && Ty.getElementType() == s16 &&
267 [=](
const LegalityQuery &Query) {
return std::make_pair(0, s16); })
272 return Query.
Types[0] == s16 && !ST.hasFullFP16();
274 [=](
const LegalityQuery &Query) {
return std::make_pair(0, s32); })
275 .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16});
278 {G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, G_FLOG10,
279 G_FEXP, G_FEXP2, G_FEXP10})
284 .libcallFor({s32, s64});
308 for (
unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
311 if (
Op == G_SEXTLOAD)
316 .legalForTypesWithMemDesc({{s32, p0, s8, 8},
324 {v2s32, p0, s64, 8}})
325 .widenScalarToNextPow2(0)
326 .clampScalar(0, s32, s64)
329 .unsupportedIfMemSizeNotPow2()
344 return HasRCPC3 && Query.
Types[0] == s128 &&
348 return Query.
Types[0] == s128 &&
351 .legalForTypesWithMemDesc({{s8, p0, s8, 8},
358 {v16s8, p0, s128, 8},
360 {v8s16, p0, s128, 8},
362 {v4s32, p0, s128, 8},
363 {v2s64, p0, s128, 8}})
365 .legalForTypesWithMemDesc({{s32, p0, s8, 8}, {s32, p0, s16, 8}})
366 .widenScalarToNextPow2(0, 8)
372 return Query.
Types[0].isScalar() &&
374 Query.
Types[0].getSizeInBits() > 32;
377 .clampMaxNumElements(0, s8, 16)
387 return HasRCPC3 && Query.
Types[0] == s128 &&
391 return Query.
Types[0] == s128 &&
394 .legalForTypesWithMemDesc(
395 {{s8, p0, s8, 8}, {s16, p0, s8, 8},
398 {s16, p0, s16, 8}, {s32, p0, s16, 8},
400 {s32, p0, s8, 8}, {s32, p0, s16, 8}, {s32, p0, s32, 8},
401 {s64, p0, s64, 8}, {s64, p0, s32, 8},
402 {p0, p0, s64, 8}, {s128, p0, s128, 8}, {v16s8, p0, s128, 8},
403 {v8s8, p0, s64, 8}, {v4s16, p0, s64, 8}, {v8s16, p0, s128, 8},
404 {v2s32, p0, s64, 8}, {v4s32, p0, s128, 8}, {v2s64, p0, s128, 8}})
405 .clampScalar(0, s8, s64)
407 return Query.
Types[0].isScalar() &&
411 .clampMaxNumElements(0, s8, 16)
433 {p0, v16s8, v16s8, 8},
434 {p0, v4s16, v4s16, 8},
435 {p0, v8s16, v8s16, 8},
436 {p0, v2s32, v2s32, 8},
437 {p0, v4s32, v4s32, 8},
438 {p0, v2s64, v2s64, 8},
444 auto IndexedLoadBasicPred = [=](
const LegalityQuery &Query) {
447 if (
llvm::find(PackedVectorAllTypesVec, LdTy) ==
448 PackedVectorAllTypesVec.
end() &&
449 llvm::find(ScalarAndPtrTypesVec, LdTy) == ScalarAndPtrTypesVec.
end() &&
474 return MemTy == s8 || MemTy == s16;
476 return MemTy == s8 || MemTy == s16 || MemTy == s32;
484 .widenScalarToNextPow2(0)
488 const auto &Ty = Query.
Types[0];
489 if (HasFP16 && Ty == s16)
491 return Ty == s32 || Ty == s64 || Ty == s128;
493 .clampScalar(0, MinFPScalar, s128);
508 .clampScalar(1, s32, s64)
509 .clampScalar(0, s32, s32)
510 .minScalarEltSameAsIf(
524 .clampNumElements(0, v2s32, v4s32);
531 const auto &Ty = Query.
Types[0];
532 return Ty.isVector() && Ty.getElementType() == s16 && !HasFP16;
534 [=](
const LegalityQuery &Query) {
return std::make_pair(0, s16); })
539 return Query.
Types[0] == s16 && !HasFP16;
541 [=](
const LegalityQuery &Query) {
return std::make_pair(0, s32); })
542 .legalFor({{s16, s16},
551 .clampScalar(1, s32, s64)
552 .clampScalar(0, s32, s32)
553 .minScalarEltSameAsIf(
561 .clampNumElements(0, v2s32, v4s32)
562 .clampMaxNumElements(1, s64, 2);
566 unsigned DstSize = Query.
Types[0].getSizeInBits();
569 if (Query.
Types[0].isVector())
572 if (DstSize < 8 || DstSize >= 128 || !
isPowerOf2_32(DstSize))
587 .legalIf(ExtLegalFunc)
588 .
legalFor({{v2s64, v2s32}, {v4s32, v4s16}, {v8s16, v8s8}})
589 .clampScalar(0, s64, s64)
596 return (Query.
Types[0].getScalarSizeInBits() >
597 Query.
Types[1].getScalarSizeInBits() * 2) &&
598 Query.
Types[0].isVector() &&
599 (Query.
Types[1].getScalarSizeInBits() == 8 ||
600 Query.
Types[1].getScalarSizeInBits() == 16);
604 .
legalFor({{v2s32, v2s64}, {v4s16, v4s32}, {v8s8, v8s16}})
606 .clampMaxNumElements(0, s8, 8)
607 .clampMaxNumElements(0, s16, 4)
608 .clampMaxNumElements(0, s32, 2)
624 .legalFor(PackedVectorAllTypeList)
631 {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
632 .clampNumElements(0, v4s16, v4s16)
638 {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
639 .clampNumElements(0, v4s32, v4s32)
645 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
648 (Query.
Types[1] == s16 || Query.
Types[1] == v4s16 ||
649 Query.
Types[1] == v8s16) &&
650 (Query.
Types[0] == s32 || Query.
Types[0] == s64 ||
651 Query.
Types[0] == v4s16 || Query.
Types[0] == v8s16);
653 .widenScalarToNextPow2(0)
660 return Query.
Types[0].getScalarSizeInBits() >
661 Query.
Types[1].getScalarSizeInBits();
666 return Query.
Types[0].getScalarSizeInBits() <
667 Query.
Types[1].getScalarSizeInBits();
670 .clampNumElements(0, v4s16, v8s16)
675 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
678 (Query.
Types[0] == s16 || Query.
Types[0] == v4s16 ||
679 Query.
Types[0] == v8s16) &&
680 (Query.
Types[1] == s32 || Query.
Types[1] == s64 ||
681 Query.
Types[1] == v4s16 || Query.
Types[1] == v8s16);
683 .widenScalarToNextPow2(1)
690 return Query.
Types[0].getScalarSizeInBits() <
691 Query.
Types[1].getScalarSizeInBits();
696 return Query.
Types[0].getScalarSizeInBits() >
697 Query.
Types[1].getScalarSizeInBits();
700 .clampNumElements(0, v4s16, v8s16)
707 .clampScalar(0, s32, s32);
711 .
legalFor({{s32, s32}, {s64, s32}, {p0, s32}})
712 .widenScalarToNextPow2(0)
727 .
legalFor({{s64, p0}, {v2s64, v2p0}})
728 .widenScalarToNextPow2(0, 64)
733 return Query.
Types[0].getSizeInBits() != Query.
Types[1].getSizeInBits();
735 .legalFor({{p0, s64}, {v2p0, v2s64}});
744 v8s16, v4s16, v2s16, v4s32, v2s32, v2s64,
753 .clampScalar(0, s8, s64)
762 return Query.
Types[0].getSizeInBits() == 128;
764 .clampScalar(0, s32, s64)
768 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND,
769 G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX,
770 G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
771 .clampScalar(0, s32, s64)
777 for (
unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
778 unsigned BigTyIdx =
Op == G_MERGE_VALUES ? 0 : 1;
779 unsigned LitTyIdx =
Op == G_MERGE_VALUES ? 1 : 0;
786 switch (Q.
Types[BigTyIdx].getSizeInBits()) {
794 switch (Q.
Types[LitTyIdx].getSizeInBits()) {
808 const LLT &EltTy = Query.
Types[1].getElementType();
809 return Query.
Types[0] != EltTy;
814 return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 ||
815 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 ||
816 VecTy == v8s8 || VecTy == v16s8 || VecTy == v2p0;
822 return Query.
Types[1].getNumElements() <= 2;
827 return Query.
Types[1].getNumElements() <= 4;
832 return Query.
Types[1].getNumElements() <= 8;
837 return Query.
Types[1].getNumElements() <= 16;
840 .minScalarOrElt(0, s8)
859 .clampNumElements(0, v4s32, v4s32)
869 {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
871 .widenScalarToNextPow2(1, 32)
872 .clampScalar(1, s32, s64)
873 .scalarSameSizeAs(0, 1);
879 .widenScalarToNextPow2(0, 32)
890 return (HasCSSC &&
typeInSet(0, {s32, s64})(Query));
893 return (!HasCSSC &&
typeInSet(0, {s32, s64})(Query));
905 {v2s64, v2p0, v2s32, v4s32, v4s16, v16s8, v8s8, v8s16}, DstTy);
910 return !Query.
Types[1].isVector();
914 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
915 Query.
Types[0].getNumElements() >
916 Query.
Types[1].getNumElements();
920 .clampNumElements(0, v4s32, v4s32)
921 .clampNumElements(0, v2s64, v2s64)
924 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
925 Query.
Types[0].getNumElements() <
926 Query.
Types[1].getNumElements();
931 .
legalFor({{v4s32, v2s32}, {v8s16, v4s16}, {v16s8, v8s8}});
948 .customForCartesianProduct({p0}, {s8}, {s64})
952 .legalForCartesianProduct({p0}, {p0}, {s64})
968 .legalFor({s32, s64});
970 .legalFor(PackedVectorAllTypeList)
977 .
legalFor({{s32, v2s32}, {s32, v4s32}, {s64, v2s64}})
979 const auto &Ty = Query.
Types[1];
980 return (Ty == v4s16 || Ty == v8s16) && HasFP16;
982 .minScalarOrElt(0, MinFPScalar)
996 .clampMaxNumElements(1, s64, 2)
1003 G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
1004 .legalFor({{s32, v4s32}, {s32, v2s32}, {s64, v2s64}})
1006 const auto &Ty = Query.
Types[1];
1007 return Query.
Types[0] == s16 && (Ty == v8s16 || Ty == v4s16) && HasFP16;
1009 .minScalarOrElt(0, MinFPScalar)
1023 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX})
1024 .legalFor({{s8, v8s8},
1030 .clampMaxNumElements(1, s64, 2)
1038 {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
1054 return std::make_pair(1, SrcTy.
divide(2));
1063 .customFor({{s32, s32}, {s32, s64}, {s64, s64}})
1067 .
legalFor({{s32, s64}, {s64, s64}})
1069 return Q.
Types[0].isScalar() && Q.
Types[1].getScalarSizeInBits() < 64;
1075 .customFor({{s32, s32}, {s64, s64}});
1081 .legalFor({{s32, s32},
1085 .customFor({{s128, s128},
1093 .legalFor({{v8s8, v8s8},
1095 .customFor({{s32, s32},
1104 .clampScalar(0, s32, s128)
1105 .widenScalarToNextPow2(0)
1106 .minScalarEltSameAsIf(always, 1, 0)
1107 .maxScalarEltSameAsIf(always, 1, 0);
1115 .legalFor({{s64, s32}, {s64, s64}});
1131 verify(*ST.getInstrInfo());
1139 switch (
MI.getOpcode()) {
1143 case TargetOpcode::G_VAARG:
1144 return legalizeVaArg(
MI,
MRI, MIRBuilder);
1145 case TargetOpcode::G_LOAD:
1146 case TargetOpcode::G_STORE:
1147 return legalizeLoadStore(
MI,
MRI, MIRBuilder, Observer);
1148 case TargetOpcode::G_SHL:
1149 case TargetOpcode::G_ASHR:
1150 case TargetOpcode::G_LSHR:
1151 return legalizeShlAshrLshr(
MI,
MRI, MIRBuilder, Observer);
1152 case TargetOpcode::G_GLOBAL_VALUE:
1153 return legalizeSmallCMGlobalValue(
MI,
MRI, MIRBuilder, Observer);
1154 case TargetOpcode::G_SBFX:
1155 case TargetOpcode::G_UBFX:
1156 return legalizeBitfieldExtract(
MI,
MRI, Helper);
1157 case TargetOpcode::G_FSHL:
1158 case TargetOpcode::G_FSHR:
1159 return legalizeFunnelShift(
MI,
MRI, MIRBuilder, Observer, Helper);
1160 case TargetOpcode::G_ROTR:
1161 return legalizeRotate(
MI,
MRI, Helper);
1162 case TargetOpcode::G_CTPOP:
1163 return legalizeCTPOP(
MI,
MRI, Helper);
1164 case TargetOpcode::G_ATOMIC_CMPXCHG:
1165 return legalizeAtomicCmpxchg128(
MI,
MRI, Helper);
1166 case TargetOpcode::G_CTTZ:
1167 return legalizeCTTZ(
MI, Helper);
1168 case TargetOpcode::G_BZERO:
1169 case TargetOpcode::G_MEMCPY:
1170 case TargetOpcode::G_MEMMOVE:
1171 case TargetOpcode::G_MEMSET:
1172 return legalizeMemOps(
MI, Helper);
1173 case TargetOpcode::G_FCOPYSIGN:
1174 return legalizeFCopySign(
MI, Helper);
1175 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1176 return legalizeExtractVectorElt(
MI,
MRI, Helper);
1177 case TargetOpcode::G_DYN_STACKALLOC:
1178 return legalizeDynStackAlloc(
MI, Helper);
1189 assert(
MI.getOpcode() == TargetOpcode::G_FSHL ||
1190 MI.getOpcode() == TargetOpcode::G_FSHR);
1194 Register ShiftNo =
MI.getOperand(3).getReg();
1195 LLT ShiftTy =
MRI.getType(ShiftNo);
1200 LLT OperationTy =
MRI.getType(
MI.getOperand(0).getReg());
1204 if (!VRegAndVal || VRegAndVal->Value.urem(
BitWidth) == 0)
1210 Amount =
MI.getOpcode() == TargetOpcode::G_FSHL ?
BitWidth - Amount : Amount;
1214 if (ShiftTy.
getSizeInBits() == 64 &&
MI.getOpcode() == TargetOpcode::G_FSHR &&
1221 if (
MI.getOpcode() == TargetOpcode::G_FSHR) {
1223 MI.getOperand(3).setReg(Cast64.getReg(0));
1228 else if (
MI.getOpcode() == TargetOpcode::G_FSHL) {
1230 {
MI.getOperand(1).
getReg(),
MI.getOperand(2).getReg(),
1232 MI.eraseFromParent();
1243 LLT AmtTy =
MRI.getType(AmtReg);
1249 MI.getOperand(2).setReg(NewAmt.getReg(0));
1254bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
1257 assert(
MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
1262 auto &GlobalOp =
MI.getOperand(1);
1263 const auto* GV = GlobalOp.getGlobal();
1264 if (GV->isThreadLocal())
1273 auto Offset = GlobalOp.getOffset();
1278 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1295 "Should not have folded in an offset for a tagged global!");
1297 .addGlobalAddress(GV, 0x100000000,
1300 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1304 .addGlobalAddress(GV,
Offset,
1306 MI.eraseFromParent();
1313 switch (IntrinsicID) {
1314 case Intrinsic::vacopy: {
1316 unsigned VaListSize =
1328 VaListSize,
Align(PtrSize)));
1332 VaListSize,
Align(PtrSize)));
1333 MI.eraseFromParent();
1336 case Intrinsic::get_dynamic_area_offset: {
1339 MI.eraseFromParent();
1342 case Intrinsic::aarch64_mops_memset_tag: {
1343 assert(
MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1347 auto &
Value =
MI.getOperand(3);
1349 Value.setReg(ExtValueReg);
1352 case Intrinsic::prefetch: {
1354 auto &AddrVal =
MI.getOperand(1);
1356 int64_t IsWrite =
MI.getOperand(2).getImm();
1357 int64_t Locality =
MI.getOperand(3).getImm();
1358 int64_t IsData =
MI.getOperand(4).getImm();
1360 bool IsStream = Locality == 0;
1361 if (Locality != 0) {
1362 assert(Locality <= 3 &&
"Prefetch locality out-of-range");
1366 Locality = 3 - Locality;
1370 (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
1373 MI.eraseFromParent();
1376 case Intrinsic::aarch64_prefetch: {
1378 auto &AddrVal =
MI.getOperand(1);
1380 int64_t IsWrite =
MI.getOperand(2).getImm();
1381 int64_t
Target =
MI.getOperand(3).getImm();
1382 int64_t IsStream =
MI.getOperand(4).getImm();
1383 int64_t IsData =
MI.getOperand(5).getImm();
1385 unsigned PrfOp = (IsWrite << 4) |
1391 MI.eraseFromParent();
1394 case Intrinsic::aarch64_neon_uaddv:
1395 case Intrinsic::aarch64_neon_saddv:
1396 case Intrinsic::aarch64_neon_umaxv:
1397 case Intrinsic::aarch64_neon_smaxv:
1398 case Intrinsic::aarch64_neon_uminv:
1399 case Intrinsic::aarch64_neon_sminv: {
1402 bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
1403 IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
1404 IntrinsicID == Intrinsic::aarch64_neon_sminv;
1406 auto OldDst =
MI.getOperand(0).getReg();
1407 auto OldDstTy =
MRI.getType(OldDst);
1408 LLT NewDstTy =
MRI.getType(
MI.getOperand(2).getReg()).getElementType();
1409 if (OldDstTy == NewDstTy)
1412 auto NewDst =
MRI.createGenericVirtualRegister(NewDstTy);
1415 MI.getOperand(0).setReg(NewDst);
1419 MIB.
buildExtOrTrunc(IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
1424 case Intrinsic::aarch64_neon_smax:
1425 case Intrinsic::aarch64_neon_smin:
1426 case Intrinsic::aarch64_neon_umax:
1427 case Intrinsic::aarch64_neon_umin:
1428 case Intrinsic::aarch64_neon_fmax:
1429 case Intrinsic::aarch64_neon_fmin: {
1431 if (IntrinsicID == Intrinsic::aarch64_neon_smax)
1433 else if (IntrinsicID == Intrinsic::aarch64_neon_smin)
1435 else if (IntrinsicID == Intrinsic::aarch64_neon_umax)
1437 else if (IntrinsicID == Intrinsic::aarch64_neon_umin)
1439 else if (IntrinsicID == Intrinsic::aarch64_neon_fmax)
1440 MIB.
buildInstr(TargetOpcode::G_FMAXIMUM, {
MI.getOperand(0)},
1441 {
MI.getOperand(2),
MI.getOperand(3)});
1442 else if (IntrinsicID == Intrinsic::aarch64_neon_fmin)
1443 MIB.
buildInstr(TargetOpcode::G_FMINIMUM, {
MI.getOperand(0)},
1444 {
MI.getOperand(2),
MI.getOperand(3)});
1445 MI.eraseFromParent();
1448 case Intrinsic::experimental_vector_reverse:
1456bool AArch64LegalizerInfo::legalizeShlAshrLshr(
1459 assert(
MI.getOpcode() == TargetOpcode::G_ASHR ||
1460 MI.getOpcode() == TargetOpcode::G_LSHR ||
1461 MI.getOpcode() == TargetOpcode::G_SHL);
1474 MI.getOperand(2).setReg(ExtCst.getReg(0));
1487 isShiftedInt<7, 3>(NewOffset)) {
1495bool AArch64LegalizerInfo::legalizeLoadStore(
1498 assert(
MI.getOpcode() == TargetOpcode::G_STORE ||
1499 MI.getOpcode() == TargetOpcode::G_LOAD);
1510 const LLT ValTy =
MRI.getType(ValReg);
1515 bool IsLoad =
MI.getOpcode() == TargetOpcode::G_LOAD;
1519 ST->hasLSE2() && ST->hasRCPC3() && (IsLoadAcquire || IsStoreRelease);
1525 Opcode = IsLoad ? AArch64::LDIAPPX : AArch64::STILPX;
1531 assert(ST->hasLSE2() &&
"ldp/stp not single copy atomic without +lse2");
1533 Opcode = IsLoad ? AArch64::LDPXi : AArch64::STPXi;
1548 NewI.
addUse(
MI.getOperand(1).getReg());
1559 *
MRI.getTargetRegisterInfo(),
1561 MI.eraseFromParent();
1567 LLVM_DEBUG(
dbgs() <<
"Tried to do custom legalization on wrong load/store");
1573 auto &MMO = **
MI.memoperands_begin();
1576 if (
MI.getOpcode() == TargetOpcode::G_STORE) {
1580 auto NewLoad = MIRBuilder.
buildLoad(NewTy,
MI.getOperand(1), MMO);
1583 MI.eraseFromParent();
1591 Align Alignment(
MI.getOperand(2).getImm());
1593 Register ListPtr =
MI.getOperand(1).getReg();
1595 LLT PtrTy =
MRI.getType(ListPtr);
1606 if (Alignment > PtrAlign) {
1610 auto ListTmp = MIRBuilder.
buildPtrAdd(PtrTy,
List, AlignMinus1.getReg(0));
1615 LLT ValTy =
MRI.getType(Dst);
1620 ValTy, std::max(Alignment, PtrAlign)));
1631 MI.eraseFromParent();
1635bool AArch64LegalizerInfo::legalizeBitfieldExtract(
1669 LLT Ty =
MRI.getType(Val);
1673 "Expected src and dst to have the same type!");
1681 auto Add = MIRBuilder.
buildAdd(s64, CTPOP1, CTPOP2);
1684 MI.eraseFromParent();
1688 if (!ST->hasNEON() ||
1689 MI.getMF()->getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) {
1701 assert((
Size == 32 ||
Size == 64 ||
Size == 128) &&
"Expected only 32, 64, or 128 bit scalars!");
1716 Opc = Intrinsic::aarch64_neon_uaddlv;
1719 Opc = Intrinsic::aarch64_neon_uaddlp;
1722 Opc = Intrinsic::aarch64_neon_uaddlp;
1726 Opc = Intrinsic::aarch64_neon_uaddlp;
1731 Opc = Intrinsic::aarch64_neon_uaddlp;
1734 Opc = Intrinsic::aarch64_neon_uaddlp;
1740 for (
LLT HTy : HAddTys) {
1750 MI.eraseFromParent();
1754bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
1758 auto Addr =
MI.getOperand(1).getReg();
1759 auto DesiredI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(2));
1760 auto NewI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(3));
1761 auto DstLo =
MRI.createGenericVirtualRegister(s64);
1762 auto DstHi =
MRI.createGenericVirtualRegister(s64);
1775 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
1779 Opcode = AArch64::CASPAX;
1782 Opcode = AArch64::CASPLX;
1786 Opcode = AArch64::CASPALX;
1794 auto CASDst =
MRI.createGenericVirtualRegister(s128);
1795 auto CASDesired =
MRI.createGenericVirtualRegister(s128);
1796 auto CASNew =
MRI.createGenericVirtualRegister(s128);
1797 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASDesired}, {})
1798 .addUse(DesiredI->getOperand(0).getReg())
1800 .
addUse(DesiredI->getOperand(1).getReg())
1801 .
addImm(AArch64::subo64);
1802 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASNew}, {})
1806 .
addImm(AArch64::subo64);
1816 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
1820 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
1823 Opcode = AArch64::CMP_SWAP_128_RELEASE;
1827 Opcode = AArch64::CMP_SWAP_128;
1830 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
1834 auto Scratch =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1836 {
Addr, DesiredI->getOperand(0),
1837 DesiredI->getOperand(1), NewI->
getOperand(0),
1843 *
MRI.getTargetRegisterInfo(),
1847 MI.eraseFromParent();
1855 LLT Ty =
MRI.getType(
MI.getOperand(1).getReg());
1857 MIRBuilder.
buildCTLZ(
MI.getOperand(0).getReg(), BitReverse);
1858 MI.eraseFromParent();
1867 if (
MI.getOpcode() == TargetOpcode::G_MEMSET) {
1870 auto &
Value =
MI.getOperand(1);
1873 Value.setReg(ExtValueReg);
1885 LLT DstTy =
MRI.getType(Dst);
1888 assert((DstSize == 32 || DstSize == 64) &&
"Unexpected dst type!");
1889 assert(
MRI.getType(
MI.getOperand(2).getReg()) == DstTy &&
1890 "Expected homogeneous types!");
1909 EltMask = 0x80000000ULL;
1919 VecTy, Undef,
MI.getOperand(1).getReg(), Zero);
1921 VecTy, Undef,
MI.getOperand(2).getReg(), Zero);
1928 auto Sel = MIRBuilder.
buildInstr(AArch64::G_BSP, {VecTy}, {
Mask, Ins2, Ins1});
1934 DstRegs.push_back(
MRI.createGenericVirtualRegister(DstTy));
1936 MI.eraseFromParent();
1940bool AArch64LegalizerInfo::legalizeExtractVectorElt(
1942 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1951bool AArch64LegalizerInfo::legalizeDynStackAlloc(
1967 Register AllocSize =
MI.getOperand(1).getReg();
1971 "Unexpected type for dynamic alloca");
1973 "Unexpected type for dynamic alloca");
1975 LLT PtrTy =
MRI.getType(Dst);
1981 MIRBuilder.
buildInstr(AArch64::PROBED_STACKALLOC_DYN, {}, {SPTmp});
1982 MRI.setRegClass(NewMI.getReg(0), &AArch64::GPR64commonRegClass);
1983 MIRBuilder.
setInsertPt(*NewMI->getParent(), NewMI);
1986 MI.eraseFromParent();
unsigned const MachineRegisterInfo * MRI
static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset, MachineRegisterInfo &MRI)
This file declares the targeting of the Machinelegalizer class for AArch64.
This file declares the targeting of the RegisterBankInfo class for AArch64.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Interface for Targets to specify which operations they can successfully select and how the others sho...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static constexpr uint32_t Opcode
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override
Called for instructions with the Custom LegalizationAction.
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
AArch64LegalizerInfo(const AArch64Subtarget &ST)
bool isTargetWindows() const
const AArch64InstrInfo * getInstrInfo() const override
bool isTargetDarwin() const
bool isTargetILP32() const
const AArch64TargetLowering * getTargetLowering() const override
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
const RegisterBankInfo * getRegBankInfo() const override
Class for arbitrary precision integers.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
int64_t getSExtValue() const
Get sign extended value.
StringRef getValueAsString() const
Return the attribute's value as a string.
This class represents an Operation in the Expression.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT divide(int Factor) const
Return a type that is Factor times smaller.
void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & unsupported()
The instruction is unsupported.
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & libcallFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & maxScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at most as wide as Ty.
LegalizeRuleSet & minScalarOrElt(unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
LegalizeRuleSet & widenVectorEltsToVectorMinSize(unsigned TypeIdx, unsigned VectorSize)
Ensure the vector size is at least as wide as VectorSize by promoting the element.
LegalizeRuleSet & lowerIfMemSizeNotPow2()
Lower a memory operation if the memory size, rounded to bytes, is not a power of 2.
LegalizeRuleSet & minScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned LargeTypeIdx)
Conditionally widen the scalar or elt to match the size of another.
LegalizeRuleSet & customForCartesianProduct(std::initializer_list< LLT > Types)
LegalizeRuleSet & lowerIfMemSizeNotByteSizePow2()
Lower a memory operation if the memory access size is not a round power of 2 byte size.
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
LegalizeRuleSet & narrowScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Narrow the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx)
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx)
Widen the scalar to match the size of another.
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
LegalizeRuleSet & minScalarOrEltIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & legalForTypesWithMemDesc(std::initializer_list< LegalityPredicates::TypePairAndMemDesc > TypesAndMemDesc)
The instruction is legal when type indexes 0 and 1 along with the memory size and minimum alignment i...
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
LegalizeResult lowerDynStackAlloc(MachineInstr &MI)
LegalizeResult lowerBitCount(MachineInstr &MI)
LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI)
Lower a vector extract or insert by writing the vector to a stack temporary and reloading the element...
const TargetLowering & getTargetLowering() const
LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
@ UnableToLegalize
Some kind of error has occurred and we could not legalize this instruction.
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
Register getDynStackAllocTargetPtr(Register SPReg, Register AllocSize, Align Alignment, LLT PtrTy)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
const LegacyLegalizerInfo & getLegacyLegalizerInfo() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ... = G_EXTRACT Src, Idx0.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMAX Op0, Op1.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildBitReverse(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITREVERSE Src.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMIN Op0, Op1.
MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op)
Build and insert Res = ExtOpc, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes of...
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FNEG Op0.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMIN Op0, Op1.
MachineInstrBuilder buildUMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMAX Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Wrapper class representing virtual and physical registers.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
const TargetMachine & getTargetMachine() const
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
Primary interface to the complete machine description for the target machine.
Target - Wrapper for Target specific information.
LLVM Value Representation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, AtomicOrdering Ordering)
True iff the specified MMO index has at an atomic ordering of at Ordering or stronger.
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min=0)
Add more elements to the type for the given type index to the next power of.
LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx)
Change the scalar size or element size to have the same scalar size as type index FromIndex.
operand_type_match m_Reg()
ConstantMatch< APInt > m_ICst(APInt &Cst)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
@ Undef
Value of the register doesn't matter.
This is an optimization pass for GlobalISel generic memory operations.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
AtomicOrdering
Atomic ordering for LLVM's memory model.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
unsigned Log2(Align A)
Returns the log2 of the alignment.
std::function< bool(const LegalityQuery &)> LegalityPredicate
This struct is a compact representation of a valid (non-zero power of two) alignment.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
This class contains a discriminated union of information about pointers in memory operands,...