32#include "llvm/IR/IntrinsicsLoongArch.h"
42#define DEBUG_TYPE "loongarch-isel-lowering"
57 cl::desc(
"Maximum number of instructions used (including code sequence "
58 "to generate the value and moving the value to FPR) when "
59 "materializing floating-point immediates (default = 3)"),
63 "Materialize FP immediate within 2 instructions"),
65 "Materialize FP immediate within 3 instructions"),
67 "Materialize FP immediate within 4 instructions"),
69 "Materialize FP immediate within 5 instructions"),
71 "Materialize FP immediate within 6 instructions "
72 "(behaves same as 5 on loongarch64)")));
75 cl::desc(
"Trap on integer division by zero."),
82 MVT GRLenVT = Subtarget.getGRLenVT();
87 if (Subtarget.hasBasicF())
89 if (Subtarget.hasBasicD())
93 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, MVT::v2f64};
95 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v8f32, MVT::v4f64};
97 if (Subtarget.hasExtLSX())
101 if (Subtarget.hasExtLASX())
102 for (
MVT VT : LASXVTs)
114 if (Subtarget.hasBasicF())
172 if (Subtarget.is64Bit()) {
200 if (!Subtarget.is64Bit()) {
206 if (Subtarget.hasBasicD())
218 if (Subtarget.hasBasicF()) {
252 if (Subtarget.is64Bit())
255 if (!Subtarget.hasBasicD()) {
257 if (Subtarget.is64Bit()) {
266 if (Subtarget.hasBasicD()) {
299 if (Subtarget.is64Bit())
305 if (Subtarget.hasExtLSX()) {
320 for (
MVT VT : LSXVTs) {
334 for (
MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
362 for (
MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
364 for (
MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
366 for (
MVT VT : {MVT::v4i32, MVT::v2i64}) {
372 for (
MVT VT : {MVT::v4f32, MVT::v2f64}) {
396 {MVT::v16i8, MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v8i16, MVT::v4i16,
397 MVT::v2i16, MVT::v4i32, MVT::v2i32, MVT::v2i64}) {
412 for (
MVT VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16})
414 for (
MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v16i32, MVT::v8i64,
421 if (Subtarget.hasExtLASX()) {
422 for (
MVT VT : LASXVTs) {
437 for (
MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) {
466 for (
MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
468 for (
MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64})
470 for (
MVT VT : {MVT::v8i32, MVT::v4i64})
472 for (
MVT VT : {MVT::v8i32, MVT::v4i32, MVT::v4i64}) {
477 for (
MVT VT : {MVT::v8f32, MVT::v4f64}) {
496 for (
MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16}) {
502 {MVT::v2i64, MVT::v4i32, MVT::v4i64, MVT::v8i16, MVT::v8i32}) {
506 for (
MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
511 if (Subtarget.hasBasicF()) {
531 if (Subtarget.has32S())
536 if (Subtarget.hasExtLSX()) {
552 if (Subtarget.hasExtLASX()) {
577 if (Subtarget.hasLAMCAS())
580 if (Subtarget.hasSCQ()) {
600 switch (
Op.getOpcode()) {
602 return lowerATOMIC_FENCE(
Op, DAG);
604 return lowerEH_DWARF_CFA(
Op, DAG);
606 return lowerGlobalAddress(
Op, DAG);
608 return lowerGlobalTLSAddress(
Op, DAG);
610 return lowerINTRINSIC_WO_CHAIN(
Op, DAG);
612 return lowerINTRINSIC_W_CHAIN(
Op, DAG);
614 return lowerINTRINSIC_VOID(
Op, DAG);
616 return lowerBlockAddress(
Op, DAG);
618 return lowerJumpTable(
Op, DAG);
620 return lowerShiftLeftParts(
Op, DAG);
622 return lowerShiftRightParts(
Op, DAG,
true);
624 return lowerShiftRightParts(
Op, DAG,
false);
626 return lowerConstantPool(
Op, DAG);
628 return lowerFP_TO_SINT(
Op, DAG);
630 return lowerFP_TO_UINT(
Op, DAG);
632 return lowerFP_TO_SINT_SAT(
Op, DAG);
634 return lowerBITCAST(
Op, DAG);
636 return lowerUINT_TO_FP(
Op, DAG);
638 return lowerSINT_TO_FP(
Op, DAG);
640 return lowerVASTART(
Op, DAG);
642 return lowerFRAMEADDR(
Op, DAG);
644 return lowerRETURNADDR(
Op, DAG);
646 return lowerSET_ROUNDING(
Op, DAG);
648 return lowerGET_ROUNDING(
Op, DAG);
650 return lowerWRITE_REGISTER(
Op, DAG);
652 return lowerINSERT_VECTOR_ELT(
Op, DAG);
654 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
656 return lowerBUILD_VECTOR(
Op, DAG);
658 return lowerCONCAT_VECTORS(
Op, DAG);
660 return lowerVECTOR_SHUFFLE(
Op, DAG);
662 return lowerBITREVERSE(
Op, DAG);
664 return lowerSCALAR_TO_VECTOR(
Op, DAG);
666 return lowerPREFETCH(
Op, DAG);
668 return lowerSELECT(
Op, DAG);
670 return lowerBRCOND(
Op, DAG);
672 return lowerFP_TO_FP16(
Op, DAG);
674 return lowerFP16_TO_FP(
Op, DAG);
676 return lowerFP_TO_BF16(
Op, DAG);
678 return lowerBF16_TO_FP(
Op, DAG);
680 return lowerVECREDUCE_ADD(
Op, DAG);
683 return lowerRotate(
Op, DAG);
691 return lowerVECREDUCE(
Op, DAG);
693 return lowerConstantFP(
Op, DAG);
695 return lowerSETCC(
Op, DAG);
697 return lowerFP_ROUND(
Op, DAG);
699 return lowerFP_EXTEND(
Op, DAG);
701 return lowerSIGN_EXTEND_VECTOR_INREG(
Op, DAG);
703 return lowerDYNAMIC_STACKALLOC(
Op, DAG);
705 return lowerANY_EXTEND(
Op, DAG);
707 return lowerVSELECT(
Op, DAG);
716 EVT VT =
V.getValueType();
722 return V.getOperand(0);
728 Not = DAG.
getBitcast(
V.getOperand(0).getValueType(), Not);
738 if (!
V->isOnlyUserOf(SplatValue.getNode()))
742 Not = DAG.
getBitcast(
V.getOperand(0).getValueType(), Not);
750 V.getOperand(0).hasOneUse() &&
V.getOperand(1).hasOneUse()) {
778 (
N->getOpcode() == LoongArchISD::VPACKEV) ||
779 (
N->getOpcode() == LoongArchISD::VPERMI)) &&
786 if (Opcode0 != Opcode1)
789 if (Opcode0 !=
ISD::FP_ROUND && Opcode0 != LoongArchISD::VFCVT)
796 EVT VT =
N.getValueType();
811 SSVT0 == MVT::v4f64) {
830 if ((
N->getOpcode() == LoongArchISD::VPACKEV ||
831 N->getOpcode() == LoongArchISD::VPERMI) &&
832 Opcode0 == LoongArchISD::VFCVT) {
837 if (!
Subtarget.hasExtLSX() || SVT0 != MVT::v4f32 || SSVT0 != MVT::v2f64)
840 if (
N->getOpcode() == LoongArchISD::VPACKEV &&
841 (VT == MVT::v2i64 || VT == MVT::v2f64)) {
847 if (
N->getOpcode() == LoongArchISD::VPERMI && VT == MVT::v4f32) {
863 MVT VT =
Op.getSimpleValueType();
864 MVT SVT =
In.getSimpleValueType();
866 if (VT == MVT::v4f32 && SVT == MVT::v4f64) {
879 EVT VT =
Op.getValueType();
881 EVT SVT = Src.getValueType();
884 VT == MVT::v2f64 && SVT == MVT::v2f32 && Subtarget.hasExtLSX();
886 VT == MVT::v4f64 && SVT == MVT::v4f32 && Subtarget.hasExtLASX();
887 if (!V2F32ToV2F64 && !V4F32ToV4F64)
914 if (
SDValue V = CheckVecHighPart(Src)) {
916 "Unexpected wide vector");
917 Opcode = LoongArchISD::VFCVTH;
920 Opcode = LoongArchISD::VFCVTL;
922 DAG.
getUNDEF(WideOpVT), Src, ZeroIdx);
927 return DAG.
getNode(Opcode,
DL, VT, VFCVTOp);
933 SmallVector<int, 8>
Mask = {0, 1, 4, 5, 2, 3, 6, 7};
948 MVT VT =
Op.getSimpleValueType();
957 APInt SplatValue, SplatUndef;
958 unsigned SplatBitSize;
963 if (BVN->
isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
965 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
970 EVT CondVT =
Cond.getValueType();
972 Mask.resize(NumElts, -1);
973 for (
int i = 0; i < NumElts; ++i) {
978 if (CondElt->getZExtValue() == 0)
979 Mask[i] = i + NumElts;
993 EVT VT =
Op.getValueType();
998 assert((VT == MVT::f32 && Subtarget.hasBasicF()) ||
999 (VT == MVT::f64 && Subtarget.hasBasicD()));
1017 if (VT == MVT::f32) {
1018 assert(Subtarget.is64Bit());
1021 return DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
DL, VT, NewVal);
1024 assert(VT == MVT::f64 && !Subtarget.is64Bit());
1027 return DAG.
getNode(LoongArchISD::MOVGR2FR_D_LO_HI,
DL, VT,
Lo,
Hi);
1035 EVT ResultVT =
Op.getValueType();
1036 EVT OperandVT =
Op.getOperand(0).getValueType();
1041 if (ResultVT == SetCCResultVT)
1044 assert(
Op.getOperand(0).getValueType() ==
Op.getOperand(1).getValueType() &&
1045 "SETCC operands must have the same type!");
1049 Op.getOperand(1),
Op.getOperand(2));
1051 if (ResultVT.
bitsGT(SetCCResultVT))
1053 else if (ResultVT.
bitsLT(SetCCResultVT))
1082SDValue LoongArchTargetLowering::lowerSIGN_EXTEND_VECTOR_INREG(
1086 MVT SrcVT = Src.getSimpleValueType();
1087 MVT DstVT =
Op.getSimpleValueType();
1093 unsigned Ratio = DstEltBits / SrcEltBits;
1102 if (Ratio == 2 && !IsSignSplat)
1103 return DAG.
getNode(LoongArchISD::VSLLWIL,
DL, DstVT, Src,
1107 for (
unsigned EltBits = SrcEltBits; EltBits < DstEltBits; EltBits *= 2) {
1110 Res = DAG.
getNode(LoongArchISD::VILVL,
DL, VT, Res, Res);
1122 assert(Subtarget.hasExtLASX());
1141 MVT OpVT =
Op.getSimpleValueType();
1148 unsigned LegalVecSize = 128;
1149 bool isLASX256Vector =
1159 if (isLASX256Vector) {
1165 for (
unsigned i = 1; i < NumEles; i *= 2, EleBits *= 2) {
1166 EleBits = std::min(EleBits, 64u);
1169 Val = DAG.
getNode(LoongArchISD::VHADDW,
DL, VecTy, Val, Val);
1172 if (isLASX256Vector) {
1198 MVT OpVT =
Op.getSimpleValueType();
1211 MVT GRLenVT = Subtarget.getGRLenVT();
1213 for (
int i = NumEles; i > 1; i /= 2) {
1216 Val = DAG.
getNode(Opcode,
DL, VecTy, Tmp, Val);
1225 unsigned IsData =
Op.getConstantOperandVal(4);
1230 return Op.getOperand(0);
1237 MVT VT =
Op.getSimpleValueType();
1243 unsigned Opcode =
Op.getOpcode();
1246 auto checkCstSplat = [](
SDValue V, APInt &CstSplatValue) {
1252 CstSplatValue =
C->getAPIntValue();
1260 APInt CstSplatValue;
1261 bool IsCstSplat = checkCstSplat(Amt, CstSplatValue);
1265 if (IsCstSplat && CstSplatValue.
urem(EltSizeInBits) == 0)
1281 return DAG.
getNode(Opcode,
DL, VT, R, Urem);
1297 if (
LHS == LHS2 &&
RHS == RHS2) {
1302 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
1310 return std::nullopt;
1318 MVT VT =
N->getSimpleValueType(0);
1349 if (~TrueVal == FalseVal) {
1389 unsigned SelOpNo = 0;
1399 unsigned ConstSelOpNo = 1;
1400 unsigned OtherSelOpNo = 2;
1407 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
1412 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
1418 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
1420 std::swap(NewConstOps[0], NewConstOps[1]);
1432 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
1434 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
1437 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
1438 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
1458 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
1472 int64_t
C = RHSC->getSExtValue();
1515 MVT VT =
Op.getSimpleValueType();
1516 MVT GRLenVT = Subtarget.getGRLenVT();
1521 if (
Op.hasOneUse()) {
1522 unsigned UseOpc =
Op->user_begin()->getOpcode();
1524 SDNode *BinOp = *
Op->user_begin();
1531 return lowerSELECT(NewSel, DAG);
1548 return DAG.
getNode(LoongArchISD::SELECT_CC,
DL, VT,
Ops);
1571 if (TrueVal - 1 == FalseVal)
1573 if (TrueVal + 1 == FalseVal)
1580 RHS == TrueV &&
LHS == FalseV) {
1605 return DAG.
getNode(LoongArchISD::SELECT_CC,
DL, VT,
Ops);
1612 MVT GRLenVT = Subtarget.getGRLenVT();
1623 return DAG.
getNode(LoongArchISD::BR_CC,
DL,
Op.getValueType(),
1624 Op.getOperand(0),
LHS,
RHS, TargetCC,
1627 return DAG.
getNode(LoongArchISD::BRCOND,
DL,
Op.getValueType(),
1628 Op.getOperand(0), CondV,
Op.getOperand(2));
1632 return DAG.
getNode(LoongArchISD::BR_CC,
DL,
Op.getValueType(),
1638LoongArchTargetLowering::lowerSCALAR_TO_VECTOR(
SDValue Op,
1641 MVT OpVT =
Op.getSimpleValueType();
1652 EVT ResTy =
Op->getValueType(0);
1657 if (!Subtarget.is64Bit() && (ResTy == MVT::v16i8 || ResTy == MVT::v32i8))
1667 for (
unsigned int i = 0; i < NewEltNum; i++) {
1670 unsigned RevOp = (ResTy == MVT::v16i8 || ResTy == MVT::v32i8)
1671 ? (
unsigned)LoongArchISD::BITREV_8B
1689 for (
unsigned int i = 0; i < NewEltNum; i++)
1690 for (
int j = OrigEltNum / NewEltNum - 1;
j >= 0;
j--)
1691 Mask.push_back(j + (OrigEltNum / NewEltNum) * i);
1709 if (EltBits > 32 || EltBits == 1)
1737 int MaskOffset,
const APInt &Zeroable) {
1738 int Size = Mask.size();
1739 unsigned SizeInBits =
Size * ScalarSizeInBits;
1741 auto CheckZeros = [&](
int Shift,
int Scale,
bool Left) {
1742 for (
int i = 0; i <
Size; i += Scale)
1743 for (
int j = 0; j < Shift; ++j)
1744 if (!Zeroable[i + j + (
Left ? 0 : (Scale - Shift))])
1752 for (
unsigned i = Pos, e = Pos +
Size; i != e; ++i,
Low += Step)
1753 if (!(Mask[i] == -1 || Mask[i] ==
Low))
1758 auto MatchShift = [&](
int Shift,
int Scale,
bool Left) {
1759 for (
int i = 0; i !=
Size; i += Scale) {
1760 unsigned Pos =
Left ? i + Shift : i;
1761 unsigned Low =
Left ? i : i + Shift;
1762 unsigned Len = Scale - Shift;
1767 int ShiftEltBits = ScalarSizeInBits * Scale;
1768 bool ByteShift = ShiftEltBits > 64;
1769 Opcode =
Left ? (ByteShift ? LoongArchISD::VBSLL : LoongArchISD::VSLLI)
1770 : (ByteShift ? LoongArchISD::VBSRL : LoongArchISD::VSRLI);
1771 int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
1775 Scale = ByteShift ? Scale / 2 : Scale;
1781 return (
int)ShiftAmt;
1784 unsigned MaxWidth = 128;
1785 for (
int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
1786 for (
int Shift = 1; Shift != Scale; ++Shift)
1787 for (
bool Left : {
true,
false})
1788 if (CheckZeros(Shift, Scale,
Left)) {
1789 int ShiftAmt = MatchShift(Shift, Scale,
Left);
1814 const APInt &Zeroable) {
1815 int Size = Mask.size();
1829 Mask,
Size, Zeroable);
1837 "Illegal integer vector type");
1846template <
typename ValType>
1849 unsigned CheckStride,
1851 ValType ExpectedIndex,
unsigned ExpectedIndexStride) {
1855 if (*
I != -1 && *
I != ExpectedIndex)
1857 ExpectedIndex += ExpectedIndexStride;
1861 for (
unsigned n = 0; n < CheckStride &&
I !=
End; ++n, ++
I)
1873 int Size = Mask.size();
1882 int VectorSizeInBits =
V1.getValueSizeInBits();
1883 int ScalarSizeInBits = VectorSizeInBits /
Size;
1884 assert(!(VectorSizeInBits % ScalarSizeInBits) &&
"Illegal shuffle mask size");
1885 (void)ScalarSizeInBits;
1887 for (
int i = 0; i <
Size; ++i) {
1893 if ((M >= 0 && M <
Size && V1IsZero) || (M >=
Size && V2IsZero)) {
1910 RepeatedMask.
assign(LaneSize, -1);
1911 int Size = Mask.size();
1912 for (
int i = 0; i <
Size; ++i) {
1913 assert(Mask[i] == -1 || Mask[i] >= 0);
1916 if ((Mask[i] %
Size) / LaneSize != i / LaneSize)
1923 Mask[i] <
Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
1924 if (RepeatedMask[i % LaneSize] < 0)
1926 RepeatedMask[i % LaneSize] = LocalM;
1927 else if (RepeatedMask[i % LaneSize] != LocalM)
1944 int NumElts = RepeatedMask.
size();
1946 int Scale = 16 / NumElts;
1948 for (
int i = 0; i < NumElts; ++i) {
1949 int M = RepeatedMask[i];
1950 assert((M == -1 || (0 <= M && M < (2 * NumElts))) &&
1951 "Unexpected mask index.");
1956 int StartIdx = i - (M % NumElts);
1963 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
1966 Rotation = CandidateRotation;
1967 else if (Rotation != CandidateRotation)
1982 else if (TargetV != MaskV)
1987 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
1988 assert((
Lo ||
Hi) &&
"Failed to find a rotated input vector!");
1997 return Rotation * Scale;
2016 if (ByteRotation <= 0)
2023 int LoByteShift = 16 - ByteRotation;
2024 int HiByteShift = ByteRotation;
2047 const APInt &Zeroable) {
2061 for (
int i = 0; i < NumElements; i++) {
2065 if (i % Scale != 0) {
2077 M = M % NumElements;
2080 Offset = M - (i / Scale);
2083 if (
Offset % (NumElements / Scale))
2085 }
else if (InputV !=
V)
2088 if (M != (
Offset + (i / Scale)))
2098 unsigned VilVLoHi = LoongArchISD::VILVL;
2099 if (
Offset >= (NumElements / 2)) {
2100 VilVLoHi = LoongArchISD::VILVH;
2101 Offset -= (NumElements / 2);
2109 InputV = DAG.
getNode(VilVLoHi,
DL, InputVT, Ext, InputV);
2113 }
while (Scale > 1);
2119 for (
int NumExtElements = Bits / 64; NumExtElements < NumElements;
2120 NumExtElements *= 2) {
2140 int SplatIndex = -1;
2141 for (
const auto &M : Mask) {
2148 if (SplatIndex == -1)
2151 assert(SplatIndex < (
int)Mask.size() &&
"Out of bounds mask index");
2153 return DAG.
getNode(LoongArchISD::VREPLVEI,
DL, VT,
V1,
2183 unsigned SubVecSize = 4;
2184 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2187 int SubMask[4] = {-1, -1, -1, -1};
2188 for (
unsigned i = 0; i < SubVecSize; ++i) {
2189 for (
unsigned j = i; j < Mask.size(); j += SubVecSize) {
2195 M -= 4 * (j / SubVecSize);
2196 if (M < 0 || M >= 4)
2202 if (SubMask[i] == -1)
2206 else if (M != -1 && M != SubMask[i])
2213 for (
int i = SubVecSize - 1; i >= 0; --i) {
2226 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2227 return DAG.
getNode(LoongArchISD::VSHUF4I_D,
DL, VT,
V1, V2,
2230 return DAG.
getNode(LoongArchISD::VSHUF4I,
DL, VT,
V1,
2248 if (VT != MVT::v16i8 && VT != MVT::v8i16 && VT != MVT::v32i8 &&
2257 for (
int i = 0; i < WidenNumElts; ++i)
2258 WidenMask[i] = WidenNumElts - 1 - i;
2266 return DAG.
getNode(LoongArchISD::VSHUF4I,
DL, VT,
2290 const auto &Begin = Mask.begin();
2291 const auto &
End = Mask.end();
2308 return DAG.
getNode(LoongArchISD::VPACKEV,
DL, VT, V2,
V1);
2330 const auto &Begin = Mask.begin();
2331 const auto &
End = Mask.end();
2348 return DAG.
getNode(LoongArchISD::VPACKOD,
DL, VT, V2,
V1);
2371 const auto &Begin = Mask.begin();
2372 const auto &
End = Mask.end();
2373 unsigned HalfSize = Mask.size() / 2;
2391 return DAG.
getNode(LoongArchISD::VILVH,
DL, VT, V2,
V1);
2414 const auto &Begin = Mask.begin();
2415 const auto &
End = Mask.end();
2432 return DAG.
getNode(LoongArchISD::VILVL,
DL, VT, V2,
V1);
2454 const auto &Begin = Mask.begin();
2455 const auto &Mid = Mask.begin() + Mask.size() / 2;
2456 const auto &
End = Mask.end();
2474 return DAG.
getNode(LoongArchISD::VPICKEV,
DL, VT, V2,
V1);
2496 const auto &Begin = Mask.begin();
2497 const auto &Mid = Mask.begin() + Mask.size() / 2;
2498 const auto &
End = Mask.end();
2515 return DAG.
getNode(LoongArchISD::VPICKOD,
DL, VT, V2,
V1);
2541 if (Mask.size() != NumElts)
2544 auto tryLowerToExtrAndIns = [&](
unsigned Base) ->
SDValue {
2547 for (
unsigned i = 0; i < NumElts; ++i) {
2550 if (Mask[i] !=
int(
Base + i)) {
2563 int DiffMask = Mask[DiffPos];
2564 if (DiffMask < 0 || DiffMask >=
int(2 * NumElts))
2570 if (
unsigned(DiffMask) < NumElts) {
2575 SrcIdx =
unsigned(DiffMask) - NumElts;
2591 if (
SDValue Result = tryLowerToExtrAndIns(0))
2593 return tryLowerToExtrAndIns(NumElts);
2601 unsigned &MaskImm) {
2602 unsigned MaskSize = Mask.size();
2605 return (M == -1) || (M >=
Off && M <
Off + 4);
2608 auto buildImm = [&](
int MLo,
int MHi,
unsigned Off,
unsigned I) {
2609 auto immPart = [&](
int M,
unsigned Off) {
2610 return (M == -1 ? 0 : (M -
Off)) & 0x3;
2612 MaskImm |= immPart(MLo,
Off) << (
I * 2);
2613 MaskImm |= immPart(MHi,
Off) << ((
I + 1) * 2);
2616 for (
unsigned i = 0; i < 4; i += 2) {
2618 int MHi = Mask[i + 1];
2620 if (MaskSize == 8) {
2621 auto isValid2 = [&](
int &M,
int M2) {
2628 if ((M2 % MaskSize) < 4)
2636 if (!isValid2(MLo, Mask[i + 4]) || !isValid2(MHi, Mask[i + 5]))
2642 buildImm(MLo, MHi, 0, i);
2645 buildImm(MLo, MHi, MaskSize, i);
2675 if ((VT != MVT::v4i32 && VT != MVT::v4f32) ||
2680 unsigned MaskImm = 0;
2684 return DAG.
getNode(LoongArchISD::VPERMI,
DL, VT, SrcVec[1], SrcVec[0],
2711 return DAG.
getNode(LoongArchISD::VSHUF,
DL, VT, MaskVec, V2,
V1);
2724 "Vector type is unsupported for lsx!");
2726 "Two operands have different types!");
2728 "Unexpected mask size for shuffle!");
2729 assert(Mask.size() % 2 == 0 &&
"Expected even mask size.");
2731 APInt KnownUndef, KnownZero;
2733 APInt Zeroable = KnownUndef | KnownZero;
2806 int SplatIndex = -1;
2807 for (
const auto &M : Mask) {
2814 if (SplatIndex == -1)
2817 const auto &Begin = Mask.begin();
2818 const auto &
End = Mask.end();
2819 int HalfSize = Mask.size() / 2;
2821 if (SplatIndex >= HalfSize)
2824 assert(SplatIndex < (
int)Mask.size() &&
"Out of bounds mask index");
2828 return DAG.
getNode(LoongArchISD::VREPLVEI,
DL, VT,
V1,
2842 if (Mask.size() == 4) {
2843 unsigned MaskImm = 0;
2844 for (
int i = 1; i >= 0; --i) {
2846 int MHi = Mask[i + 2];
2847 if (!(MLo == -1 || (MLo >= 0 && MLo <= 1) || (MLo >= 4 && MLo <= 5)) ||
2848 !(MHi == -1 || (MHi >= 2 && MHi <= 3) || (MHi >= 6 && MHi <= 7)))
2850 if (MHi != -1 && MLo != -1 && MHi != MLo + 2)
2855 MaskImm |= ((MLo <= 1) ? MLo : (MLo - 2)) & 0x3;
2857 MaskImm |= ((MHi <= 3) ? (MHi - 2) : (MHi - 4)) & 0x3;
2860 return DAG.
getNode(LoongArchISD::VSHUF4I_D,
DL, VT,
V1, V2,
2881 return DAG.
getNode(LoongArchISD::XVREPLVE0,
DL, VT,
V1);
2890 unsigned MaskSize = Mask.size();
2895 if (VT == MVT::v8i32 || VT == MVT::v8f32) {
2897 unsigned MaskImm = 0;
2901 return DAG.
getNode(LoongArchISD::VPERMI,
DL, VT, SrcVec[1], SrcVec[0],
2906 if (VT == MVT::v4i64 || VT == MVT::v4f64) {
2907 unsigned MaskImm = 0;
2908 for (
unsigned i = 0; i < MaskSize; ++i) {
2911 if (Mask[i] >= (
int)MaskSize)
2913 MaskImm |= Mask[i] << (i * 2);
2916 return DAG.
getNode(LoongArchISD::XVPERMI,
DL, VT,
V1,
2928 if (Mask.size() != 8 || (VT != MVT::v8i32 && VT != MVT::v8f32))
2932 unsigned HalfSize = NumElts / 2;
2933 bool FrontLo =
true, FrontHi =
true;
2934 bool BackLo =
true, BackHi =
true;
2936 auto inRange = [](
int val,
int low,
int high) {
2937 return (val == -1) || (val >= low && val < high);
2940 for (
unsigned i = 0; i < HalfSize; ++i) {
2941 int Fronti = Mask[i];
2942 int Backi = Mask[i + HalfSize];
2944 FrontLo &=
inRange(Fronti, 0, HalfSize);
2945 FrontHi &=
inRange(Fronti, HalfSize, NumElts);
2946 BackLo &=
inRange(Backi, 0, HalfSize);
2947 BackHi &=
inRange(Backi, HalfSize, NumElts);
2953 if ((FrontLo || FrontHi) && (BackLo || BackHi))
2958 for (
unsigned i = 0; i < NumElts; ++i)
2963 return DAG.
getNode(LoongArchISD::XVPERM,
DL, VT,
V1, MaskVec);
2985 const auto &Begin = Mask.begin();
2986 const auto &
End = Mask.end();
2987 unsigned HalfSize = Mask.size() / 2;
2988 unsigned LeftSize = HalfSize / 2;
2996 Mask.size() + HalfSize - LeftSize, 1) &&
2998 Mask.size() + HalfSize + LeftSize, 1))
3009 Mask.size() + HalfSize - LeftSize, 1) &&
3011 Mask.size() + HalfSize + LeftSize, 1))
3016 return DAG.
getNode(LoongArchISD::VILVH,
DL, VT, V2,
V1);
3024 const auto &Begin = Mask.begin();
3025 const auto &
End = Mask.end();
3026 unsigned HalfSize = Mask.size() / 2;
3034 Mask.size() + HalfSize, 1))
3045 Mask.size() + HalfSize, 1))
3050 return DAG.
getNode(LoongArchISD::VILVL,
DL, VT, V2,
V1);
3058 const auto &Begin = Mask.begin();
3059 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
3060 const auto &Mid = Mask.begin() + Mask.size() / 2;
3061 const auto &RightMid = Mask.end() - Mask.size() / 4;
3062 const auto &
End = Mask.end();
3063 unsigned HalfSize = Mask.size() / 2;
3085 return DAG.
getNode(LoongArchISD::VPICKEV,
DL, VT, V2,
V1);
3093 const auto &Begin = Mask.begin();
3094 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
3095 const auto &Mid = Mask.begin() + Mask.size() / 2;
3096 const auto &RightMid = Mask.end() - Mask.size() / 4;
3097 const auto &
End = Mask.end();
3098 unsigned HalfSize = Mask.size() / 2;
3121 return DAG.
getNode(LoongArchISD::VPICKOD,
DL, VT, V2,
V1);
3130 int HalfSize = NumElts / 2;
3134 if ((
int)Mask.size() != NumElts)
3137 auto tryLowerToExtrAndIns = [&](
int Base) ->
SDValue {
3139 for (
int i = 0; i < NumElts; ++i) {
3142 if (Mask[i] !=
Base + i) {
3144 if (DiffPos.
size() > 2)
3152 if (DiffPos.
size() == 1) {
3153 if (DiffPos[0] < HalfSize && Mask[DiffPos[0] + HalfSize] == -1)
3154 DiffPos.
push_back(DiffPos[0] + HalfSize);
3155 else if (DiffPos[0] >= HalfSize && Mask[DiffPos[0] - HalfSize] == -1)
3156 DiffPos.
insert(DiffPos.
begin(), DiffPos[0] - HalfSize);
3160 if (DiffPos.
size() != 2 || DiffPos[1] != DiffPos[0] + HalfSize)
3164 int DiffMaskLo = Mask[DiffPos[0]];
3165 int DiffMaskHi = Mask[DiffPos[1]];
3166 DiffMaskLo = DiffMaskLo == -1 ? DiffMaskHi - HalfSize : DiffMaskLo;
3167 DiffMaskHi = DiffMaskHi == -1 ? DiffMaskLo + HalfSize : DiffMaskHi;
3168 if (!(DiffMaskLo >= 0 && DiffMaskLo < HalfSize) &&
3169 !(DiffMaskLo >= NumElts && DiffMaskLo < NumElts + HalfSize))
3171 if (!(DiffMaskHi >= HalfSize && DiffMaskHi < NumElts) &&
3172 !(DiffMaskHi >= NumElts + HalfSize && DiffMaskHi < 2 * NumElts))
3174 if (DiffMaskHi != DiffMaskLo + HalfSize)
3178 SDValue SrcVec = (DiffMaskLo < HalfSize) ?
V1 : V2;
3180 (DiffMaskLo < HalfSize) ? DiffMaskLo : (DiffMaskLo - NumElts);
3201 if (
SDValue Result = tryLowerToExtrAndIns(0))
3203 return tryLowerToExtrAndIns(NumElts);
3212 if (VT != MVT::v8i32 && VT != MVT::v8f32 && VT != MVT::v4i64 &&
3217 int MaskSize = Mask.size();
3223 auto checkReplaceOne = [&](
int Base,
int Replaced) ->
int {
3225 for (
int i = 0; i < MaskSize; ++i) {
3226 if (Mask[i] ==
Base + i || Mask[i] == -1)
3228 if (Mask[i] != Replaced)
3239 int Idx = checkReplaceOne(0, MaskSize);
3241 return DAG.
getNode(LoongArchISD::XVINSVE0,
DL, VT,
V1, V2,
3245 Idx = checkReplaceOne(MaskSize, 0);
3247 return DAG.
getNode(LoongArchISD::XVINSVE0,
DL, VT, V2,
V1,
3258 int MaskSize = Mask.size();
3259 int HalfSize = Mask.size() / 2;
3260 const auto &Begin = Mask.begin();
3261 const auto &Mid = Mask.begin() + HalfSize;
3262 const auto &
End = Mask.end();
3274 for (
auto it = Begin; it < Mid; it++) {
3277 else if ((*it >= 0 && *it < HalfSize) ||
3278 (*it >= MaskSize && *it < MaskSize + HalfSize)) {
3279 int M = *it < HalfSize ? *it : *it - HalfSize;
3284 assert((
int)MaskAlloc.
size() == HalfSize &&
"xvshuf convert failed!");
3286 for (
auto it = Mid; it <
End; it++) {
3289 else if ((*it >= HalfSize && *it < MaskSize) ||
3290 (*it >= MaskSize + HalfSize && *it < MaskSize * 2)) {
3291 int M = *it < MaskSize ? *it - HalfSize : *it - MaskSize;
3296 assert((
int)MaskAlloc.
size() == MaskSize &&
"xvshuf convert failed!");
3300 return DAG.
getNode(LoongArchISD::VSHUF,
DL, VT, MaskVec, V2,
V1);
3328 enum HalfMaskType { HighLaneTy, LowLaneTy,
None };
3330 int MaskSize = Mask.size();
3331 int HalfSize = Mask.size() / 2;
3334 HalfMaskType preMask =
None, postMask =
None;
3336 if (std::all_of(Mask.begin(), Mask.begin() + HalfSize, [&](
int M) {
3337 return M < 0 || (M >= 0 && M < HalfSize) ||
3338 (M >= MaskSize && M < MaskSize + HalfSize);
3340 preMask = HighLaneTy;
3341 else if (std::all_of(Mask.begin(), Mask.begin() + HalfSize, [&](
int M) {
3342 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3343 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3345 preMask = LowLaneTy;
3347 if (std::all_of(Mask.begin() + HalfSize, Mask.end(), [&](
int M) {
3348 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3349 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3351 postMask = LowLaneTy;
3352 else if (std::all_of(Mask.begin() + HalfSize, Mask.end(), [&](
int M) {
3353 return M < 0 || (M >= 0 && M < HalfSize) ||
3354 (M >= MaskSize && M < MaskSize + HalfSize);
3356 postMask = HighLaneTy;
3364 if (preMask == HighLaneTy && postMask == LowLaneTy) {
3367 if (preMask == LowLaneTy && postMask == HighLaneTy) {
3375 V2 = DAG.
getNode(LoongArchISD::XVPERMI,
DL, MVT::v4i64, V2,
3380 for (
auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3381 *it = *it < 0 ? *it : *it - HalfSize;
3383 for (
auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3384 *it = *it < 0 ? *it : *it + HalfSize;
3386 }
else if (preMask == LowLaneTy && postMask == LowLaneTy) {
3394 V2 = DAG.
getNode(LoongArchISD::XVPERMI,
DL, MVT::v4i64, V2,
3399 for (
auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3400 *it = *it < 0 ? *it : *it - HalfSize;
3402 }
else if (preMask == HighLaneTy && postMask == HighLaneTy) {
3410 V2 = DAG.
getNode(LoongArchISD::XVPERMI,
DL, MVT::v4i64, V2,
3415 for (
auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3416 *it = *it < 0 ? *it : *it + HalfSize;
3443 int Lo = LaneMask[0] < 0 ? 0 : LaneMask[0];
3444 int Hi = LaneMask[1] < 0 ? 1 : LaneMask[1];
3446 LoongArchISD::XVPERMIQ,
DL, VT, V2,
V1,
3479 int Size = Mask.size();
3480 int LaneSize =
Size / 2;
3484 int LaneSrcs[2][2] = {{-1, -1}, {-1, -1}};
3486 bool LaneCrossing =
false;
3487 for (
int i = 0; i <
Size; ++i) {
3491 int Lane = i / LaneSize;
3492 int SrcLane = M / LaneSize;
3496 int Op = (SrcLane % 2) != Lane;
3498 if (LaneSrcs[Lane][
Op] >= 0 && LaneSrcs[Lane][
Op] != SrcLane)
3500 if (LaneSrcs[Lane][
Op] >= 0 && LaneSrcs[Lane][
Op] != SrcLane)
3502 LaneSrcs[Lane][
Op] = SrcLane;
3503 InLaneMask[i] =
Op *
Size + Lane * LaneSize + M % LaneSize;
3514 for (
int i = LaneSize; i <
Size; ++i)
3515 if (SwappedMask[i] >= 0)
3516 SwappedMask[i] = (SwappedMask[i] +
Size) % (2 *
Size);
3518 std::swap(LaneSrcs[1][0], LaneSrcs[1][1]);
3519 InLaneMask = SwappedMask;
3525 for (
int Op = 0;
Op < 2; ++
Op) {
3527 for (
int Lane = 0; Lane < 2; ++Lane)
3528 if (LaneSrcs[Lane][
Op] >= 0)
3529 for (
int i = 0; i < LaneSize; ++i)
3530 LaneMask[Lane * LaneSize + i] = LaneSrcs[Lane][
Op] * LaneSize + i;
3536 if (SVN->getMask() == Mask)
3552 "Vector type is unsupported for lasx!");
3554 "Two operands have different types!");
3556 "Unexpected mask size for shuffle!");
3557 assert(Mask.size() % 2 == 0 &&
"Expected even mask size.");
3558 assert(Mask.size() >= 4 &&
"Mask size is less than 4.");
3560 APInt KnownUndef, KnownZero;
3562 APInt Zeroable = KnownUndef | KnownZero;
3653 ArrayRef<int> OrigMask = SVOp->
getMask();
3656 MVT VT =
Op.getSimpleValueType();
3660 bool V1IsUndef =
V1.isUndef();
3661 bool V2IsUndef = V2.
isUndef();
3662 if (V1IsUndef && V2IsUndef)
3675 any_of(OrigMask, [NumElements](
int M) {
return M >= NumElements; })) {
3676 SmallVector<int, 8> NewMask(OrigMask);
3677 for (
int &M : NewMask)
3678 if (M >= NumElements)
3684 int MaskUpperLimit = OrigMask.
size() * (V2IsUndef ? 1 : 2);
3685 (void)MaskUpperLimit;
3687 [&](
int M) {
return -1 <=
M &&
M < MaskUpperLimit; }) &&
3688 "Out of bounds shuffle index");
3710 std::tie(Res, Chain) =
3711 makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions,
DL, Chain);
3712 if (Subtarget.is64Bit())
3713 return DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Res);
3725 SDValue Arg = Subtarget.is64Bit() ? DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
3729 std::tie(Res, Chain) =
makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
3730 CallOptions,
DL, Chain);
3736 assert(Subtarget.hasBasicF() &&
"Unexpected custom legalization");
3742 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
3743 if (Subtarget.is64Bit())
3744 return DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Res);
3750 assert(Subtarget.hasBasicF() &&
"Unexpected custom legalization");
3751 MVT VT =
Op.getSimpleValueType();
3756 SDValue Res = Subtarget.is64Bit() ? DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
3777 "Unsupported vector type for broadcast.");
3780 bool IsIdeneity =
true;
3782 for (
int i = 0; i !=
NumOps; i++) {
3784 if (
Op.getOpcode() !=
ISD::LOAD || (IdentitySrc &&
Op != IdentitySrc)) {
3796 auto ExtType = LN->getExtensionType();
3801 assert(LN->isUnindexed() &&
"Unexpected indexed load.");
3806 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
3824 for (
unsigned i = 1; i <
Ops.size(); ++i) {
3838 EVT ResTy,
unsigned first) {
3841 assert(first + NumElts <= Node->getSimpleValueType(0).getVectorNumElements());
3844 Node->op_begin() + first + NumElts);
3853 MVT VT =
Node->getSimpleValueType(0);
3854 EVT ResTy =
Op->getValueType(0);
3857 APInt SplatValue, SplatUndef;
3858 unsigned SplatBitSize;
3861 bool UseSameConstant =
true;
3866 if ((!Subtarget.hasExtLSX() || !Is128Vec) &&
3867 (!Subtarget.hasExtLASX() || !Is256Vec))
3873 if (
Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
3875 SplatBitSize <= 64) {
3877 if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&
3881 if (SplatBitSize == 64 && !Subtarget.is64Bit()) {
3888 if ((Is128Vec && ResTy == MVT::v4i32) ||
3889 (Is256Vec && ResTy == MVT::v8i32))
3895 switch (SplatBitSize) {
3899 ViaVecTy = Is128Vec ? MVT::v16i8 : MVT::v32i8;
3902 ViaVecTy = Is128Vec ? MVT::v8i16 : MVT::v16i16;
3905 ViaVecTy = Is128Vec ? MVT::v4i32 : MVT::v8i32;
3908 ViaVecTy = Is128Vec ? MVT::v2i64 : MVT::v4i64;
3916 if (ViaVecTy != ResTy)
3925 for (
unsigned i = 0; i < NumElts; ++i) {
3930 ConstantValue = Opi;
3931 else if (ConstantValue != Opi)
3932 UseSameConstant =
false;
3937 if (IsConstant && UseSameConstant && ResTy != MVT::v2f64) {
3939 for (
unsigned i = 0; i < NumElts; ++i) {
3950 BitVector UndefElements;
3951 bool IsRepeated =
Node->getRepeatedSequence(Sequence, &UndefElements);
3953 unsigned NumUndefElts = UndefElements.
count();
3957 if (IsRepeated && SeqLen == 1) {
3958 bool IsSplatBetter =
true;
3966 IsSplatBetter =
false;
3975 if (NumUndefElts == NumElts - 1)
3976 IsSplatBetter =
false;
3977 if (NumUndefElts == NumElts - 2 && !UndefElements[0])
3978 IsSplatBetter =
false;
3979 if (ResTy == MVT::v8f32 && NumUndefElts == NumElts - 2 &&
3980 ((!UndefElements[1] && !UndefElements[2]) ||
3981 (!UndefElements[1] && !UndefElements[3]) ||
3982 (!UndefElements[2] && !UndefElements[3])))
3983 IsSplatBetter =
false;
3997 if (IsRepeated && NumUndefElts == 0) {
4001 EVT FillTy = Is256Vec
4007 fillVector(Sequence, DAG,
DL, Subtarget, FillVec, FillTy);
4009 unsigned SplatLen = NumElts / SeqLen;
4015 if (SplatEltTy == MVT::i128)
4016 SplatTy = MVT::v4i64;
4024 DAG.
getNode((SplatEltTy == MVT::i128) ? LoongArchISD::XVREPLVE0Q
4025 : LoongArchISD::XVREPLVE0,
4026 DL, SplatTy, SrcVec);
4028 SplatVec = DAG.
getNode(LoongArchISD::VREPLVEI,
DL, SplatTy, SrcVec,
4041 if (ResTy == MVT::v8i32 || ResTy == MVT::v8f32 || ResTy == MVT::v4i64 ||
4042 ResTy == MVT::v4f64) {
4043 unsigned NonUndefCount = 0;
4044 for (
unsigned i = NumElts / 2; i < NumElts; ++i) {
4045 if (!
Node->getOperand(i).isUndef()) {
4047 if (NonUndefCount > 1)
4051 if (NonUndefCount == 1)
4064 VecTy, NumElts / 2);
4075 MVT ResVT =
Op.getSimpleValueType();
4083 unsigned NumFreezeUndef = 0;
4084 unsigned NumZero = 0;
4085 unsigned NumNonZero = 0;
4086 unsigned NonZeros = 0;
4087 SmallSet<SDValue, 4> Undefs;
4088 for (
unsigned i = 0; i != NumOperands; ++i) {
4103 assert(i <
sizeof(NonZeros) * CHAR_BIT);
4110 if (NumNonZero > 2) {
4114 Ops.slice(0, NumOperands / 2));
4116 Ops.slice(NumOperands / 2));
4129 MVT SubVT =
Op.getOperand(0).getSimpleValueType();
4131 for (
unsigned i = 0; i != NumOperands; ++i) {
4132 if ((NonZeros & (1 << i)) == 0)
4143LoongArchTargetLowering::lowerEXTRACT_VECTOR_ELT(
SDValue Op,
4145 MVT EltVT =
Op.getSimpleValueType();
4150 MVT GRLenVT = Subtarget.getGRLenVT();
4178 ? DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
DL, MVT::f32, Idx)
4182 DAG.
getBitcast((VecTy == MVT::v4f64) ? MVT::v4i64 : VecTy, IdxVec);
4184 DAG.
getNode(LoongArchISD::VSHUF,
DL, VecTy, MaskVec, TmpVec, Vec);
4193 DAG.
getNode(LoongArchISD::XVPERM,
DL, VecTy, Vec, SplatIdx);
4202LoongArchTargetLowering::lowerINSERT_VECTOR_ELT(
SDValue Op,
4204 MVT VT =
Op.getSimpleValueType();
4226 auto ExtType = LN->getExtensionType();
4227 if (LN->isSimple() &&
4229 LN->getMemoryVT().getSizeInBits() == EltSizeInBits) {
4231 assert(LN->isUnindexed() &&
"Unexpected indexed load.");
4233 if (EltSizeInBits >= 32) {
4238 FltVT,
DL, LN->getChain(), LN->getBasePtr(), LN->getPointerInfo(),
4239 LN->getAlign(), LN->getMemOperand()->getFlags(), LN->getAAInfo());
4245 SDVTList Tys = DAG.
getVTList(VT, MVT::Other);
4246 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
4248 LN->getMemoryVT(), LN->getMemOperand());
4269 if (!Subtarget.is64Bit() && IdxTy == MVT::i64) {
4271 for (
unsigned i = 0; i < NumElts; ++i) {
4279 for (
unsigned i = 0; i < NumElts; ++i) {
4288 for (
unsigned i = 0; i < NumElts; ++i)
4341 MVT GRLenVT = Subtarget.getGRLenVT();
4353 "rounding mode is not supported by LoongArch hardware",
4354 DiagnosticLocation(
DL.getDebugLoc()),
DS_Error));
4371 FCSRNo, RMValue, Chain);
4377 MVT GRLenVT = Subtarget.getGRLenVT();
4384 MVT::Other, FCSRNo, Chain);
4400 if (Subtarget.is64Bit() &&
Op.getOperand(2).getValueType() == MVT::i32) {
4402 "On LA64, only 64-bit registers can be written.");
4403 return Op.getOperand(0);
4406 if (!Subtarget.is64Bit() &&
Op.getOperand(2).getValueType() == MVT::i64) {
4408 "On LA32, only 32-bit registers can be written.");
4409 return Op.getOperand(0);
4419 "be a constant integer");
4425 Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF);
4426 EVT VT =
Op.getValueType();
4429 unsigned Depth =
Op.getConstantOperandVal(0);
4430 int GRLenInBytes = Subtarget.getGRLen() / 8;
4433 int Offset = -(GRLenInBytes * 2);
4445 if (
Op.getConstantOperandVal(0) != 0) {
4447 "return address can only be determined for the current frame");
4453 MVT GRLenVT = Subtarget.getGRLenVT();
4465 auto Size = Subtarget.getGRLen() / 8;
4473 auto *FuncInfo = MF.
getInfo<LoongArchMachineFunctionInfo>();
4483 MachinePointerInfo(SV));
4501 EVT SrcVT = Src.getValueType();
4502 EVT DstVT =
Op.getValueType();
4533 EVT VT =
Op.getValueType();
4543 if (Subtarget.hasExtLASX() && Op0VT.
getScalarType() == MVT::i64 &&
4547 return lowerUINT_TO_FPToF32(
Op, DAG);
4559 if (Subtarget.hasExtLSX() && Op0VT == MVT::i64 && VT == MVT::f64) {
4567 if (!Subtarget.is64Bit() || !Subtarget.hasBasicF() || Subtarget.hasBasicD())
4570 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4571 !Subtarget.hasBasicD() &&
"unexpected target features");
4575 if (
C &&
C->getZExtValue() < UINT64_C(0xFFFFFFFF))
4579 if (Op0->
getOpcode() == LoongArchISD::BSTRPICK &&
4589 EVT RetVT =
Op.getValueType();
4595 std::tie(Result, Chain) =
4602 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4603 !Subtarget.hasBasicD() &&
"unexpected target features");
4614 EVT RetVT =
Op.getValueType();
4620 std::tie(Result, Chain) =
4629 EVT VT =
Op.getValueType();
4633 if (
Op.getValueType() == MVT::f32 && Op0VT == MVT::i32 &&
4634 Subtarget.is64Bit() && Subtarget.hasBasicF()) {
4636 return DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
DL, MVT::f32, NewOp0);
4638 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit()) {
4641 return DAG.
getNode(LoongArchISD::BUILD_PAIR_F64,
DL, MVT::f64,
Lo,
Hi);
4655 if (
Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() &&
4656 !Subtarget.hasBasicD()) {
4658 return DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Dst);
4673 EVT VT =
Op.getValueType();
4674 EVT SrcVT = Src.getValueType();
4675 unsigned SatWidth =
cast<VTSDNode>(
Op.getOperand(1))->getVT().getSizeInBits();
4685 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4688 if (SatWidth == 32) {
4690 if (Subtarget.is64Bit())
4691 return DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, VT, Ftint);
4695 if (SatWidth == 64 && Subtarget.is64Bit() && Subtarget.hasBasicD()) {
4705 if (!Subtarget.hasExtLSX())
4710 EVT VT =
Op.getValueType();
4711 EVT SrcVT = Src.getValueType();
4716 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4719 if (SrcVT == MVT::f32)
4741 N->getOffset(), Flags);
4749template <
class NodeTy>
4752 bool IsLocal)
const {
4763 assert(Subtarget.is64Bit() &&
"Large code model requires LA64");
4844 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
4846 const GlobalValue *GV =
N->getGlobal();
4858 unsigned Opc,
bool UseGOT,
4862 MVT GRLenVT = Subtarget.getGRLenVT();
4876 if (
Opc == LoongArch::PseudoLA_TLS_LE && !Large)
4917 TargetLowering::CallLoweringInfo CLI(DAG);
4932 const GlobalValue *GV =
N->getGlobal();
4946LoongArchTargetLowering::lowerGlobalTLSAddress(
SDValue Op,
4953 assert((!Large || Subtarget.is64Bit()) &&
"Large code model requires LA64");
4956 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
4969 return getDynamicTLSAddr(
N, DAG,
4970 Large ? LoongArch::PseudoLA_TLS_GD_LARGE
4971 : LoongArch::PseudoLA_TLS_GD,
4978 return getDynamicTLSAddr(
N, DAG,
4979 Large ? LoongArch::PseudoLA_TLS_LD_LARGE
4980 : LoongArch::PseudoLA_TLS_LD,
4985 return getStaticTLSAddr(
N, DAG,
4986 Large ? LoongArch::PseudoLA_TLS_IE_LARGE
4987 : LoongArch::PseudoLA_TLS_IE,
4994 return getStaticTLSAddr(
N, DAG, LoongArch::PseudoLA_TLS_LE,
4998 return getTLSDescAddr(
N, DAG,
4999 Large ? LoongArch::PseudoLA_TLS_DESC_LARGE
5000 : LoongArch::PseudoLA_TLS_DESC,
5004template <
unsigned N>
5009 if ((IsSigned && !
isInt<N>(CImm->getSExtValue())) ||
5010 (!IsSigned && !
isUInt<N>(CImm->getZExtValue()))) {
5012 ": argument out of range.");
5019LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(
SDValue Op,
5021 switch (
Op.getConstantOperandVal(0)) {
5024 case Intrinsic::thread_pointer: {
5028 case Intrinsic::loongarch_lsx_vpickve2gr_d:
5029 case Intrinsic::loongarch_lsx_vpickve2gr_du:
5030 case Intrinsic::loongarch_lsx_vreplvei_d:
5031 case Intrinsic::loongarch_lasx_xvrepl128vei_d:
5033 case Intrinsic::loongarch_lsx_vreplvei_w:
5034 case Intrinsic::loongarch_lasx_xvrepl128vei_w:
5035 case Intrinsic::loongarch_lasx_xvpickve2gr_d:
5036 case Intrinsic::loongarch_lasx_xvpickve2gr_du:
5037 case Intrinsic::loongarch_lasx_xvpickve_d:
5038 case Intrinsic::loongarch_lasx_xvpickve_d_f:
5040 case Intrinsic::loongarch_lasx_xvinsve0_d:
5042 case Intrinsic::loongarch_lsx_vsat_b:
5043 case Intrinsic::loongarch_lsx_vsat_bu:
5044 case Intrinsic::loongarch_lsx_vrotri_b:
5045 case Intrinsic::loongarch_lsx_vsllwil_h_b:
5046 case Intrinsic::loongarch_lsx_vsllwil_hu_bu:
5047 case Intrinsic::loongarch_lsx_vsrlri_b:
5048 case Intrinsic::loongarch_lsx_vsrari_b:
5049 case Intrinsic::loongarch_lsx_vreplvei_h:
5050 case Intrinsic::loongarch_lasx_xvsat_b:
5051 case Intrinsic::loongarch_lasx_xvsat_bu:
5052 case Intrinsic::loongarch_lasx_xvrotri_b:
5053 case Intrinsic::loongarch_lasx_xvsllwil_h_b:
5054 case Intrinsic::loongarch_lasx_xvsllwil_hu_bu:
5055 case Intrinsic::loongarch_lasx_xvsrlri_b:
5056 case Intrinsic::loongarch_lasx_xvsrari_b:
5057 case Intrinsic::loongarch_lasx_xvrepl128vei_h:
5058 case Intrinsic::loongarch_lasx_xvpickve_w:
5059 case Intrinsic::loongarch_lasx_xvpickve_w_f:
5061 case Intrinsic::loongarch_lasx_xvinsve0_w:
5063 case Intrinsic::loongarch_lsx_vsat_h:
5064 case Intrinsic::loongarch_lsx_vsat_hu:
5065 case Intrinsic::loongarch_lsx_vrotri_h:
5066 case Intrinsic::loongarch_lsx_vsllwil_w_h:
5067 case Intrinsic::loongarch_lsx_vsllwil_wu_hu:
5068 case Intrinsic::loongarch_lsx_vsrlri_h:
5069 case Intrinsic::loongarch_lsx_vsrari_h:
5070 case Intrinsic::loongarch_lsx_vreplvei_b:
5071 case Intrinsic::loongarch_lasx_xvsat_h:
5072 case Intrinsic::loongarch_lasx_xvsat_hu:
5073 case Intrinsic::loongarch_lasx_xvrotri_h:
5074 case Intrinsic::loongarch_lasx_xvsllwil_w_h:
5075 case Intrinsic::loongarch_lasx_xvsllwil_wu_hu:
5076 case Intrinsic::loongarch_lasx_xvsrlri_h:
5077 case Intrinsic::loongarch_lasx_xvsrari_h:
5078 case Intrinsic::loongarch_lasx_xvrepl128vei_b:
5080 case Intrinsic::loongarch_lsx_vsrlni_b_h:
5081 case Intrinsic::loongarch_lsx_vsrani_b_h:
5082 case Intrinsic::loongarch_lsx_vsrlrni_b_h:
5083 case Intrinsic::loongarch_lsx_vsrarni_b_h:
5084 case Intrinsic::loongarch_lsx_vssrlni_b_h:
5085 case Intrinsic::loongarch_lsx_vssrani_b_h:
5086 case Intrinsic::loongarch_lsx_vssrlni_bu_h:
5087 case Intrinsic::loongarch_lsx_vssrani_bu_h:
5088 case Intrinsic::loongarch_lsx_vssrlrni_b_h:
5089 case Intrinsic::loongarch_lsx_vssrarni_b_h:
5090 case Intrinsic::loongarch_lsx_vssrlrni_bu_h:
5091 case Intrinsic::loongarch_lsx_vssrarni_bu_h:
5092 case Intrinsic::loongarch_lasx_xvsrlni_b_h:
5093 case Intrinsic::loongarch_lasx_xvsrani_b_h:
5094 case Intrinsic::loongarch_lasx_xvsrlrni_b_h:
5095 case Intrinsic::loongarch_lasx_xvsrarni_b_h:
5096 case Intrinsic::loongarch_lasx_xvssrlni_b_h:
5097 case Intrinsic::loongarch_lasx_xvssrani_b_h:
5098 case Intrinsic::loongarch_lasx_xvssrlni_bu_h:
5099 case Intrinsic::loongarch_lasx_xvssrani_bu_h:
5100 case Intrinsic::loongarch_lasx_xvssrlrni_b_h:
5101 case Intrinsic::loongarch_lasx_xvssrarni_b_h:
5102 case Intrinsic::loongarch_lasx_xvssrlrni_bu_h:
5103 case Intrinsic::loongarch_lasx_xvssrarni_bu_h:
5105 case Intrinsic::loongarch_lsx_vsat_w:
5106 case Intrinsic::loongarch_lsx_vsat_wu:
5107 case Intrinsic::loongarch_lsx_vrotri_w:
5108 case Intrinsic::loongarch_lsx_vsllwil_d_w:
5109 case Intrinsic::loongarch_lsx_vsllwil_du_wu:
5110 case Intrinsic::loongarch_lsx_vsrlri_w:
5111 case Intrinsic::loongarch_lsx_vsrari_w:
5112 case Intrinsic::loongarch_lsx_vslei_bu:
5113 case Intrinsic::loongarch_lsx_vslei_hu:
5114 case Intrinsic::loongarch_lsx_vslei_wu:
5115 case Intrinsic::loongarch_lsx_vslei_du:
5116 case Intrinsic::loongarch_lsx_vslti_bu:
5117 case Intrinsic::loongarch_lsx_vslti_hu:
5118 case Intrinsic::loongarch_lsx_vslti_wu:
5119 case Intrinsic::loongarch_lsx_vslti_du:
5120 case Intrinsic::loongarch_lsx_vbsll_v:
5121 case Intrinsic::loongarch_lsx_vbsrl_v:
5122 case Intrinsic::loongarch_lasx_xvsat_w:
5123 case Intrinsic::loongarch_lasx_xvsat_wu:
5124 case Intrinsic::loongarch_lasx_xvrotri_w:
5125 case Intrinsic::loongarch_lasx_xvsllwil_d_w:
5126 case Intrinsic::loongarch_lasx_xvsllwil_du_wu:
5127 case Intrinsic::loongarch_lasx_xvsrlri_w:
5128 case Intrinsic::loongarch_lasx_xvsrari_w:
5129 case Intrinsic::loongarch_lasx_xvslei_bu:
5130 case Intrinsic::loongarch_lasx_xvslei_hu:
5131 case Intrinsic::loongarch_lasx_xvslei_wu:
5132 case Intrinsic::loongarch_lasx_xvslei_du:
5133 case Intrinsic::loongarch_lasx_xvslti_bu:
5134 case Intrinsic::loongarch_lasx_xvslti_hu:
5135 case Intrinsic::loongarch_lasx_xvslti_wu:
5136 case Intrinsic::loongarch_lasx_xvslti_du:
5137 case Intrinsic::loongarch_lasx_xvbsll_v:
5138 case Intrinsic::loongarch_lasx_xvbsrl_v:
5140 case Intrinsic::loongarch_lsx_vseqi_b:
5141 case Intrinsic::loongarch_lsx_vseqi_h:
5142 case Intrinsic::loongarch_lsx_vseqi_w:
5143 case Intrinsic::loongarch_lsx_vseqi_d:
5144 case Intrinsic::loongarch_lsx_vslei_b:
5145 case Intrinsic::loongarch_lsx_vslei_h:
5146 case Intrinsic::loongarch_lsx_vslei_w:
5147 case Intrinsic::loongarch_lsx_vslei_d:
5148 case Intrinsic::loongarch_lsx_vslti_b:
5149 case Intrinsic::loongarch_lsx_vslti_h:
5150 case Intrinsic::loongarch_lsx_vslti_w:
5151 case Intrinsic::loongarch_lsx_vslti_d:
5152 case Intrinsic::loongarch_lasx_xvseqi_b:
5153 case Intrinsic::loongarch_lasx_xvseqi_h:
5154 case Intrinsic::loongarch_lasx_xvseqi_w:
5155 case Intrinsic::loongarch_lasx_xvseqi_d:
5156 case Intrinsic::loongarch_lasx_xvslei_b:
5157 case Intrinsic::loongarch_lasx_xvslei_h:
5158 case Intrinsic::loongarch_lasx_xvslei_w:
5159 case Intrinsic::loongarch_lasx_xvslei_d:
5160 case Intrinsic::loongarch_lasx_xvslti_b:
5161 case Intrinsic::loongarch_lasx_xvslti_h:
5162 case Intrinsic::loongarch_lasx_xvslti_w:
5163 case Intrinsic::loongarch_lasx_xvslti_d:
5165 case Intrinsic::loongarch_lsx_vsrlni_h_w:
5166 case Intrinsic::loongarch_lsx_vsrani_h_w:
5167 case Intrinsic::loongarch_lsx_vsrlrni_h_w:
5168 case Intrinsic::loongarch_lsx_vsrarni_h_w:
5169 case Intrinsic::loongarch_lsx_vssrlni_h_w:
5170 case Intrinsic::loongarch_lsx_vssrani_h_w:
5171 case Intrinsic::loongarch_lsx_vssrlni_hu_w:
5172 case Intrinsic::loongarch_lsx_vssrani_hu_w:
5173 case Intrinsic::loongarch_lsx_vssrlrni_h_w:
5174 case Intrinsic::loongarch_lsx_vssrarni_h_w:
5175 case Intrinsic::loongarch_lsx_vssrlrni_hu_w:
5176 case Intrinsic::loongarch_lsx_vssrarni_hu_w:
5177 case Intrinsic::loongarch_lsx_vfrstpi_b:
5178 case Intrinsic::loongarch_lsx_vfrstpi_h:
5179 case Intrinsic::loongarch_lasx_xvsrlni_h_w:
5180 case Intrinsic::loongarch_lasx_xvsrani_h_w:
5181 case Intrinsic::loongarch_lasx_xvsrlrni_h_w:
5182 case Intrinsic::loongarch_lasx_xvsrarni_h_w:
5183 case Intrinsic::loongarch_lasx_xvssrlni_h_w:
5184 case Intrinsic::loongarch_lasx_xvssrani_h_w:
5185 case Intrinsic::loongarch_lasx_xvssrlni_hu_w:
5186 case Intrinsic::loongarch_lasx_xvssrani_hu_w:
5187 case Intrinsic::loongarch_lasx_xvssrlrni_h_w:
5188 case Intrinsic::loongarch_lasx_xvssrarni_h_w:
5189 case Intrinsic::loongarch_lasx_xvssrlrni_hu_w:
5190 case Intrinsic::loongarch_lasx_xvssrarni_hu_w:
5191 case Intrinsic::loongarch_lasx_xvfrstpi_b:
5192 case Intrinsic::loongarch_lasx_xvfrstpi_h:
5194 case Intrinsic::loongarch_lsx_vsat_d:
5195 case Intrinsic::loongarch_lsx_vsat_du:
5196 case Intrinsic::loongarch_lsx_vrotri_d:
5197 case Intrinsic::loongarch_lsx_vsrlri_d:
5198 case Intrinsic::loongarch_lsx_vsrari_d:
5199 case Intrinsic::loongarch_lasx_xvsat_d:
5200 case Intrinsic::loongarch_lasx_xvsat_du:
5201 case Intrinsic::loongarch_lasx_xvrotri_d:
5202 case Intrinsic::loongarch_lasx_xvsrlri_d:
5203 case Intrinsic::loongarch_lasx_xvsrari_d:
5205 case Intrinsic::loongarch_lsx_vsrlni_w_d:
5206 case Intrinsic::loongarch_lsx_vsrani_w_d:
5207 case Intrinsic::loongarch_lsx_vsrlrni_w_d:
5208 case Intrinsic::loongarch_lsx_vsrarni_w_d:
5209 case Intrinsic::loongarch_lsx_vssrlni_w_d:
5210 case Intrinsic::loongarch_lsx_vssrani_w_d:
5211 case Intrinsic::loongarch_lsx_vssrlni_wu_d:
5212 case Intrinsic::loongarch_lsx_vssrani_wu_d:
5213 case Intrinsic::loongarch_lsx_vssrlrni_w_d:
5214 case Intrinsic::loongarch_lsx_vssrarni_w_d:
5215 case Intrinsic::loongarch_lsx_vssrlrni_wu_d:
5216 case Intrinsic::loongarch_lsx_vssrarni_wu_d:
5217 case Intrinsic::loongarch_lasx_xvsrlni_w_d:
5218 case Intrinsic::loongarch_lasx_xvsrani_w_d:
5219 case Intrinsic::loongarch_lasx_xvsrlrni_w_d:
5220 case Intrinsic::loongarch_lasx_xvsrarni_w_d:
5221 case Intrinsic::loongarch_lasx_xvssrlni_w_d:
5222 case Intrinsic::loongarch_lasx_xvssrani_w_d:
5223 case Intrinsic::loongarch_lasx_xvssrlni_wu_d:
5224 case Intrinsic::loongarch_lasx_xvssrani_wu_d:
5225 case Intrinsic::loongarch_lasx_xvssrlrni_w_d:
5226 case Intrinsic::loongarch_lasx_xvssrarni_w_d:
5227 case Intrinsic::loongarch_lasx_xvssrlrni_wu_d:
5228 case Intrinsic::loongarch_lasx_xvssrarni_wu_d:
5230 case Intrinsic::loongarch_lsx_vsrlni_d_q:
5231 case Intrinsic::loongarch_lsx_vsrani_d_q:
5232 case Intrinsic::loongarch_lsx_vsrlrni_d_q:
5233 case Intrinsic::loongarch_lsx_vsrarni_d_q:
5234 case Intrinsic::loongarch_lsx_vssrlni_d_q:
5235 case Intrinsic::loongarch_lsx_vssrani_d_q:
5236 case Intrinsic::loongarch_lsx_vssrlni_du_q:
5237 case Intrinsic::loongarch_lsx_vssrani_du_q:
5238 case Intrinsic::loongarch_lsx_vssrlrni_d_q:
5239 case Intrinsic::loongarch_lsx_vssrarni_d_q:
5240 case Intrinsic::loongarch_lsx_vssrlrni_du_q:
5241 case Intrinsic::loongarch_lsx_vssrarni_du_q:
5242 case Intrinsic::loongarch_lasx_xvsrlni_d_q:
5243 case Intrinsic::loongarch_lasx_xvsrani_d_q:
5244 case Intrinsic::loongarch_lasx_xvsrlrni_d_q:
5245 case Intrinsic::loongarch_lasx_xvsrarni_d_q:
5246 case Intrinsic::loongarch_lasx_xvssrlni_d_q:
5247 case Intrinsic::loongarch_lasx_xvssrani_d_q:
5248 case Intrinsic::loongarch_lasx_xvssrlni_du_q:
5249 case Intrinsic::loongarch_lasx_xvssrani_du_q:
5250 case Intrinsic::loongarch_lasx_xvssrlrni_d_q:
5251 case Intrinsic::loongarch_lasx_xvssrarni_d_q:
5252 case Intrinsic::loongarch_lasx_xvssrlrni_du_q:
5253 case Intrinsic::loongarch_lasx_xvssrarni_du_q:
5255 case Intrinsic::loongarch_lsx_vnori_b:
5256 case Intrinsic::loongarch_lsx_vshuf4i_b:
5257 case Intrinsic::loongarch_lsx_vshuf4i_h:
5258 case Intrinsic::loongarch_lsx_vshuf4i_w:
5259 case Intrinsic::loongarch_lasx_xvnori_b:
5260 case Intrinsic::loongarch_lasx_xvshuf4i_b:
5261 case Intrinsic::loongarch_lasx_xvshuf4i_h:
5262 case Intrinsic::loongarch_lasx_xvshuf4i_w:
5263 case Intrinsic::loongarch_lasx_xvpermi_d:
5265 case Intrinsic::loongarch_lsx_vshuf4i_d:
5266 case Intrinsic::loongarch_lsx_vpermi_w:
5267 case Intrinsic::loongarch_lsx_vbitseli_b:
5268 case Intrinsic::loongarch_lsx_vextrins_b:
5269 case Intrinsic::loongarch_lsx_vextrins_h:
5270 case Intrinsic::loongarch_lsx_vextrins_w:
5271 case Intrinsic::loongarch_lsx_vextrins_d:
5272 case Intrinsic::loongarch_lasx_xvshuf4i_d:
5273 case Intrinsic::loongarch_lasx_xvpermi_w:
5274 case Intrinsic::loongarch_lasx_xvpermi_q:
5275 case Intrinsic::loongarch_lasx_xvbitseli_b:
5276 case Intrinsic::loongarch_lasx_xvextrins_b:
5277 case Intrinsic::loongarch_lasx_xvextrins_h:
5278 case Intrinsic::loongarch_lasx_xvextrins_w:
5279 case Intrinsic::loongarch_lasx_xvextrins_d:
5281 case Intrinsic::loongarch_lsx_vrepli_b:
5282 case Intrinsic::loongarch_lsx_vrepli_h:
5283 case Intrinsic::loongarch_lsx_vrepli_w:
5284 case Intrinsic::loongarch_lsx_vrepli_d:
5285 case Intrinsic::loongarch_lasx_xvrepli_b:
5286 case Intrinsic::loongarch_lasx_xvrepli_h:
5287 case Intrinsic::loongarch_lasx_xvrepli_w:
5288 case Intrinsic::loongarch_lasx_xvrepli_d:
5290 case Intrinsic::loongarch_lsx_vldi:
5291 case Intrinsic::loongarch_lasx_xvldi:
5307LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(
SDValue Op,
5310 MVT GRLenVT = Subtarget.getGRLenVT();
5311 EVT VT =
Op.getValueType();
5313 const StringRef ErrorMsgOOR =
"argument out of range";
5314 const StringRef ErrorMsgReqLA64 =
"requires loongarch64";
5315 const StringRef ErrorMsgReqF =
"requires basic 'f' target feature";
5317 switch (
Op.getConstantOperandVal(1)) {
5320 case Intrinsic::loongarch_crc_w_b_w:
5321 case Intrinsic::loongarch_crc_w_h_w:
5322 case Intrinsic::loongarch_crc_w_w_w:
5323 case Intrinsic::loongarch_crc_w_d_w:
5324 case Intrinsic::loongarch_crcc_w_b_w:
5325 case Intrinsic::loongarch_crcc_w_h_w:
5326 case Intrinsic::loongarch_crcc_w_w_w:
5327 case Intrinsic::loongarch_crcc_w_d_w:
5329 case Intrinsic::loongarch_csrrd_w:
5330 case Intrinsic::loongarch_csrrd_d: {
5331 unsigned Imm =
Op.getConstantOperandVal(2);
5334 : DAG.
getNode(LoongArchISD::CSRRD,
DL, {GRLenVT, MVT::Other},
5337 case Intrinsic::loongarch_csrwr_w:
5338 case Intrinsic::loongarch_csrwr_d: {
5339 unsigned Imm =
Op.getConstantOperandVal(3);
5342 : DAG.
getNode(LoongArchISD::CSRWR,
DL, {GRLenVT, MVT::Other},
5343 {Chain,
Op.getOperand(2),
5346 case Intrinsic::loongarch_csrxchg_w:
5347 case Intrinsic::loongarch_csrxchg_d: {
5348 unsigned Imm =
Op.getConstantOperandVal(4);
5351 : DAG.
getNode(LoongArchISD::CSRXCHG,
DL, {GRLenVT, MVT::Other},
5352 {Chain,
Op.getOperand(2),
Op.getOperand(3),
5355 case Intrinsic::loongarch_iocsrrd_d: {
5357 LoongArchISD::IOCSRRD_D,
DL, {GRLenVT, MVT::Other},
5360#define IOCSRRD_CASE(NAME, NODE) \
5361 case Intrinsic::loongarch_##NAME: { \
5362 return DAG.getNode(LoongArchISD::NODE, DL, {GRLenVT, MVT::Other}, \
5363 {Chain, Op.getOperand(2)}); \
5369 case Intrinsic::loongarch_cpucfg: {
5370 return DAG.
getNode(LoongArchISD::CPUCFG,
DL, {GRLenVT, MVT::Other},
5371 {Chain,
Op.getOperand(2)});
5373 case Intrinsic::loongarch_lddir_d: {
5374 unsigned Imm =
Op.getConstantOperandVal(3);
5379 case Intrinsic::loongarch_movfcsr2gr: {
5380 if (!Subtarget.hasBasicF())
5382 unsigned Imm =
Op.getConstantOperandVal(2);
5385 : DAG.
getNode(LoongArchISD::MOVFCSR2GR,
DL, {VT, MVT::Other},
5388 case Intrinsic::loongarch_lsx_vld:
5389 case Intrinsic::loongarch_lsx_vldrepl_b:
5390 case Intrinsic::loongarch_lasx_xvld:
5391 case Intrinsic::loongarch_lasx_xvldrepl_b:
5395 case Intrinsic::loongarch_lsx_vldrepl_h:
5396 case Intrinsic::loongarch_lasx_xvldrepl_h:
5400 Op,
"argument out of range or not a multiple of 2", DAG)
5402 case Intrinsic::loongarch_lsx_vldrepl_w:
5403 case Intrinsic::loongarch_lasx_xvldrepl_w:
5407 Op,
"argument out of range or not a multiple of 4", DAG)
5409 case Intrinsic::loongarch_lsx_vldrepl_d:
5410 case Intrinsic::loongarch_lasx_xvldrepl_d:
5414 Op,
"argument out of range or not a multiple of 8", DAG)
5425 return Op.getOperand(0);
5431 MVT GRLenVT = Subtarget.getGRLenVT();
5433 uint64_t IntrinsicEnum =
Op.getConstantOperandVal(1);
5435 const StringRef ErrorMsgOOR =
"argument out of range";
5436 const StringRef ErrorMsgReqLA64 =
"requires loongarch64";
5437 const StringRef ErrorMsgReqLA32 =
"requires loongarch32";
5438 const StringRef ErrorMsgReqF =
"requires basic 'f' target feature";
5440 switch (IntrinsicEnum) {
5444 case Intrinsic::loongarch_cacop_d:
5445 case Intrinsic::loongarch_cacop_w: {
5446 if (IntrinsicEnum == Intrinsic::loongarch_cacop_d && !Subtarget.is64Bit())
5448 if (IntrinsicEnum == Intrinsic::loongarch_cacop_w && Subtarget.is64Bit())
5457 case Intrinsic::loongarch_dbar: {
5464 case Intrinsic::loongarch_ibar: {
5471 case Intrinsic::loongarch_break: {
5478 case Intrinsic::loongarch_movgr2fcsr: {
5479 if (!Subtarget.hasBasicF())
5489 case Intrinsic::loongarch_syscall: {
5496#define IOCSRWR_CASE(NAME, NODE) \
5497 case Intrinsic::loongarch_##NAME: { \
5498 SDValue Op3 = Op.getOperand(3); \
5499 return Subtarget.is64Bit() \
5500 ? DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, \
5501 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5502 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op3)) \
5503 : DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, Op2, \
5510 case Intrinsic::loongarch_iocsrwr_d: {
5511 return !Subtarget.is64Bit()
5518#define ASRT_LE_GT_CASE(NAME) \
5519 case Intrinsic::loongarch_##NAME: { \
5520 return !Subtarget.is64Bit() \
5521 ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG) \
5526#undef ASRT_LE_GT_CASE
5527 case Intrinsic::loongarch_ldpte_d: {
5528 unsigned Imm =
Op.getConstantOperandVal(3);
5529 return !Subtarget.is64Bit()
5534 case Intrinsic::loongarch_lsx_vst:
5535 case Intrinsic::loongarch_lasx_xvst:
5539 case Intrinsic::loongarch_lasx_xvstelm_b:
5544 case Intrinsic::loongarch_lsx_vstelm_b:
5549 case Intrinsic::loongarch_lasx_xvstelm_h:
5554 Op,
"argument out of range or not a multiple of 2", DAG)
5556 case Intrinsic::loongarch_lsx_vstelm_h:
5561 Op,
"argument out of range or not a multiple of 2", DAG)
5563 case Intrinsic::loongarch_lasx_xvstelm_w:
5568 Op,
"argument out of range or not a multiple of 4", DAG)
5570 case Intrinsic::loongarch_lsx_vstelm_w:
5575 Op,
"argument out of range or not a multiple of 4", DAG)
5577 case Intrinsic::loongarch_lasx_xvstelm_d:
5582 Op,
"argument out of range or not a multiple of 8", DAG)
5584 case Intrinsic::loongarch_lsx_vstelm_d:
5589 Op,
"argument out of range or not a multiple of 8", DAG)
5600 EVT VT =
Lo.getValueType();
5641 EVT VT =
Lo.getValueType();
5695 return LoongArchISD::DIV_W;
5697 return LoongArchISD::DIV_WU;
5699 return LoongArchISD::MOD_W;
5701 return LoongArchISD::MOD_WU;
5703 return LoongArchISD::SLL_W;
5705 return LoongArchISD::SRA_W;
5707 return LoongArchISD::SRL_W;
5710 return LoongArchISD::ROTR_W;
5712 return LoongArchISD::CTZ_W;
5714 return LoongArchISD::CLZ_W;
5733 NewOp0 = DAG.
getNode(ExtOpc,
DL, MVT::i64,
N->getOperand(0));
5734 NewRes = DAG.
getNode(WOpcode,
DL, MVT::i64, NewOp0);
5738 NewOp0 = DAG.
getNode(ExtOpc,
DL, MVT::i64,
N->getOperand(0));
5744 NewRes = DAG.
getNode(WOpcode,
DL, MVT::i64, NewOp0, NewOp1);
5771 StringRef ErrorMsg,
bool WithChain =
true) {
5776 Results.push_back(
N->getOperand(0));
5779template <
unsigned N>
5784 const StringRef ErrorMsgOOR =
"argument out of range";
5785 unsigned Imm =
Node->getConstantOperandVal(2);
5819 switch (
N->getConstantOperandVal(0)) {
5822 case Intrinsic::loongarch_lsx_vpickve2gr_b:
5824 LoongArchISD::VPICK_SEXT_ELT);
5826 case Intrinsic::loongarch_lsx_vpickve2gr_h:
5827 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
5829 LoongArchISD::VPICK_SEXT_ELT);
5831 case Intrinsic::loongarch_lsx_vpickve2gr_w:
5833 LoongArchISD::VPICK_SEXT_ELT);
5835 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
5837 LoongArchISD::VPICK_ZEXT_ELT);
5839 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
5840 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
5842 LoongArchISD::VPICK_ZEXT_ELT);
5844 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
5846 LoongArchISD::VPICK_ZEXT_ELT);
5848 case Intrinsic::loongarch_lsx_bz_b:
5849 case Intrinsic::loongarch_lsx_bz_h:
5850 case Intrinsic::loongarch_lsx_bz_w:
5851 case Intrinsic::loongarch_lsx_bz_d:
5852 case Intrinsic::loongarch_lasx_xbz_b:
5853 case Intrinsic::loongarch_lasx_xbz_h:
5854 case Intrinsic::loongarch_lasx_xbz_w:
5855 case Intrinsic::loongarch_lasx_xbz_d:
5857 LoongArchISD::VALL_ZERO);
5859 case Intrinsic::loongarch_lsx_bz_v:
5860 case Intrinsic::loongarch_lasx_xbz_v:
5862 LoongArchISD::VANY_ZERO);
5864 case Intrinsic::loongarch_lsx_bnz_b:
5865 case Intrinsic::loongarch_lsx_bnz_h:
5866 case Intrinsic::loongarch_lsx_bnz_w:
5867 case Intrinsic::loongarch_lsx_bnz_d:
5868 case Intrinsic::loongarch_lasx_xbnz_b:
5869 case Intrinsic::loongarch_lasx_xbnz_h:
5870 case Intrinsic::loongarch_lasx_xbnz_w:
5871 case Intrinsic::loongarch_lasx_xbnz_d:
5873 LoongArchISD::VALL_NONZERO);
5875 case Intrinsic::loongarch_lsx_bnz_v:
5876 case Intrinsic::loongarch_lasx_xbnz_v:
5878 LoongArchISD::VANY_NONZERO);
5886 assert(
N->getValueType(0) == MVT::i128 &&
5887 "AtomicCmpSwap on types less than 128 should be legal");
5891 switch (
MemOp->getMergedOrdering()) {
5895 Opcode = LoongArch::PseudoCmpXchg128Acquire;
5899 Opcode = LoongArch::PseudoCmpXchg128;
5906 auto CmpVal = DAG.
SplitScalar(
N->getOperand(2),
DL, MVT::i64, MVT::i64);
5907 auto NewVal = DAG.
SplitScalar(
N->getOperand(3),
DL, MVT::i64, MVT::i64);
5908 SDValue Ops[] = {
N->getOperand(1), CmpVal.first, CmpVal.second,
5909 NewVal.first, NewVal.second,
N->getOperand(0)};
5912 Opcode,
SDLoc(
N), DAG.
getVTList(MVT::i64, MVT::i64, MVT::i64, MVT::Other),
5923 EVT VT =
N->getValueType(0);
5924 switch (
N->getOpcode()) {
5929 assert(
N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5930 "Unexpected custom legalisation");
5937 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5938 "Unexpected custom legalisation");
5940 Subtarget.hasDiv32() && VT == MVT::i32
5947 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5948 "Unexpected custom legalisation");
5956 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5957 "Unexpected custom legalisation");
5964 MVT VT =
N->getSimpleValueType(0);
5965 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5966 "Unexpected custom legalisation");
5968 "Unexpected type action!");
5973 Ld->getPointerInfo(), Ld->getBaseAlign(),
5974 Ld->getMemOperand()->getFlags());
5985 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5986 "Unexpected custom legalisation");
5993 if (Src.getValueType() == MVT::f16)
6004 EVT OpVT = Src.getValueType();
6008 std::tie(Result, Chain) =
6015 EVT SrcVT = Src.getValueType();
6016 if (VT == MVT::i32 && SrcVT == MVT::f32 && Subtarget.is64Bit() &&
6017 Subtarget.hasBasicF()) {
6019 DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Src);
6021 }
else if (VT == MVT::i64 && SrcVT == MVT::f64 && !Subtarget.is64Bit()) {
6023 DAG.
getVTList(MVT::i32, MVT::i32), Src);
6031 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
6032 "Unexpected custom legalisation");
6035 TLI.expandFP_TO_UINT(
N, Tmp1, Tmp2, DAG);
6040 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
6041 "Unexpected custom legalisation");
6047 if (OpVT == MVT::v2f64) {
6057 assert((VT == MVT::i16 || VT == MVT::i32) &&
6058 "Unexpected custom legalization");
6059 MVT GRLenVT = Subtarget.getGRLenVT();
6066 Tmp = DAG.
getNode(LoongArchISD::REVB_2H,
DL, GRLenVT, NewSrc);
6071 Tmp = DAG.
getNode(LoongArchISD::REVB_2W,
DL, GRLenVT, NewSrc);
6079 assert((VT == MVT::i8 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
6080 "Unexpected custom legalization");
6081 MVT GRLenVT = Subtarget.getGRLenVT();
6088 Tmp = DAG.
getNode(LoongArchISD::BITREV_4B,
DL, GRLenVT, NewSrc);
6091 Tmp = DAG.
getNode(LoongArchISD::BITREV_W,
DL, GRLenVT, NewSrc);
6099 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
6100 "Unexpected custom legalisation");
6107 MVT GRLenVT = Subtarget.getGRLenVT();
6108 const StringRef ErrorMsgOOR =
"argument out of range";
6109 const StringRef ErrorMsgReqLA64 =
"requires loongarch64";
6110 const StringRef ErrorMsgReqF =
"requires basic 'f' target feature";
6112 switch (
N->getConstantOperandVal(1)) {
6115 case Intrinsic::loongarch_movfcsr2gr: {
6116 if (!Subtarget.hasBasicF()) {
6126 LoongArchISD::MOVFCSR2GR,
SDLoc(
N), {MVT::i64, MVT::Other},
6133#define CRC_CASE_EXT_BINARYOP(NAME, NODE) \
6134 case Intrinsic::loongarch_##NAME: { \
6135 SDValue NODE = DAG.getNode( \
6136 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
6137 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
6138 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
6139 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
6140 Results.push_back(NODE.getValue(1)); \
6149#undef CRC_CASE_EXT_BINARYOP
6151#define CRC_CASE_EXT_UNARYOP(NAME, NODE) \
6152 case Intrinsic::loongarch_##NAME: { \
6153 SDValue NODE = DAG.getNode( \
6154 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
6156 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
6157 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
6158 Results.push_back(NODE.getValue(1)); \
6163#undef CRC_CASE_EXT_UNARYOP
6164#define CSR_CASE(ID) \
6165 case Intrinsic::loongarch_##ID: { \
6166 if (!Subtarget.is64Bit()) \
6167 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64); \
6175 case Intrinsic::loongarch_csrrd_w: {
6182 DAG.
getNode(LoongArchISD::CSRRD,
DL, {GRLenVT, MVT::Other},
6189 case Intrinsic::loongarch_csrwr_w: {
6190 unsigned Imm =
N->getConstantOperandVal(3);
6196 DAG.
getNode(LoongArchISD::CSRWR,
DL, {GRLenVT, MVT::Other},
6204 case Intrinsic::loongarch_csrxchg_w: {
6205 unsigned Imm =
N->getConstantOperandVal(4);
6211 LoongArchISD::CSRXCHG,
DL, {GRLenVT, MVT::Other},
6220#define IOCSRRD_CASE(NAME, NODE) \
6221 case Intrinsic::loongarch_##NAME: { \
6222 SDValue IOCSRRDResults = \
6223 DAG.getNode(LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
6224 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)}); \
6225 Results.push_back( \
6226 DAG.getNode(ISD::TRUNCATE, DL, VT, IOCSRRDResults.getValue(0))); \
6227 Results.push_back(IOCSRRDResults.getValue(1)); \
6234 case Intrinsic::loongarch_cpucfg: {
6236 DAG.
getNode(LoongArchISD::CPUCFG,
DL, {GRLenVT, MVT::Other},
6243 case Intrinsic::loongarch_lddir_d: {
6244 if (!Subtarget.is64Bit()) {
6254 if (Subtarget.is64Bit())
6256 "On LA64, only 64-bit registers can be read.");
6259 "On LA32, only 32-bit registers can be read.");
6261 Results.push_back(
N->getOperand(0));
6272 OpVT == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
6285 MVT VT =
N->getSimpleValueType(0);
6291 EVT InVT = In.getValueType();
6301 In = DAG.
getNode(
N->getOpcode(),
DL, InVT, In);
6306 if ((InVT == MVT::v8i32 || InVT == MVT::v4i64) &&
6309 In = DAG.
getNode(
N->getOpcode(),
DL, InVT, In);
6318 for (
unsigned I = 0;
I < MinElts; ++
I)
6319 TruncMask[
I] = Scale *
I;
6321 unsigned WidenNumElts = 128 / In.getScalarValueSizeInBits();
6322 MVT SVT = In.getSimpleValueType().getScalarType();
6328 "Illegal vector type in truncation");
6340 if (!Subtarget.hasExtLSX() || Subtarget.hasExtLASX())
6343 EVT DstVT =
N->getValueType(0);
6345 MVT SrcVT = Src.getSimpleValueType();
6363 unsigned WidenSrcElts = 128 / SrcEltBits;
6370 unsigned FirstStageEltBits = 128 / NumElts;
6374 SrcVT = FirstStageVT;
6375 SrcEltBits = FirstStageEltBits;
6383 while (SrcEltBits < DstEltBits) {
6384 unsigned NextEltBits = SrcEltBits * 2;
6387 unsigned NextEltsPerBlock = CurEltsPerBlock / 2;
6406 Blocks = std::move(NextBlocks);
6407 SrcVT = NextBlockVT;
6408 SrcEltBits = NextEltBits;
6425 assert(
N->getOpcode() ==
ISD::AND &&
"Unexpected opcode combine into ANDN");
6427 MVT VT =
N->getSimpleValueType(0);
6446 return DAG.
getNode(LoongArchISD::VANDN,
DL, VT,
X,
Y);
6450 unsigned MinSizeInBits) {
6458 unsigned SplatBitSize;
6461 return Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
6462 HasAnyUndefs, MinSizeInBits,
6474 for (
unsigned i = 0, NumElts = BV->getNumOperands(); i < NumElts; ++i) {
6485 unsigned EltIdx = IdxC->getZExtValue();
6487 Start = (int)EltIdx - (
int)(i * 2);
6488 if (Start < 0 || Start > 1 || EltIdx != (
unsigned)(Start + (
int)(i * 2)))
6494 else if (Src != CurSrc)
6498 if (!Src || Start < 0)
6512 if (EltBits < 8 || EltBits > 64)
6521 EVT VT = Ld->getValueType(0);
6522 for (
SDUse &U : Ld->uses()) {
6523 if (U.getResNo() != 0)
6526 switch (
User->getOpcode()) {
6529 if (
User->getValueType(0) == VT)
6534 if (U.getOperandNo() == 1)
6551 switch (
V.getOpcode()) {
6558 *Ld->getMemOperand(), &
Fast) &&
6564 switch (Src.getOpcode()) {
6597 EVT VT =
N->getValueType(0);
6603 bool UseLASX =
Subtarget.hasExtLASX() && NumElts >= 2;
6604 MVT PartVT = UseLASX ? MVT::v4i64 : MVT::v2i64;
6619 auto ConvertOperand = [&](
SDValue V) {
6624 DAG.
getLoad(NewVT,
SDLoc(Ld), Ld->getChain(), Ld->getBasePtr(),
6625 Ld->getPointerInfo(), Ld->getBaseAlign(),
6626 Ld->getMemOperand()->getFlags(), Ld->getAAInfo());
6635 N->getOpcode() ==
ISD::ADD ? LoongArchISD::VADD_Q : LoongArchISD::VSUB_Q;
6637 for (
unsigned Idx = 0,
End = NumElts * 2; Idx !=
End;) {
6638 MVT CurVT = UseLASX &&
End - Idx >= 4 ? MVT::v4i64 : MVT::v2i64;
6646 for (
auto [Ld, NewLd] : LoadsToReplace)
6657 unsigned Opc =
N->getOpcode();
6660 EVT VT =
N->getValueType(0);
6667 unsigned ExtOpc =
LHS.getOpcode();
6675 if (ExtOpc !=
RHS.getOpcode())
6678 if (!
LHS.hasOneUse() || !
RHS.hasOneUse())
6681 unsigned OddIdx, EvenIdx;
6685 if (!LHSVec || !RHSVec)
6687 if (OddIdx != 1 || EvenIdx != 0)
6697 if (!
TLI.isTypeLegal(VT) || !
TLI.isTypeLegal(SrcVT))
6710 TargetOpc =
isSigned ? LoongArchISD::VHADDW : LoongArchISD::VHADDW_U;
6712 TargetOpc =
isSigned ? LoongArchISD::VHSUBW : LoongArchISD::VHSUBW_U;
6714 return DAG.
getNode(TargetOpc,
DL, VT, LHSVec, RHSVec);
6729 EVT VT =
N->getValueType(0);
6795 if (
And.getOperand(0) ==
X) {
6824 if (ShiftVal != (SplatVal + 1))
6833 : LoongArchISD::VSRAR,
6843 SDValue FirstOperand =
N->getOperand(0);
6844 SDValue SecondOperand =
N->getOperand(1);
6845 unsigned FirstOperandOpc = FirstOperand.
getOpcode();
6846 EVT ValTy =
N->getValueType(0);
6849 unsigned SMIdx, SMLen;
6914 NewOperand = FirstOperand;
6917 msb = lsb + SMLen - 1;
6921 if (FirstOperandOpc ==
ISD::SRA || FirstOperandOpc ==
ISD::SRL || lsb == 0)
6935 EVT DstVT =
N.getValueType();
6944 EVT SrcVT = Src.getValueType();
6952 if (
N.getConstantOperandVal(1) != (isLow ? 0 : NumElts))
6963 if (NumElts % 2 != 0)
6969 for (
unsigned I = 0;
I != NumElts; ++
I) {
6984 SrcVT = Src.getValueType();
6994 }
else if (ThisSrc != Src) {
6998 unsigned Half = NumElts / 2;
6999 unsigned ExpectedIdx = (
I < Half) ?
I : (
I + Half);
7000 ExpectedIdx += isLow ? 0 : Half;
7002 if (CI->getZExtValue() != ExpectedIdx)
7014 EVT VT =
N->getValueType(0);
7021 unsigned ExtOpc =
LHS.getOpcode();
7029 if (!
LHS.hasOneUse())
7033 N->getValueSizeInBits(0) !=
LHS->getOperand(0).getValueSizeInBits() * 2)
7049 unsigned Opc =
isSigned ? LoongArchISD::VSLLWIL : LoongArchISD::VSLLWIL_U;
7070 SDValue FirstOperand =
N->getOperand(0);
7072 EVT ValTy =
N->getValueType(0);
7075 unsigned MaskIdx, MaskLen;
7090 if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1)
7091 return DAG.
getNode(LoongArchISD::BSTRPICK,
DL, ValTy,
7106 EVT VT =
N->getValueType(0);
7117 if (N0C->isZero() && !
N->hasOneUse())
7123 if (
N->hasOneUse()) {
7124 unsigned UserOpc =
N->use_begin()->getUser()->getOpcode();
7132 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
7168 if (
N->getValueType(0) !=
Subtarget.getGRLenVT())
7173 EVT VT =
N->getValueType(0);
7201 switch (Src.getOpcode()) {
7204 return Src.getOperand(0).getValueSizeInBits() ==
Size;
7214 return Src.getOperand(0).getScalarValueSizeInBits() == 1 &&
7226 switch (Src.getOpcode()) {
7236 Src.getOpcode(),
DL, SExtVT,
7242 DL, SExtVT, Src.getOperand(0),
7254 EVT VT =
N->getValueType(0);
7256 EVT SrcVT = Src.getValueType();
7258 if (Src.getOpcode() !=
ISD::SETCC || !Src.hasOneUse())
7263 EVT CmpVT = Src.getOperand(0).getValueType();
7275 switch (Src.getOperandCondCode(2)) {
7281 Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ;
7286 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
7291 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
7296 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
7298 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
7303 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
7305 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
7310 Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ;
7327 EVT VT =
N->getValueType(0);
7329 EVT SrcVT = Src.getValueType();
7346 bool UseLASX =
false;
7347 bool PropagateSExt =
false;
7349 if (Src.getOpcode() ==
ISD::SETCC && Src.hasOneUse()) {
7350 EVT CmpVT = Src.getOperand(0).getValueType();
7359 SExtVT = MVT::v2i64;
7362 SExtVT = MVT::v4i32;
7364 SExtVT = MVT::v4i64;
7366 PropagateSExt =
true;
7370 SExtVT = MVT::v8i16;
7372 SExtVT = MVT::v8i32;
7374 PropagateSExt =
true;
7378 SExtVT = MVT::v16i8;
7380 SExtVT = MVT::v16i16;
7382 PropagateSExt =
true;
7386 SExtVT = MVT::v32i8;
7395 if (Src.getSimpleValueType() == MVT::v32i8) {
7403 }
else if (UseLASX) {
7409 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
7422 EVT ValTy =
N->getValueType(0);
7423 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
7427 unsigned MaskIdx0, MaskLen0, MaskIdx1, MaskLen1;
7429 bool SwapAndRetried =
false;
7438 if (ValBits != 32 && ValBits != 64)
7453 MaskIdx0 == MaskIdx1 && MaskLen0 == MaskLen1 &&
7456 (MaskIdx0 + MaskLen0 <= ValBits)) {
7477 MaskLen0 == MaskLen1 && MaskIdx1 == 0 &&
7478 (MaskIdx0 + MaskLen0 <= ValBits)) {
7495 (MaskIdx0 + MaskLen0 <= 64) &&
7503 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
7504 : (MaskIdx0 + MaskLen0 - 1),
7520 (MaskIdx0 + MaskLen0 <= ValBits)) {
7543 DAG.
getConstant(ValBits == 32 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
7544 : (MaskIdx0 + MaskLen0 - 1),
7559 unsigned MaskIdx, MaskLen;
7560 if (N1.getOpcode() ==
ISD::SHL && N1.getOperand(0).getOpcode() ==
ISD::AND &&
7569 return DAG.
getNode(LoongArchISD::BSTRINS,
DL, ValTy, N0,
7587 N1.getOperand(0).getOpcode() ==
ISD::SHL &&
7593 return DAG.
getNode(LoongArchISD::BSTRINS,
DL, ValTy, N0,
7601 if (!SwapAndRetried) {
7603 SwapAndRetried =
true;
7607 SwapAndRetried =
false;
7624 return DAG.
getNode(LoongArchISD::BSTRINS,
DL, ValTy, N0,
7633 if (!SwapAndRetried) {
7635 SwapAndRetried =
true;
7644 EVT VT =
N->getValueType(0);
7651 SDValue XorOps[] = {
N->getOperand(0),
N->getOperand(1)};
7652 for (
unsigned AndIdx = 0; AndIdx != 2; ++AndIdx) {
7654 SDValue Mask = XorOps[1 - AndIdx];
7658 for (
unsigned MaskIdx = 0; MaskIdx != 2; ++MaskIdx) {
7663 if (
And.getOperand(MaskIdx) != Mask)
7676 switch (
V.getNode()->getOpcode()) {
7688 if ((TypeNode->
getVT() == MVT::i8) || (TypeNode->
getVT() == MVT::i16)) {
7696 if ((TypeNode->
getVT() == MVT::i8) || (TypeNode->
getVT() == MVT::i16)) {
7726 unsigned MskOpc =
LHS.getOpcode();
7727 if (MskOpc != LoongArchISD::VMSKLTZ && MskOpc != LoongArchISD::XVMSKLTZ)
7730 if (!
LHS.hasOneUse())
7734 EVT SrcVT = Src.getValueType();
7741 : LoongArchISD::VALLZERO,
7809 if (
N->getValueType(0) ==
Subtarget.getGRLenVT())
7814 SDNode *AndNode =
N->getOperand(0).getNode();
7822 SDValue CmpInputValue =
N->getOperand(1);
7831 if (!CN || !CN->
isZero())
7833 AndInputValue1 = AndInputValue1.
getOperand(0);
7837 if (AndInputValue2 != CmpInputValue)
7870 TruncInputValue1, TruncInputValue2);
7912static std::tuple<unsigned, SDValue, EVT>
7916 unsigned Opc = Inner.getOpcode();
7921 if (Inner.getOperand(0) ==
X)
7922 return {
Opc, Inner.getOperand(1), ExtVT};
7923 if (Inner.getOperand(1) ==
X)
7924 return {
Opc, Inner.getOperand(0), ExtVT};
7928 if (Inner.getOperand(0) ==
X)
7929 return {
Opc, Inner.getOperand(1), ExtVT};
7951 EVT VT =
N->getValueType(0);
7963 bool BinIsTrueArm) ->
SDValue {
7971 if (Delta.getNode() ==
N)
7993 if (
SDValue R = TryFold(TrueV, FalseV,
true))
7995 if (
SDValue R = TryFold(FalseV, TrueV,
false))
8009 if (Src.getOpcode() != LoongArchISD::REVB_2W)
8012 return DAG.
getNode(LoongArchISD::BITREV_4B,
SDLoc(
N),
N->getValueType(0),
8037 LHS.getOperand(0).getValueType() ==
Subtarget.getGRLenVT()) {
8041 CCVal =
LHS.getOperandCondCode(2);
8065 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
8092 unsigned Sht =
LHS.getConstantOperandVal(1);
8096 if ((EleBits + Sht) ==
Subtarget.getGRLen()) {
8106 EVT VT =
LHS.getValueType();
8131 return DAG.
getNode(LoongArchISD::BR_CC,
DL,
N->getValueType(0),
8132 N->getOperand(0),
V,
8137 return DAG.
getNode(LoongArchISD::BR_CC,
DL,
N->getValueType(0),
8138 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
8154 EVT VT =
N->getValueType(0);
8157 if (TrueV == FalseV)
8188 return DAG.
getNode(LoongArchISD::SELECT_CC,
DL,
N->getValueType(0),
8189 {LHS, RHS, CC, TrueV, FalseV});
8194template <
unsigned N>
8198 bool IsSigned =
false) {
8202 if ((IsSigned && !
isInt<N>(CImm->getSExtValue())) ||
8203 (!IsSigned && !
isUInt<N>(CImm->getZExtValue()))) {
8205 ": argument out of range.");
8211template <
unsigned N>
8215 EVT ResTy =
Node->getValueType(0);
8219 if ((IsSigned && !
isInt<N>(CImm->getSExtValue())) ||
8220 (!IsSigned && !
isUInt<N>(CImm->getZExtValue()))) {
8222 ": argument out of range.");
8227 IsSigned ? CImm->getSExtValue() : CImm->getZExtValue(), IsSigned),
8233 EVT ResTy =
Node->getValueType(0);
8241 EVT ResTy =
Node->getValueType(0);
8250template <
unsigned N>
8253 EVT ResTy =
Node->getValueType(0);
8258 ": argument out of range.");
8268template <
unsigned N>
8271 EVT ResTy =
Node->getValueType(0);
8276 ": argument out of range.");
8285template <
unsigned N>
8288 EVT ResTy =
Node->getValueType(0);
8293 ": argument out of range.");
8302template <
unsigned W>
8305 unsigned Imm =
N->getConstantOperandVal(2);
8307 const StringRef ErrorMsg =
"argument out of range";
8309 return DAG.
getUNDEF(
N->getValueType(0));
8315 return DAG.
getNode(ResOp,
DL,
N->getValueType(0), Vec, Idx, EltVT);
8323 switch (
N->getConstantOperandVal(0)) {
8326 case Intrinsic::loongarch_lsx_vadd_b:
8327 case Intrinsic::loongarch_lsx_vadd_h:
8328 case Intrinsic::loongarch_lsx_vadd_w:
8329 case Intrinsic::loongarch_lsx_vadd_d:
8330 case Intrinsic::loongarch_lasx_xvadd_b:
8331 case Intrinsic::loongarch_lasx_xvadd_h:
8332 case Intrinsic::loongarch_lasx_xvadd_w:
8333 case Intrinsic::loongarch_lasx_xvadd_d:
8336 case Intrinsic::loongarch_lsx_vaddi_bu:
8337 case Intrinsic::loongarch_lsx_vaddi_hu:
8338 case Intrinsic::loongarch_lsx_vaddi_wu:
8339 case Intrinsic::loongarch_lsx_vaddi_du:
8340 case Intrinsic::loongarch_lasx_xvaddi_bu:
8341 case Intrinsic::loongarch_lasx_xvaddi_hu:
8342 case Intrinsic::loongarch_lasx_xvaddi_wu:
8343 case Intrinsic::loongarch_lasx_xvaddi_du:
8346 case Intrinsic::loongarch_lsx_vsub_b:
8347 case Intrinsic::loongarch_lsx_vsub_h:
8348 case Intrinsic::loongarch_lsx_vsub_w:
8349 case Intrinsic::loongarch_lsx_vsub_d:
8350 case Intrinsic::loongarch_lasx_xvsub_b:
8351 case Intrinsic::loongarch_lasx_xvsub_h:
8352 case Intrinsic::loongarch_lasx_xvsub_w:
8353 case Intrinsic::loongarch_lasx_xvsub_d:
8356 case Intrinsic::loongarch_lsx_vsubi_bu:
8357 case Intrinsic::loongarch_lsx_vsubi_hu:
8358 case Intrinsic::loongarch_lsx_vsubi_wu:
8359 case Intrinsic::loongarch_lsx_vsubi_du:
8360 case Intrinsic::loongarch_lasx_xvsubi_bu:
8361 case Intrinsic::loongarch_lasx_xvsubi_hu:
8362 case Intrinsic::loongarch_lasx_xvsubi_wu:
8363 case Intrinsic::loongarch_lasx_xvsubi_du:
8366 case Intrinsic::loongarch_lsx_vneg_b:
8367 case Intrinsic::loongarch_lsx_vneg_h:
8368 case Intrinsic::loongarch_lsx_vneg_w:
8369 case Intrinsic::loongarch_lsx_vneg_d:
8370 case Intrinsic::loongarch_lasx_xvneg_b:
8371 case Intrinsic::loongarch_lasx_xvneg_h:
8372 case Intrinsic::loongarch_lasx_xvneg_w:
8373 case Intrinsic::loongarch_lasx_xvneg_d:
8377 APInt(
N->getValueType(0).getScalarType().getSizeInBits(), 0,
8379 SDLoc(
N),
N->getValueType(0)),
8381 case Intrinsic::loongarch_lsx_vmax_b:
8382 case Intrinsic::loongarch_lsx_vmax_h:
8383 case Intrinsic::loongarch_lsx_vmax_w:
8384 case Intrinsic::loongarch_lsx_vmax_d:
8385 case Intrinsic::loongarch_lasx_xvmax_b:
8386 case Intrinsic::loongarch_lasx_xvmax_h:
8387 case Intrinsic::loongarch_lasx_xvmax_w:
8388 case Intrinsic::loongarch_lasx_xvmax_d:
8391 case Intrinsic::loongarch_lsx_vmax_bu:
8392 case Intrinsic::loongarch_lsx_vmax_hu:
8393 case Intrinsic::loongarch_lsx_vmax_wu:
8394 case Intrinsic::loongarch_lsx_vmax_du:
8395 case Intrinsic::loongarch_lasx_xvmax_bu:
8396 case Intrinsic::loongarch_lasx_xvmax_hu:
8397 case Intrinsic::loongarch_lasx_xvmax_wu:
8398 case Intrinsic::loongarch_lasx_xvmax_du:
8401 case Intrinsic::loongarch_lsx_vmaxi_b:
8402 case Intrinsic::loongarch_lsx_vmaxi_h:
8403 case Intrinsic::loongarch_lsx_vmaxi_w:
8404 case Intrinsic::loongarch_lsx_vmaxi_d:
8405 case Intrinsic::loongarch_lasx_xvmaxi_b:
8406 case Intrinsic::loongarch_lasx_xvmaxi_h:
8407 case Intrinsic::loongarch_lasx_xvmaxi_w:
8408 case Intrinsic::loongarch_lasx_xvmaxi_d:
8411 case Intrinsic::loongarch_lsx_vmaxi_bu:
8412 case Intrinsic::loongarch_lsx_vmaxi_hu:
8413 case Intrinsic::loongarch_lsx_vmaxi_wu:
8414 case Intrinsic::loongarch_lsx_vmaxi_du:
8415 case Intrinsic::loongarch_lasx_xvmaxi_bu:
8416 case Intrinsic::loongarch_lasx_xvmaxi_hu:
8417 case Intrinsic::loongarch_lasx_xvmaxi_wu:
8418 case Intrinsic::loongarch_lasx_xvmaxi_du:
8421 case Intrinsic::loongarch_lsx_vmin_b:
8422 case Intrinsic::loongarch_lsx_vmin_h:
8423 case Intrinsic::loongarch_lsx_vmin_w:
8424 case Intrinsic::loongarch_lsx_vmin_d:
8425 case Intrinsic::loongarch_lasx_xvmin_b:
8426 case Intrinsic::loongarch_lasx_xvmin_h:
8427 case Intrinsic::loongarch_lasx_xvmin_w:
8428 case Intrinsic::loongarch_lasx_xvmin_d:
8431 case Intrinsic::loongarch_lsx_vmin_bu:
8432 case Intrinsic::loongarch_lsx_vmin_hu:
8433 case Intrinsic::loongarch_lsx_vmin_wu:
8434 case Intrinsic::loongarch_lsx_vmin_du:
8435 case Intrinsic::loongarch_lasx_xvmin_bu:
8436 case Intrinsic::loongarch_lasx_xvmin_hu:
8437 case Intrinsic::loongarch_lasx_xvmin_wu:
8438 case Intrinsic::loongarch_lasx_xvmin_du:
8441 case Intrinsic::loongarch_lsx_vmini_b:
8442 case Intrinsic::loongarch_lsx_vmini_h:
8443 case Intrinsic::loongarch_lsx_vmini_w:
8444 case Intrinsic::loongarch_lsx_vmini_d:
8445 case Intrinsic::loongarch_lasx_xvmini_b:
8446 case Intrinsic::loongarch_lasx_xvmini_h:
8447 case Intrinsic::loongarch_lasx_xvmini_w:
8448 case Intrinsic::loongarch_lasx_xvmini_d:
8451 case Intrinsic::loongarch_lsx_vmini_bu:
8452 case Intrinsic::loongarch_lsx_vmini_hu:
8453 case Intrinsic::loongarch_lsx_vmini_wu:
8454 case Intrinsic::loongarch_lsx_vmini_du:
8455 case Intrinsic::loongarch_lasx_xvmini_bu:
8456 case Intrinsic::loongarch_lasx_xvmini_hu:
8457 case Intrinsic::loongarch_lasx_xvmini_wu:
8458 case Intrinsic::loongarch_lasx_xvmini_du:
8461 case Intrinsic::loongarch_lsx_vmul_b:
8462 case Intrinsic::loongarch_lsx_vmul_h:
8463 case Intrinsic::loongarch_lsx_vmul_w:
8464 case Intrinsic::loongarch_lsx_vmul_d:
8465 case Intrinsic::loongarch_lasx_xvmul_b:
8466 case Intrinsic::loongarch_lasx_xvmul_h:
8467 case Intrinsic::loongarch_lasx_xvmul_w:
8468 case Intrinsic::loongarch_lasx_xvmul_d:
8471 case Intrinsic::loongarch_lsx_vmadd_b:
8472 case Intrinsic::loongarch_lsx_vmadd_h:
8473 case Intrinsic::loongarch_lsx_vmadd_w:
8474 case Intrinsic::loongarch_lsx_vmadd_d:
8475 case Intrinsic::loongarch_lasx_xvmadd_b:
8476 case Intrinsic::loongarch_lasx_xvmadd_h:
8477 case Intrinsic::loongarch_lasx_xvmadd_w:
8478 case Intrinsic::loongarch_lasx_xvmadd_d: {
8479 EVT ResTy =
N->getValueType(0);
8484 case Intrinsic::loongarch_lsx_vmsub_b:
8485 case Intrinsic::loongarch_lsx_vmsub_h:
8486 case Intrinsic::loongarch_lsx_vmsub_w:
8487 case Intrinsic::loongarch_lsx_vmsub_d:
8488 case Intrinsic::loongarch_lasx_xvmsub_b:
8489 case Intrinsic::loongarch_lasx_xvmsub_h:
8490 case Intrinsic::loongarch_lasx_xvmsub_w:
8491 case Intrinsic::loongarch_lasx_xvmsub_d: {
8492 EVT ResTy =
N->getValueType(0);
8497 case Intrinsic::loongarch_lsx_vdiv_b:
8498 case Intrinsic::loongarch_lsx_vdiv_h:
8499 case Intrinsic::loongarch_lsx_vdiv_w:
8500 case Intrinsic::loongarch_lsx_vdiv_d:
8501 case Intrinsic::loongarch_lasx_xvdiv_b:
8502 case Intrinsic::loongarch_lasx_xvdiv_h:
8503 case Intrinsic::loongarch_lasx_xvdiv_w:
8504 case Intrinsic::loongarch_lasx_xvdiv_d:
8507 case Intrinsic::loongarch_lsx_vdiv_bu:
8508 case Intrinsic::loongarch_lsx_vdiv_hu:
8509 case Intrinsic::loongarch_lsx_vdiv_wu:
8510 case Intrinsic::loongarch_lsx_vdiv_du:
8511 case Intrinsic::loongarch_lasx_xvdiv_bu:
8512 case Intrinsic::loongarch_lasx_xvdiv_hu:
8513 case Intrinsic::loongarch_lasx_xvdiv_wu:
8514 case Intrinsic::loongarch_lasx_xvdiv_du:
8517 case Intrinsic::loongarch_lsx_vmod_b:
8518 case Intrinsic::loongarch_lsx_vmod_h:
8519 case Intrinsic::loongarch_lsx_vmod_w:
8520 case Intrinsic::loongarch_lsx_vmod_d:
8521 case Intrinsic::loongarch_lasx_xvmod_b:
8522 case Intrinsic::loongarch_lasx_xvmod_h:
8523 case Intrinsic::loongarch_lasx_xvmod_w:
8524 case Intrinsic::loongarch_lasx_xvmod_d:
8527 case Intrinsic::loongarch_lsx_vmod_bu:
8528 case Intrinsic::loongarch_lsx_vmod_hu:
8529 case Intrinsic::loongarch_lsx_vmod_wu:
8530 case Intrinsic::loongarch_lsx_vmod_du:
8531 case Intrinsic::loongarch_lasx_xvmod_bu:
8532 case Intrinsic::loongarch_lasx_xvmod_hu:
8533 case Intrinsic::loongarch_lasx_xvmod_wu:
8534 case Intrinsic::loongarch_lasx_xvmod_du:
8537 case Intrinsic::loongarch_lsx_vand_v:
8538 case Intrinsic::loongarch_lasx_xvand_v:
8541 case Intrinsic::loongarch_lsx_vor_v:
8542 case Intrinsic::loongarch_lasx_xvor_v:
8545 case Intrinsic::loongarch_lsx_vxor_v:
8546 case Intrinsic::loongarch_lasx_xvxor_v:
8549 case Intrinsic::loongarch_lsx_vnor_v:
8550 case Intrinsic::loongarch_lasx_xvnor_v: {
8555 case Intrinsic::loongarch_lsx_vandi_b:
8556 case Intrinsic::loongarch_lasx_xvandi_b:
8559 case Intrinsic::loongarch_lsx_vori_b:
8560 case Intrinsic::loongarch_lasx_xvori_b:
8563 case Intrinsic::loongarch_lsx_vxori_b:
8564 case Intrinsic::loongarch_lasx_xvxori_b:
8567 case Intrinsic::loongarch_lsx_vsll_b:
8568 case Intrinsic::loongarch_lsx_vsll_h:
8569 case Intrinsic::loongarch_lsx_vsll_w:
8570 case Intrinsic::loongarch_lsx_vsll_d:
8571 case Intrinsic::loongarch_lasx_xvsll_b:
8572 case Intrinsic::loongarch_lasx_xvsll_h:
8573 case Intrinsic::loongarch_lasx_xvsll_w:
8574 case Intrinsic::loongarch_lasx_xvsll_d:
8577 case Intrinsic::loongarch_lsx_vslli_b:
8578 case Intrinsic::loongarch_lasx_xvslli_b:
8581 case Intrinsic::loongarch_lsx_vslli_h:
8582 case Intrinsic::loongarch_lasx_xvslli_h:
8585 case Intrinsic::loongarch_lsx_vslli_w:
8586 case Intrinsic::loongarch_lasx_xvslli_w:
8589 case Intrinsic::loongarch_lsx_vslli_d:
8590 case Intrinsic::loongarch_lasx_xvslli_d:
8593 case Intrinsic::loongarch_lsx_vsrl_b:
8594 case Intrinsic::loongarch_lsx_vsrl_h:
8595 case Intrinsic::loongarch_lsx_vsrl_w:
8596 case Intrinsic::loongarch_lsx_vsrl_d:
8597 case Intrinsic::loongarch_lasx_xvsrl_b:
8598 case Intrinsic::loongarch_lasx_xvsrl_h:
8599 case Intrinsic::loongarch_lasx_xvsrl_w:
8600 case Intrinsic::loongarch_lasx_xvsrl_d:
8603 case Intrinsic::loongarch_lsx_vsrli_b:
8604 case Intrinsic::loongarch_lasx_xvsrli_b:
8607 case Intrinsic::loongarch_lsx_vsrli_h:
8608 case Intrinsic::loongarch_lasx_xvsrli_h:
8611 case Intrinsic::loongarch_lsx_vsrli_w:
8612 case Intrinsic::loongarch_lasx_xvsrli_w:
8615 case Intrinsic::loongarch_lsx_vsrli_d:
8616 case Intrinsic::loongarch_lasx_xvsrli_d:
8619 case Intrinsic::loongarch_lsx_vsra_b:
8620 case Intrinsic::loongarch_lsx_vsra_h:
8621 case Intrinsic::loongarch_lsx_vsra_w:
8622 case Intrinsic::loongarch_lsx_vsra_d:
8623 case Intrinsic::loongarch_lasx_xvsra_b:
8624 case Intrinsic::loongarch_lasx_xvsra_h:
8625 case Intrinsic::loongarch_lasx_xvsra_w:
8626 case Intrinsic::loongarch_lasx_xvsra_d:
8629 case Intrinsic::loongarch_lsx_vsrai_b:
8630 case Intrinsic::loongarch_lasx_xvsrai_b:
8633 case Intrinsic::loongarch_lsx_vsrai_h:
8634 case Intrinsic::loongarch_lasx_xvsrai_h:
8637 case Intrinsic::loongarch_lsx_vsrai_w:
8638 case Intrinsic::loongarch_lasx_xvsrai_w:
8641 case Intrinsic::loongarch_lsx_vsrai_d:
8642 case Intrinsic::loongarch_lasx_xvsrai_d:
8645 case Intrinsic::loongarch_lsx_vclz_b:
8646 case Intrinsic::loongarch_lsx_vclz_h:
8647 case Intrinsic::loongarch_lsx_vclz_w:
8648 case Intrinsic::loongarch_lsx_vclz_d:
8649 case Intrinsic::loongarch_lasx_xvclz_b:
8650 case Intrinsic::loongarch_lasx_xvclz_h:
8651 case Intrinsic::loongarch_lasx_xvclz_w:
8652 case Intrinsic::loongarch_lasx_xvclz_d:
8654 case Intrinsic::loongarch_lsx_vpcnt_b:
8655 case Intrinsic::loongarch_lsx_vpcnt_h:
8656 case Intrinsic::loongarch_lsx_vpcnt_w:
8657 case Intrinsic::loongarch_lsx_vpcnt_d:
8658 case Intrinsic::loongarch_lasx_xvpcnt_b:
8659 case Intrinsic::loongarch_lasx_xvpcnt_h:
8660 case Intrinsic::loongarch_lasx_xvpcnt_w:
8661 case Intrinsic::loongarch_lasx_xvpcnt_d:
8663 case Intrinsic::loongarch_lsx_vbitclr_b:
8664 case Intrinsic::loongarch_lsx_vbitclr_h:
8665 case Intrinsic::loongarch_lsx_vbitclr_w:
8666 case Intrinsic::loongarch_lsx_vbitclr_d:
8667 case Intrinsic::loongarch_lasx_xvbitclr_b:
8668 case Intrinsic::loongarch_lasx_xvbitclr_h:
8669 case Intrinsic::loongarch_lasx_xvbitclr_w:
8670 case Intrinsic::loongarch_lasx_xvbitclr_d:
8672 case Intrinsic::loongarch_lsx_vbitclri_b:
8673 case Intrinsic::loongarch_lasx_xvbitclri_b:
8675 case Intrinsic::loongarch_lsx_vbitclri_h:
8676 case Intrinsic::loongarch_lasx_xvbitclri_h:
8678 case Intrinsic::loongarch_lsx_vbitclri_w:
8679 case Intrinsic::loongarch_lasx_xvbitclri_w:
8681 case Intrinsic::loongarch_lsx_vbitclri_d:
8682 case Intrinsic::loongarch_lasx_xvbitclri_d:
8684 case Intrinsic::loongarch_lsx_vbitset_b:
8685 case Intrinsic::loongarch_lsx_vbitset_h:
8686 case Intrinsic::loongarch_lsx_vbitset_w:
8687 case Intrinsic::loongarch_lsx_vbitset_d:
8688 case Intrinsic::loongarch_lasx_xvbitset_b:
8689 case Intrinsic::loongarch_lasx_xvbitset_h:
8690 case Intrinsic::loongarch_lasx_xvbitset_w:
8691 case Intrinsic::loongarch_lasx_xvbitset_d: {
8692 EVT VecTy =
N->getValueType(0);
8698 case Intrinsic::loongarch_lsx_vbitseti_b:
8699 case Intrinsic::loongarch_lasx_xvbitseti_b:
8701 case Intrinsic::loongarch_lsx_vbitseti_h:
8702 case Intrinsic::loongarch_lasx_xvbitseti_h:
8704 case Intrinsic::loongarch_lsx_vbitseti_w:
8705 case Intrinsic::loongarch_lasx_xvbitseti_w:
8707 case Intrinsic::loongarch_lsx_vbitseti_d:
8708 case Intrinsic::loongarch_lasx_xvbitseti_d:
8710 case Intrinsic::loongarch_lsx_vbitrev_b:
8711 case Intrinsic::loongarch_lsx_vbitrev_h:
8712 case Intrinsic::loongarch_lsx_vbitrev_w:
8713 case Intrinsic::loongarch_lsx_vbitrev_d:
8714 case Intrinsic::loongarch_lasx_xvbitrev_b:
8715 case Intrinsic::loongarch_lasx_xvbitrev_h:
8716 case Intrinsic::loongarch_lasx_xvbitrev_w:
8717 case Intrinsic::loongarch_lasx_xvbitrev_d: {
8718 EVT VecTy =
N->getValueType(0);
8724 case Intrinsic::loongarch_lsx_vbitrevi_b:
8725 case Intrinsic::loongarch_lasx_xvbitrevi_b:
8727 case Intrinsic::loongarch_lsx_vbitrevi_h:
8728 case Intrinsic::loongarch_lasx_xvbitrevi_h:
8730 case Intrinsic::loongarch_lsx_vbitrevi_w:
8731 case Intrinsic::loongarch_lasx_xvbitrevi_w:
8733 case Intrinsic::loongarch_lsx_vbitrevi_d:
8734 case Intrinsic::loongarch_lasx_xvbitrevi_d:
8736 case Intrinsic::loongarch_lsx_vfadd_s:
8737 case Intrinsic::loongarch_lsx_vfadd_d:
8738 case Intrinsic::loongarch_lasx_xvfadd_s:
8739 case Intrinsic::loongarch_lasx_xvfadd_d:
8742 case Intrinsic::loongarch_lsx_vfsub_s:
8743 case Intrinsic::loongarch_lsx_vfsub_d:
8744 case Intrinsic::loongarch_lasx_xvfsub_s:
8745 case Intrinsic::loongarch_lasx_xvfsub_d:
8748 case Intrinsic::loongarch_lsx_vfmul_s:
8749 case Intrinsic::loongarch_lsx_vfmul_d:
8750 case Intrinsic::loongarch_lasx_xvfmul_s:
8751 case Intrinsic::loongarch_lasx_xvfmul_d:
8754 case Intrinsic::loongarch_lsx_vfdiv_s:
8755 case Intrinsic::loongarch_lsx_vfdiv_d:
8756 case Intrinsic::loongarch_lasx_xvfdiv_s:
8757 case Intrinsic::loongarch_lasx_xvfdiv_d:
8760 case Intrinsic::loongarch_lsx_vfmadd_s:
8761 case Intrinsic::loongarch_lsx_vfmadd_d:
8762 case Intrinsic::loongarch_lasx_xvfmadd_s:
8763 case Intrinsic::loongarch_lasx_xvfmadd_d:
8765 N->getOperand(2),
N->getOperand(3));
8766 case Intrinsic::loongarch_lsx_vinsgr2vr_b:
8768 N->getOperand(1),
N->getOperand(2),
8770 case Intrinsic::loongarch_lsx_vinsgr2vr_h:
8771 case Intrinsic::loongarch_lasx_xvinsgr2vr_w:
8773 N->getOperand(1),
N->getOperand(2),
8775 case Intrinsic::loongarch_lsx_vinsgr2vr_w:
8776 case Intrinsic::loongarch_lasx_xvinsgr2vr_d:
8778 N->getOperand(1),
N->getOperand(2),
8780 case Intrinsic::loongarch_lsx_vinsgr2vr_d:
8782 N->getOperand(1),
N->getOperand(2),
8784 case Intrinsic::loongarch_lsx_vreplgr2vr_b:
8785 case Intrinsic::loongarch_lsx_vreplgr2vr_h:
8786 case Intrinsic::loongarch_lsx_vreplgr2vr_w:
8787 case Intrinsic::loongarch_lsx_vreplgr2vr_d:
8788 case Intrinsic::loongarch_lasx_xvreplgr2vr_b:
8789 case Intrinsic::loongarch_lasx_xvreplgr2vr_h:
8790 case Intrinsic::loongarch_lasx_xvreplgr2vr_w:
8791 case Intrinsic::loongarch_lasx_xvreplgr2vr_d:
8792 return DAG.
getNode(LoongArchISD::VREPLGR2VR,
DL,
N->getValueType(0),
8795 case Intrinsic::loongarch_lsx_vreplve_b:
8796 case Intrinsic::loongarch_lsx_vreplve_h:
8797 case Intrinsic::loongarch_lsx_vreplve_w:
8798 case Intrinsic::loongarch_lsx_vreplve_d:
8799 case Intrinsic::loongarch_lasx_xvreplve_b:
8800 case Intrinsic::loongarch_lasx_xvreplve_h:
8801 case Intrinsic::loongarch_lasx_xvreplve_w:
8802 case Intrinsic::loongarch_lasx_xvreplve_d:
8803 return DAG.
getNode(LoongArchISD::VREPLVE,
DL,
N->getValueType(0),
8807 case Intrinsic::loongarch_lsx_vpickve2gr_b:
8811 case Intrinsic::loongarch_lsx_vpickve2gr_h:
8812 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
8816 case Intrinsic::loongarch_lsx_vpickve2gr_w:
8820 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
8824 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
8825 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
8829 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
8833 case Intrinsic::loongarch_lsx_bz_b:
8834 case Intrinsic::loongarch_lsx_bz_h:
8835 case Intrinsic::loongarch_lsx_bz_w:
8836 case Intrinsic::loongarch_lsx_bz_d:
8837 case Intrinsic::loongarch_lasx_xbz_b:
8838 case Intrinsic::loongarch_lasx_xbz_h:
8839 case Intrinsic::loongarch_lasx_xbz_w:
8840 case Intrinsic::loongarch_lasx_xbz_d:
8842 return DAG.
getNode(LoongArchISD::VALL_ZERO,
DL,
N->getValueType(0),
8845 case Intrinsic::loongarch_lsx_bz_v:
8846 case Intrinsic::loongarch_lasx_xbz_v:
8848 return DAG.
getNode(LoongArchISD::VANY_ZERO,
DL,
N->getValueType(0),
8851 case Intrinsic::loongarch_lsx_bnz_b:
8852 case Intrinsic::loongarch_lsx_bnz_h:
8853 case Intrinsic::loongarch_lsx_bnz_w:
8854 case Intrinsic::loongarch_lsx_bnz_d:
8855 case Intrinsic::loongarch_lasx_xbnz_b:
8856 case Intrinsic::loongarch_lasx_xbnz_h:
8857 case Intrinsic::loongarch_lasx_xbnz_w:
8858 case Intrinsic::loongarch_lasx_xbnz_d:
8860 return DAG.
getNode(LoongArchISD::VALL_NONZERO,
DL,
N->getValueType(0),
8863 case Intrinsic::loongarch_lsx_bnz_v:
8864 case Intrinsic::loongarch_lasx_xbnz_v:
8866 return DAG.
getNode(LoongArchISD::VANY_NONZERO,
DL,
N->getValueType(0),
8869 case Intrinsic::loongarch_lasx_concat_128_s:
8870 case Intrinsic::loongarch_lasx_concat_128_d:
8871 case Intrinsic::loongarch_lasx_concat_128:
8873 N->getOperand(1),
N->getOperand(2));
8885 if (Op0.
getOpcode() == LoongArchISD::MOVFR2GR_S_LA64)
8897 if (Op0->
getOpcode() == LoongArchISD::MOVGR2FR_W_LA64) {
8899 "Unexpected value type!");
8908 MVT VT =
N->getSimpleValueType(0);
8914 if (
TLI.SimplifyDemandedBits(
SDValue(
N, 0), DemandedMask, DCI))
8929 if (Op0->
getOpcode() == LoongArchISD::BUILD_PAIR_F64)
8942 APInt V =
C->getValueAPF().bitcastToAPInt();
8957 MVT VT =
N->getSimpleValueType(0);
9035 EVT VT =
N->getValueType(0);
9037 EVT SrcVT = Src.getValueType();
9043 if (SrcEltBits >= DstEltBits)
9050 return DAG.
getNode(
N->getOpcode(),
DL, VT, Extend);
9062 unsigned BlockBits) {
9064 MVT DstVT =
N->getSimpleValueType(0);
9066 MVT SrcVT = Src.getSimpleValueType();
9073 Src.getOperand(0).getValueType() == BlockVT) {
9074 for (
unsigned i = 0; i < Src.getNumOperands(); ++i)
9076 }
else if (SrcBits > BlockBits) {
9078 for (
unsigned i = 0; i < SrcBits / BlockBits; ++i)
9083 BlockBits = SrcBits;
9090 for (
unsigned i = 0; i < Blocks.
size(); i += 2) {
9095 if (BlockBits == 256) {
9105 if (Blocks.
size() == 1)
9115 EVT VT =
N->getValueType(0);
9117 EVT SrcVT = Src.getValueType();
9123 unsigned BlockBits =
Subtarget.hasExtLASX() ? 256 : 128;
9126 if (SrcEltBits <= DstEltBits)
9129 if (SrcEltBits != 64 || DstEltBits != 32 || !
isPowerOf2_32(NumElts))
9143 if (VT != MVT::f32 && VT != MVT::f64)
9145 if (VT == MVT::f32 && !
Subtarget.hasBasicF())
9147 if (VT == MVT::f64 && !
Subtarget.hasBasicD())
9179 EVT VT =
N->getValueType(0);
9200 EVT DstVT =
N->getValueType(0);
9202 EVT SrcVT = Src.getValueType();
9212 unsigned BlockBits =
Subtarget.hasExtLASX() ? 256 : 128;
9217 if (SrcBits % BlockBits != 0 && SrcBits != 128)
9220 if (DstEltBits < 32) {
9226 if (SrcEltBits != 64 || DstEltBits != 32)
9265 if (!
TLI.isOperationLegalOrPromote(
N.getOpcode(), VT))
9304 return DAG.
getNode(
N.getOpcode(),
DL, VT, N0, N1);
9314 EVT VT =
N.getValueType();
9331 switch (
N.getOpcode()) {
9347 EVT VT =
N->getValueType(0);
9355 N->getValueSizeInBits(0) !=
N->getOperand(0).getValueSizeInBits() * 2)
9360 return DAG.
getNode(LoongArchISD::VEXTH,
DL, VT, R);
9362 return DAG.
getNode(LoongArchISD::VEXTH_U,
DL, VT, R);
9374 EVT VT =
N->getValueType(0);
9376 if (VT.
isVector() &&
N->getNumOperands() == 2)
9389 EVT VT =
N->getValueType(0);
9401 SDValue TrueVal =
N->getOperand(1);
9402 SDValue FalseVal =
N->getOperand(2);
9435 if (FalseVal.getOpcode() !=
ISD::ADD)
9438 SDValue Add0 = FalseVal.getOperand(0);
9439 SDValue Add1 = FalseVal.getOperand(1);
9509 : LoongArchISD::VSRAR,
9520 EVT ResTy =
N->getValueType(0);
9525 if (ResTy != MVT::v8i16 && ResTy != MVT::v4i32 && ResTy != MVT::v2i64 &&
9526 ResTy != MVT::i128 && ResTy != MVT::v16i16 && ResTy != MVT::v8i32 &&
9527 ResTy != MVT::v4i64 && ResTy != MVT::v2i128)
9529 if (HasLSXOnly && (ResTy.
is256BitVector() || ResTy == MVT::v2i128))
9542 auto getExtType = [](
unsigned Op0,
unsigned Op1) ->
int {
9560 bool IsScalar = (ResTy == MVT::i128);
9561 if (IsScalar && ExtType < 2 &&
Subtarget.is64Bit())
9574 unsigned Src0Bits = Src0.getValueType().getScalarSizeInBits();
9575 unsigned Src1Bits = Src1.getValueType().getScalarSizeInBits();
9576 if (Src0Bits != Src1Bits || ResBits != Src0Bits * 2)
9584 for (
unsigned i = 0; i < Src0.getNumOperands(); ++i)
9585 Elems.
emplace_back(Src0.getOperand(i), Src1.getOperand(i));
9591 for (
auto &[Op0, Op1] : Elems) {
9605 unsigned CurIdx = IdxC->getZExtValue();
9607 if (CurIdx != 0 && CurIdx != 1)
9609 OrigN0 = Op0.getOperand(0);
9613 if (CurIdx != Index + 2 || Op0.getOperand(0) != OrigN0 ||
9628 bool IsEven = (Index % 2 == 0);
9630 static const unsigned OpcTable[3][2] = {
9631 {LoongArchISD::VMULWOD, LoongArchISD::VMULWEV},
9632 {LoongArchISD::VMULWODU, LoongArchISD::VMULWEVU},
9633 {LoongArchISD::VMULWODUS, LoongArchISD::VMULWEVUS}};
9636 Result = DAG.
getNode(OpcTable[2][IsEven],
DL, OrigTy, OrigN1, OrigN0);
9638 Result = DAG.
getNode(OpcTable[ExtType][IsEven],
DL, OrigTy, OrigN0, OrigN1);
9648 EVT VT =
N->getValueType(0);
9650 unsigned Opc = Src.getOpcode();
9656 if (
N->hasOneUse()) {
9683 return DAG.
getNode(LoongArchISD::SLL_W,
DL, VT, Src.getOperand(0),
9690 if (!
N->hasOneUse())
9710 if (Src.hasOneUse() && Src.getOperand(0).getValueType().isInteger()) {
9720 SDLoc(
N), Src.getValueType(), Src.getOperand(0),
9772 if (
N->hasOneUse() &&
N->use_begin()->getUser()->getOpcode() ==
ISD::SUB)
9796 switch (
N->getOpcode()) {
9832 case LoongArchISD::BITREV_W:
9834 case LoongArchISD::BR_CC:
9836 case LoongArchISD::SELECT_CC:
9840 case LoongArchISD::MOVGR2FR_W_LA64:
9842 case LoongArchISD::MOVFR2GR_S_LA64:
9844 case LoongArchISD::CRC_W_B_W:
9845 case LoongArchISD::CRC_W_H_W:
9846 case LoongArchISD::CRCC_W_B_W:
9847 case LoongArchISD::CRCC_W_H_W:
9848 case LoongArchISD::VMSKLTZ:
9849 case LoongArchISD::XVMSKLTZ:
9851 case LoongArchISD::SPLIT_PAIR_F64:
9853 case LoongArchISD::VANDN:
9859 case LoongArchISD::VPACKEV:
9860 case LoongArchISD::VPERMI:
9889 MF->
insert(It, BreakMBB);
9893 SinkMBB->splice(SinkMBB->end(),
MBB, std::next(
MI.getIterator()),
MBB->end());
9894 SinkMBB->transferSuccessorsAndUpdatePHIs(
MBB);
9906 MBB->addSuccessor(BreakMBB);
9907 MBB->addSuccessor(SinkMBB);
9913 BreakMBB->addSuccessor(SinkMBB);
9925 unsigned BroadcastOp;
9927 switch (
MI.getOpcode()) {
9930 case LoongArch::PseudoXVINSGR2VR_B:
9932 BroadcastOp = LoongArch::XVREPLGR2VR_B;
9933 InsOp = LoongArch::XVEXTRINS_B;
9935 case LoongArch::PseudoXVINSGR2VR_H:
9937 BroadcastOp = LoongArch::XVREPLGR2VR_H;
9938 InsOp = LoongArch::XVEXTRINS_H;
9950 unsigned Idx =
MI.getOperand(3).getImm();
9952 if (XSrc.
isVirtual() &&
MRI.getVRegDef(XSrc)->isImplicitDef() &&
9954 Register ScratchSubReg1 =
MRI.createVirtualRegister(SubRC);
9955 Register ScratchSubReg2 =
MRI.createVirtualRegister(SubRC);
9958 .
addReg(XSrc, {}, LoongArch::sub_128);
9960 TII->get(HalfSize == 8 ? LoongArch::VINSGR2VR_H
9961 : LoongArch::VINSGR2VR_B),
9969 .
addImm(LoongArch::sub_128);
9971 Register ScratchReg1 =
MRI.createVirtualRegister(RC);
9972 Register ScratchReg2 =
MRI.createVirtualRegister(RC);
9976 BuildMI(*BB,
MI,
DL,
TII->get(LoongArch::XVPERMI_Q), ScratchReg2)
9979 .
addImm(Idx >= HalfSize ? 48 : 18);
9984 .
addImm((Idx >= HalfSize ? Idx - HalfSize : Idx) * 17);
9987 MI.eraseFromParent();
10002 unsigned BroadcastOp, CTOp, PickOp;
10003 switch (
MI.getOpcode()) {
10006 case LoongArch::PseudoCTPOP_B:
10007 BroadcastOp = LoongArch::VREPLGR2VR_B;
10008 CTOp = LoongArch::VPCNT_B;
10009 PickOp = LoongArch::VPICKVE2GR_B;
10011 case LoongArch::PseudoCTPOP_H:
10012 case LoongArch::PseudoCTPOP_H_LA32:
10013 BroadcastOp = LoongArch::VREPLGR2VR_H;
10014 CTOp = LoongArch::VPCNT_H;
10015 PickOp = LoongArch::VPICKVE2GR_H;
10017 case LoongArch::PseudoCTPOP_W:
10018 case LoongArch::PseudoCTPOP_W_LA32:
10019 BroadcastOp = LoongArch::VREPLGR2VR_W;
10020 CTOp = LoongArch::VPCNT_W;
10021 PickOp = LoongArch::VPICKVE2GR_W;
10023 case LoongArch::PseudoCTPOP_D:
10024 BroadcastOp = LoongArch::VREPLGR2VR_D;
10025 CTOp = LoongArch::VPCNT_D;
10026 PickOp = LoongArch::VPICKVE2GR_D;
10030 Register ScratchReg1 =
MRI.createVirtualRegister(RC);
10031 Register ScratchReg2 =
MRI.createVirtualRegister(RC);
10036 MI.eraseFromParent();
10050 unsigned EleBits = 8;
10051 unsigned NotOpc = 0;
10054 switch (
MI.getOpcode()) {
10057 case LoongArch::PseudoVMSKLTZ_B:
10058 MskOpc = LoongArch::VMSKLTZ_B;
10060 case LoongArch::PseudoVMSKLTZ_H:
10061 MskOpc = LoongArch::VMSKLTZ_H;
10064 case LoongArch::PseudoVMSKLTZ_W:
10065 MskOpc = LoongArch::VMSKLTZ_W;
10068 case LoongArch::PseudoVMSKLTZ_D:
10069 MskOpc = LoongArch::VMSKLTZ_D;
10072 case LoongArch::PseudoVMSKGEZ_B:
10073 MskOpc = LoongArch::VMSKGEZ_B;
10075 case LoongArch::PseudoVMSKEQZ_B:
10076 MskOpc = LoongArch::VMSKNZ_B;
10077 NotOpc = LoongArch::VNOR_V;
10079 case LoongArch::PseudoVMSKNEZ_B:
10080 MskOpc = LoongArch::VMSKNZ_B;
10082 case LoongArch::PseudoXVMSKLTZ_B:
10083 MskOpc = LoongArch::XVMSKLTZ_B;
10084 RC = &LoongArch::LASX256RegClass;
10086 case LoongArch::PseudoXVMSKLTZ_H:
10087 MskOpc = LoongArch::XVMSKLTZ_H;
10088 RC = &LoongArch::LASX256RegClass;
10091 case LoongArch::PseudoXVMSKLTZ_W:
10092 MskOpc = LoongArch::XVMSKLTZ_W;
10093 RC = &LoongArch::LASX256RegClass;
10096 case LoongArch::PseudoXVMSKLTZ_D:
10097 MskOpc = LoongArch::XVMSKLTZ_D;
10098 RC = &LoongArch::LASX256RegClass;
10101 case LoongArch::PseudoXVMSKGEZ_B:
10102 MskOpc = LoongArch::XVMSKGEZ_B;
10103 RC = &LoongArch::LASX256RegClass;
10105 case LoongArch::PseudoXVMSKEQZ_B:
10106 MskOpc = LoongArch::XVMSKNZ_B;
10107 NotOpc = LoongArch::XVNOR_V;
10108 RC = &LoongArch::LASX256RegClass;
10110 case LoongArch::PseudoXVMSKNEZ_B:
10111 MskOpc = LoongArch::XVMSKNZ_B;
10112 RC = &LoongArch::LASX256RegClass;
10125 if (
TRI->getRegSizeInBits(*RC) > 128) {
10126 Register Lo =
MRI.createVirtualRegister(&LoongArch::GPRRegClass);
10127 Register Hi =
MRI.createVirtualRegister(&LoongArch::GPRRegClass);
10136 : LoongArch::BSTRINS_W),
10140 .
addImm(256 / EleBits - 1)
10148 MI.eraseFromParent();
10155 assert(
MI.getOpcode() == LoongArch::SplitPairF64Pseudo &&
10156 "Unexpected instruction");
10163 Register SrcReg =
MI.getOperand(2).getReg();
10168 MI.eraseFromParent();
10175 assert(
MI.getOpcode() == LoongArch::BuildPairF64Pseudo &&
10176 "Unexpected instruction");
10182 Register TmpReg =
MRI.createVirtualRegister(&LoongArch::FPR64RegClass);
10183 Register DstReg =
MI.getOperand(0).getReg();
10192 MI.eraseFromParent();
10197 switch (
MI.getOpcode()) {
10200 case LoongArch::Select_GPR_Using_CC_GPR:
10236 if (
MI.getOperand(2).isReg())
10237 RHS =
MI.getOperand(2).getReg();
10238 auto CC =
static_cast<unsigned>(
MI.getOperand(3).
getImm());
10242 SelectDests.
insert(
MI.getOperand(0).getReg());
10246 SequenceMBBI !=
E; ++SequenceMBBI) {
10247 if (SequenceMBBI->isDebugInstr())
10250 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
10251 !SequenceMBBI->getOperand(2).isReg() ||
10252 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
10253 SequenceMBBI->getOperand(3).getImm() !=
CC ||
10254 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
10255 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
10257 LastSelectPseudo = &*SequenceMBBI;
10259 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
10262 if (SequenceMBBI->hasUnmodeledSideEffects() ||
10263 SequenceMBBI->mayLoadOrStore() ||
10264 SequenceMBBI->usesCustomInsertionHook())
10267 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
10282 F->insert(
I, IfFalseMBB);
10283 F->insert(
I, TailMBB);
10286 unsigned CallFrameSize =
TII.getCallFrameSizeAt(*LastSelectPseudo);
10292 TailMBB->
push_back(DebugInstr->removeFromParent());
10296 TailMBB->
splice(TailMBB->
end(), HeadMBB,
10306 if (
MI.getOperand(2).isImm())
10309 .
addImm(
MI.getOperand(2).getImm())
10318 auto SelectMBBI =
MI.getIterator();
10319 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
10321 while (SelectMBBI != SelectEnd) {
10322 auto Next = std::next(SelectMBBI);
10326 TII.get(LoongArch::PHI), SelectMBBI->getOperand(0).getReg())
10327 .
addReg(SelectMBBI->getOperand(4).getReg())
10329 .
addReg(SelectMBBI->getOperand(5).getReg())
10336 F->getProperties().resetNoPHIs();
10342 const TargetInstrInfo *
TII = Subtarget.getInstrInfo();
10345 switch (
MI.getOpcode()) {
10348 case LoongArch::DIV_W:
10349 case LoongArch::DIV_WU:
10350 case LoongArch::MOD_W:
10351 case LoongArch::MOD_WU:
10352 case LoongArch::DIV_D:
10353 case LoongArch::DIV_DU:
10354 case LoongArch::MOD_D:
10355 case LoongArch::MOD_DU:
10358 case LoongArch::WRFCSR: {
10360 LoongArch::FCSR0 +
MI.getOperand(0).getImm())
10361 .
addReg(
MI.getOperand(1).getReg());
10362 MI.eraseFromParent();
10365 case LoongArch::RDFCSR: {
10366 MachineInstr *ReadFCSR =
10368 MI.getOperand(0).getReg())
10369 .
addReg(LoongArch::FCSR0 +
MI.getOperand(1).getImm());
10371 MI.eraseFromParent();
10374 case LoongArch::Select_GPR_Using_CC_GPR:
10376 case LoongArch::BuildPairF64Pseudo:
10378 case LoongArch::SplitPairF64Pseudo:
10380 case LoongArch::PseudoXVINSGR2VR_B:
10381 case LoongArch::PseudoXVINSGR2VR_H:
10383 case LoongArch::PseudoCTPOP_B:
10384 case LoongArch::PseudoCTPOP_H:
10385 case LoongArch::PseudoCTPOP_W:
10386 case LoongArch::PseudoCTPOP_D:
10387 case LoongArch::PseudoCTPOP_H_LA32:
10388 case LoongArch::PseudoCTPOP_W_LA32:
10390 case LoongArch::PseudoVMSKLTZ_B:
10391 case LoongArch::PseudoVMSKLTZ_H:
10392 case LoongArch::PseudoVMSKLTZ_W:
10393 case LoongArch::PseudoVMSKLTZ_D:
10394 case LoongArch::PseudoVMSKGEZ_B:
10395 case LoongArch::PseudoVMSKEQZ_B:
10396 case LoongArch::PseudoVMSKNEZ_B:
10397 case LoongArch::PseudoXVMSKLTZ_B:
10398 case LoongArch::PseudoXVMSKLTZ_H:
10399 case LoongArch::PseudoXVMSKLTZ_W:
10400 case LoongArch::PseudoXVMSKLTZ_D:
10401 case LoongArch::PseudoXVMSKGEZ_B:
10402 case LoongArch::PseudoXVMSKEQZ_B:
10403 case LoongArch::PseudoXVMSKNEZ_B:
10405 case TargetOpcode::STATEPOINT:
10411 MI.addOperand(*
MI.getMF(),
10413 LoongArch::R1,
true,
10416 if (!Subtarget.is64Bit())
10419 case LoongArch::PROBED_STACKALLOC_DYN:
10426 unsigned *
Fast)
const {
10427 if (!Subtarget.hasUAL())
10445 LoongArch::R7, LoongArch::R8, LoongArch::R9,
10446 LoongArch::R10, LoongArch::R11};
10461 LoongArch::R23, LoongArch::R24, LoongArch::R25, LoongArch::R26,
10462 LoongArch::R27, LoongArch::R28, LoongArch::R29, LoongArch::R30,
10463 LoongArch::R4, LoongArch::R5, LoongArch::R6, LoongArch::R7,
10464 LoongArch::R8, LoongArch::R9, LoongArch::R10, LoongArch::R11,
10465 LoongArch::R12, LoongArch::R13, LoongArch::R14, LoongArch::R15,
10466 LoongArch::R16, LoongArch::R17, LoongArch::R18, LoongArch::R19,
10472 LoongArch::F3, LoongArch::F4, LoongArch::F5,
10473 LoongArch::F6, LoongArch::F7};
10476 LoongArch::F0_64, LoongArch::F1_64, LoongArch::F2_64, LoongArch::F3_64,
10477 LoongArch::F4_64, LoongArch::F5_64, LoongArch::F6_64, LoongArch::F7_64};
10480 LoongArch::VR3, LoongArch::VR4, LoongArch::VR5,
10481 LoongArch::VR6, LoongArch::VR7};
10484 LoongArch::XR3, LoongArch::XR4, LoongArch::XR5,
10485 LoongArch::XR6, LoongArch::XR7};
10488 switch (State.getCallingConv()) {
10490 if (!State.isVarArg())
10494 return State.AllocateReg(
ArgGPRs);
10502 unsigned ValNo2,
MVT ValVT2,
MVT LocVT2,
10504 unsigned GRLenInBytes = GRLen / 8;
10515 State.AllocateStack(GRLenInBytes, StackAlign),
10518 ValNo2, ValVT2, State.AllocateStack(GRLenInBytes,
Align(GRLenInBytes)),
10529 ValNo2, ValVT2, State.AllocateStack(GRLenInBytes,
Align(GRLenInBytes)),
10537 unsigned ValNo,
MVT ValVT,
10540 unsigned GRLen =
DL.getLargestLegalIntTypeSizeInBits();
10541 assert((GRLen == 32 || GRLen == 64) &&
"Unspport GRLen");
10542 MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64;
10547 if (IsRet && ValNo > 1)
10551 bool UseGPRForFloat =
true;
10561 UseGPRForFloat = ArgFlags.
isVarArg();
10574 unsigned TwoGRLenInBytes = (2 * GRLen) / 8;
10577 DL.getTypeAllocSize(OrigTy) == TwoGRLenInBytes) {
10578 unsigned RegIdx = State.getFirstUnallocated(
ArgGPRs);
10580 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
10586 State.getPendingArgFlags();
10589 "PendingLocs and PendingArgFlags out of sync");
10593 UseGPRForFloat =
true;
10595 if (UseGPRForFloat && ValVT == MVT::f32) {
10598 }
else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) {
10601 }
else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) {
10604 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
10646 PendingLocs.
size() <= 2) {
10647 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
10652 PendingLocs.
clear();
10653 PendingArgFlags.
clear();
10660 unsigned StoreSizeBytes = GRLen / 8;
10663 if (ValVT == MVT::f32 && !UseGPRForFloat) {
10665 }
else if (ValVT == MVT::f64 && !UseGPRForFloat) {
10669 UseGPRForFloat =
false;
10670 StoreSizeBytes = 16;
10671 StackAlign =
Align(16);
10674 UseGPRForFloat =
false;
10675 StoreSizeBytes = 32;
10676 StackAlign =
Align(32);
10682 Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
10686 if (!PendingLocs.
empty()) {
10688 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
10689 for (
auto &It : PendingLocs) {
10691 It.convertToReg(
Reg);
10696 PendingLocs.clear();
10697 PendingArgFlags.
clear();
10700 assert((!UseGPRForFloat || LocVT == GRLenVT) &&
10701 "Expected an GRLenVT at this stage");
10718void LoongArchTargetLowering::analyzeInputArgs(
10721 LoongArchCCAssignFn Fn)
const {
10723 for (
unsigned i = 0, e = Ins.
size(); i != e; ++i) {
10724 MVT ArgVT = Ins[i].VT;
10725 Type *ArgTy =
nullptr;
10727 ArgTy = FType->getReturnType();
10728 else if (Ins[i].isOrigArg())
10729 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
10733 CCInfo, IsRet, ArgTy)) {
10734 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type " << ArgVT
10741void LoongArchTargetLowering::analyzeOutputArgs(
10744 CallLoweringInfo *CLI, LoongArchCCAssignFn Fn)
const {
10745 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
10746 MVT ArgVT = Outs[i].VT;
10747 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
10751 CCInfo, IsRet, OrigTy)) {
10752 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type " << ArgVT
10771 Val = DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
DL, MVT::f32, Val);
10793 if (In.isOrigArg()) {
10798 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
10799 (
BitWidth < 32 && In.Flags.isZExt())) {
10849 Register LoVReg =
RegInfo.createVirtualRegister(&LoongArch::GPRRegClass);
10862 Register HiVReg =
RegInfo.createVirtualRegister(&LoongArch::GPRRegClass);
10866 return DAG.
getNode(LoongArchISD::BUILD_PAIR_F64,
DL, MVT::f64,
Lo,
Hi);
10880 Val = DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Val);
10892 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
10896 LoongArch::R23, LoongArch::R24, LoongArch::R25,
10897 LoongArch::R26, LoongArch::R27, LoongArch::R28,
10898 LoongArch::R29, LoongArch::R30, LoongArch::R31};
10905 if (LocVT == MVT::f32) {
10908 static const MCPhysReg FPR32List[] = {LoongArch::F24, LoongArch::F25,
10909 LoongArch::F26, LoongArch::F27};
10916 if (LocVT == MVT::f64) {
10919 static const MCPhysReg FPR64List[] = {LoongArch::F28_64, LoongArch::F29_64,
10920 LoongArch::F30_64, LoongArch::F31_64};
10939 switch (CallConv) {
10951 "GHC calling convention requires the F and D extensions");
10956 MVT GRLenVT = Subtarget.getGRLenVT();
10957 unsigned GRLenInBytes = Subtarget.getGRLen() / 8;
10966 return CI->isMustTailCall();
10971 std::vector<SDValue> OutChains;
10980 analyzeInputArgs(MF, CCInfo, Ins,
false,
CC_LoongArch);
10982 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
10999 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
11008 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
11009 assert(ArgPartOffset == 0);
11010 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
11012 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
11036 int VaArgOffset, VarArgsSaveSize;
11040 if (ArgRegs.
size() == Idx) {
11042 VarArgsSaveSize = 0;
11044 VarArgsSaveSize = GRLenInBytes * (ArgRegs.
size() - Idx);
11045 VaArgOffset = -VarArgsSaveSize;
11051 LoongArchFI->setVarArgsFrameIndex(FI);
11059 VarArgsSaveSize += GRLenInBytes;
11064 for (
unsigned I = Idx;
I < ArgRegs.
size();
11065 ++
I, VaArgOffset += GRLenInBytes) {
11066 const Register Reg = RegInfo.createVirtualRegister(RC);
11067 RegInfo.addLiveIn(ArgRegs[
I], Reg);
11075 ->setValue((
Value *)
nullptr);
11076 OutChains.push_back(
Store);
11078 LoongArchFI->setVarArgsSaveSize(VarArgsSaveSize);
11083 if (!OutChains.empty()) {
11084 OutChains.push_back(Chain);
11099 if (
N->getNumValues() != 1)
11101 if (!
N->hasNUsesOfValue(1, 0))
11104 SDNode *Copy = *
N->user_begin();
11110 if (Copy->getGluedNode())
11114 bool HasRet =
false;
11116 if (
Node->getOpcode() != LoongArchISD::RET)
11124 Chain = Copy->getOperand(0);
11129bool LoongArchTargetLowering::isEligibleForTailCallOptimization(
11133 auto CalleeCC = CLI.CallConv;
11134 auto &Outs = CLI.Outs;
11136 auto CallerCC = Caller.getCallingConv();
11138 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
11147 for (
auto &Arg : Outs)
11148 if (Arg.Flags.isByVal())
11164 for (
auto &VA : ArgLocs)
11170 auto IsCallerStructRet = Caller.hasStructRetAttr();
11171 auto IsCalleeStructRet = Outs.empty() ?
false : Outs[0].Flags.isSRet();
11172 if (IsCallerStructRet || IsCalleeStructRet)
11177 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
11178 if (CalleeCC != CallerCC) {
11179 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
11180 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
11206 MVT GRLenVT = Subtarget.getGRLenVT();
11218 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
CC_LoongArch);
11222 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
11228 "site marked musttail");
11235 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
11237 if (!Flags.isByVal())
11241 unsigned Size = Flags.getByValSize();
11242 Align Alignment = Flags.getNonZeroByValAlign();
11249 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment, Alignment,
11251 false,
nullptr, std::nullopt,
11263 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
11266 SDValue ArgValue = OutVals[OutIdx];
11274 DAG.
getNode(LoongArchISD::SPLIT_PAIR_F64,
DL,
11275 DAG.
getVTList(MVT::i32, MVT::i32), ArgValue);
11287 if (!StackPtr.getNode())
11299 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
11313 unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex;
11333 const Argument *FormalArg =
nullptr;
11334 unsigned FilteredIdx = 0;
11335 for (
const auto &CallArg : CLI.
CB->
args()) {
11336 if (CallArg->getType()->isEmptyTy())
11338 if (FilteredIdx == CallArgIdx) {
11347 unsigned FormalArgIdx = CallArgIdx;
11349 FormalArgIdx = FormalArg->
getArgNo();
11353 if (Arg.getType()->isEmptyTy())
11355 if (FilteredIdx == CallArgIdx) {
11356 FormalArgIdx = Arg.getArgNo();
11369 SDValue IncomingPtr = CopyOp;
11386 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
11387 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
11388 SDValue PartValue = OutVals[OutIdx + 1];
11389 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
11400 ArgValue = IncomingPtr;
11404 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
11416 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
11417 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
11418 assert(ArgPartOffset == 0);
11423 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
11424 SDValue PartValue = OutVals[OutIdx + 1];
11425 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
11437 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
11439 for (
const auto &Part : Parts) {
11440 SDValue PartValue = Part.first;
11441 SDValue PartOffset = Part.second;
11448 ArgValue = SpillSlot;
11455 if (Flags.isByVal())
11456 ArgValue = ByValArgs[j++];
11464 "Tail call not allowed if stack is used for passing parameters");
11467 if (!StackPtr.getNode())
11480 if (!MemOpChains.
empty())
11486 for (
auto &Reg : RegsToPass) {
11487 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
11509 Ops.push_back(Chain);
11510 Ops.push_back(Callee);
11514 for (
auto &Reg : RegsToPass)
11515 Ops.push_back(DAG.
getRegister(Reg.first, Reg.second.getValueType()));
11520 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
11521 assert(Mask &&
"Missing call preserved mask for calling convention");
11527 Ops.push_back(Glue);
11536 Op = IsTailCall ? LoongArchISD::TAIL : LoongArchISD::CALL;
11539 Op = IsTailCall ? LoongArchISD::TAIL_MEDIUM : LoongArchISD::CALL_MEDIUM;
11542 assert(Subtarget.is64Bit() &&
"Large code model requires LA64");
11543 Op = IsTailCall ? LoongArchISD::TAIL_LARGE : LoongArchISD::CALL_LARGE;
11565 analyzeInputArgs(MF, RetCCInfo, Ins,
true,
CC_LoongArch);
11568 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
11569 auto &VA = RVLocs[i];
11577 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
11578 assert(VA.needsCustom());
11583 RetValue = DAG.
getNode(LoongArchISD::BUILD_PAIR_F64,
DL, MVT::f64,
11584 RetValue, RetValue2);
11597 const Type *RetTy)
const {
11599 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
11601 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
11605 Outs[i].Flags, CCInfo,
true,
nullptr))
11631 for (
unsigned i = 0, e = RVLocs.
size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
11632 SDValue Val = OutVals[OutIdx];
11641 DAG.
getVTList(MVT::i32, MVT::i32), Val);
11645 Register RegHi = RVLocs[++i].getLocReg();
11670 return DAG.
getNode(LoongArchISD::RET,
DL, MVT::Other, RetOps);
11678 const APInt &SplatValue,
const unsigned SplatBitSize)
const {
11679 uint64_t RequiredImm = 0;
11681 if (SplatBitSize == 16 && !(V & 0x00FF)) {
11683 RequiredImm = (0b10101 << 8) | (V >> 8);
11684 return {
true, RequiredImm};
11685 }
else if (SplatBitSize == 32) {
11687 if (!(V & 0xFFFF00FF)) {
11688 RequiredImm = (0b10001 << 8) | (V >> 8);
11689 return {
true, RequiredImm};
11692 if (!(V & 0xFF00FFFF)) {
11693 RequiredImm = (0b10010 << 8) | (V >> 16);
11694 return {
true, RequiredImm};
11697 if (!(V & 0x00FFFFFF)) {
11698 RequiredImm = (0b10011 << 8) | (V >> 24);
11699 return {
true, RequiredImm};
11702 if ((V & 0xFFFF00FF) == 0xFF) {
11703 RequiredImm = (0b10110 << 8) | (V >> 8);
11704 return {
true, RequiredImm};
11707 if ((V & 0xFF00FFFF) == 0xFFFF) {
11708 RequiredImm = (0b10111 << 8) | (V >> 16);
11709 return {
true, RequiredImm};
11712 if ((V & 0x7E07FFFF) == 0x3E000000 || (V & 0x7E07FFFF) == 0x40000000) {
11714 (0b11010 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
11715 return {
true, RequiredImm};
11717 }
else if (SplatBitSize == 64) {
11719 if ((V & 0xFFFFFFFF7E07FFFFULL) == 0x3E000000ULL ||
11720 (V & 0xFFFFFFFF7E07FFFFULL) == 0x40000000ULL) {
11722 (0b11011 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
11723 return {
true, RequiredImm};
11726 if ((V & 0x7FC0FFFFFFFFFFFFULL) == 0x4000000000000000ULL ||
11727 (V & 0x7FC0FFFFFFFFFFFFULL) == 0x3FC0000000000000ULL) {
11729 (0b11100 << 8) | (((V >> 56) & 0xC0) ^ 0x40) | ((V >> 48) & 0x3F);
11730 return {
true, RequiredImm};
11733 auto sameBitsPreByte = [](uint64_t x) -> std::pair<bool, uint8_t> {
11735 for (
int i = 0; i < 8; ++i) {
11737 if (
byte == 0 ||
byte == 0xFF)
11738 res |= ((
byte & 1) << i);
11743 return {
true, res};
11745 auto [IsSame, Suffix] = sameBitsPreByte(V);
11747 RequiredImm = (0b11001 << 8) | Suffix;
11748 return {
true, RequiredImm};
11751 return {
false, RequiredImm};
11756 if (!Subtarget.hasExtLSX())
11759 if (VT == MVT::f32) {
11760 uint64_t masked =
Imm.bitcastToAPInt().getZExtValue() & 0x7e07ffff;
11761 return (masked == 0x3e000000 || masked == 0x40000000);
11764 if (VT == MVT::f64) {
11765 uint64_t masked =
Imm.bitcastToAPInt().getZExtValue() & 0x7fc0ffffffffffff;
11766 return (masked == 0x3fc0000000000000 || masked == 0x4000000000000000);
11775bool LoongArchTargetLowering::shouldConvertFpToSat(
unsigned Op,
EVT FPVT,
11781 if (FPVT == MVT::f32)
11783 return FPVT == MVT::f64 &&
Subtarget.hasBasicD();
11787LoongArchTargetLowering::getFPImmAction(
const APFloat &
Imm,
EVT VT,
11788 bool ForCodeSize)
const {
11790 if (VT == MVT::f32 && !
Subtarget.hasBasicF())
11792 if (VT == MVT::f64 && !
Subtarget.hasBasicD())
11810 int InsNum = Seq.size() + ((VT == MVT::f64 && !Subtarget.is64Bit()) ? 2 : 1);
11822bool LoongArchTargetLowering::shouldInsertFencesForAtomic(
11832 Type *Ty =
I->getOperand(0)->getType();
11834 unsigned Size = Ty->getIntegerBitWidth();
11854 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
11859 if (Subtarget.hasExtLASX())
11861 else if (Subtarget.hasExtLSX())
11868 EVT VT =
Y.getValueType();
11871 return Subtarget.hasExtLSX() && VT.
isInteger();
11882 case Intrinsic::loongarch_masked_atomicrmw_xchg_i32:
11883 case Intrinsic::loongarch_masked_atomicrmw_add_i32:
11884 case Intrinsic::loongarch_masked_atomicrmw_sub_i32:
11885 case Intrinsic::loongarch_masked_atomicrmw_nand_i32: {
11888 Info.memVT = MVT::i32;
11889 Info.ptrVal =
I.getArgOperand(0);
11891 Info.align =
Align(4);
11910 "Unable to expand");
11911 unsigned MinWordSize = 4;
11923 Value *AlignedAddr = Builder.CreateIntrinsic(
11924 Intrinsic::ptrmask, {PtrTy, IntTy},
11925 {Addr, ConstantInt::get(IntTy, ~(uint64_t)(MinWordSize - 1))},
nullptr,
11928 Value *AddrInt = Builder.CreatePtrToInt(Addr, IntTy);
11929 Value *PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1,
"PtrLSB");
11930 Value *ShiftAmt = Builder.CreateShl(PtrLSB, 3);
11931 ShiftAmt = Builder.CreateTrunc(ShiftAmt, WordType,
"ShiftAmt");
11932 Value *Mask = Builder.CreateShl(
11933 ConstantInt::get(WordType,
11934 (1 << (
DL.getTypeStoreSize(
ValueType) * 8)) - 1),
11936 Value *Inv_Mask = Builder.CreateNot(Mask,
"Inv_Mask");
11937 Value *ValOperand_Shifted =
11938 Builder.CreateShl(Builder.CreateZExt(AI->
getValOperand(), WordType),
11939 ShiftAmt,
"ValOperand_Shifted");
11942 NewOperand = Builder.CreateOr(ValOperand_Shifted, Inv_Mask,
"AndOperand");
11944 NewOperand = ValOperand_Shifted;
11947 Builder.CreateAtomicRMW(
Op, AlignedAddr, NewOperand,
Align(MinWordSize),
11950 Value *Shift = Builder.CreateLShr(NewAI, ShiftAmt,
"shifted");
11951 Value *Trunc = Builder.CreateTrunc(Shift,
ValueType,
"extracted");
11952 Value *FinalOldResult = Builder.CreateBitCast(Trunc,
ValueType);
11971 if (Subtarget.hasLAM_BH() && Subtarget.is64Bit() &&
11979 if (Subtarget.hasLAMCAS()) {
12001 return Intrinsic::loongarch_masked_atomicrmw_xchg_i64;
12003 return Intrinsic::loongarch_masked_atomicrmw_add_i64;
12005 return Intrinsic::loongarch_masked_atomicrmw_sub_i64;
12007 return Intrinsic::loongarch_masked_atomicrmw_nand_i64;
12009 return Intrinsic::loongarch_masked_atomicrmw_umax_i64;
12011 return Intrinsic::loongarch_masked_atomicrmw_umin_i64;
12013 return Intrinsic::loongarch_masked_atomicrmw_max_i64;
12015 return Intrinsic::loongarch_masked_atomicrmw_min_i64;
12025 return Intrinsic::loongarch_masked_atomicrmw_xchg_i32;
12027 return Intrinsic::loongarch_masked_atomicrmw_add_i32;
12029 return Intrinsic::loongarch_masked_atomicrmw_sub_i32;
12031 return Intrinsic::loongarch_masked_atomicrmw_nand_i32;
12033 return Intrinsic::loongarch_masked_atomicrmw_umax_i32;
12035 return Intrinsic::loongarch_masked_atomicrmw_umin_i32;
12037 return Intrinsic::loongarch_masked_atomicrmw_max_i32;
12039 return Intrinsic::loongarch_masked_atomicrmw_min_i32;
12051 if (Subtarget.hasLAMCAS())
12063 unsigned GRLen = Subtarget.getGRLen();
12065 Value *FailureOrdering =
12066 Builder.getIntN(Subtarget.getGRLen(),
static_cast<uint64_t
>(FailOrd));
12067 Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i32;
12069 CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
12070 CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
12071 NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
12072 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
12075 Value *Result = Builder.CreateIntrinsic(
12076 CmpXchgIntrID, Tys, {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
12078 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
12094 Builder.CreateNot(Mask,
"Inv_Mask"),
12101 unsigned GRLen = Subtarget.getGRLen();
12103 Builder.getIntN(GRLen,
static_cast<uint64_t
>(AI->
getOrdering()));
12110 Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
12111 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
12112 ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
12125 unsigned ValWidth =
12128 Builder.CreateSub(Builder.getIntN(GRLen, GRLen - ValWidth), ShiftAmt);
12129 Result = Builder.CreateCall(LlwOpScwLoop,
12130 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
12133 Builder.CreateCall(LlwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
12137 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
12161 return LoongArch::R4;
12166 return LoongArch::R5;
12177 int RefinementSteps = VT.
getScalarType() == MVT::f64 ? 2 : 1;
12178 return RefinementSteps;
12184 "Reciprocal estimate queried on unsupported target");
12212 int &RefinementSteps,
12213 bool &UseOneConstNR,
12214 bool Reciprocal)
const {
12216 "Enabled should never be Disabled here");
12218 if (!Subtarget.hasFrecipe())
12233 UseOneConstNR =
false;
12239 if (Reciprocal || RefinementSteps > 0)
12249 int &RefinementSteps)
const {
12251 "Enabled should never be Disabled here");
12253 if (!Subtarget.hasFrecipe())
12267 return DAG.
getNode(LoongArchISD::FRECIPE,
DL, VT, Operand);
12275LoongArchTargetLowering::getConstraintType(
StringRef Constraint)
const {
12295 if (Constraint.
size() == 1) {
12296 switch (Constraint[0]) {
12312 if (Constraint ==
"ZC" || Constraint ==
"ZB")
12321 return StringSwitch<InlineAsm::ConstraintCode>(ConstraintCode)
12328std::pair<unsigned, const TargetRegisterClass *>
12329LoongArchTargetLowering::getRegForInlineAsmConstraint(
12333 if (Constraint.
size() == 1) {
12334 switch (Constraint[0]) {
12339 return std::make_pair(0U, &LoongArch::GPRRegClass);
12341 return std::make_pair(0U, &LoongArch::GPRNoR0R1RegClass);
12343 if (Subtarget.hasBasicF() && VT == MVT::f32)
12344 return std::make_pair(0U, &LoongArch::FPR32RegClass);
12345 if (Subtarget.hasBasicD() && VT == MVT::f64)
12346 return std::make_pair(0U, &LoongArch::FPR64RegClass);
12347 if (Subtarget.hasExtLSX() &&
12348 TRI->isTypeLegalForClass(LoongArch::LSX128RegClass, VT))
12349 return std::make_pair(0U, &LoongArch::LSX128RegClass);
12350 if (Subtarget.hasExtLSX() && VT == MVT::i128)
12351 return std::make_pair(0U, &LoongArch::LSX128RegClass);
12352 if (Subtarget.hasExtLASX() &&
12353 TRI->isTypeLegalForClass(LoongArch::LASX256RegClass, VT))
12354 return std::make_pair(0U, &LoongArch::LASX256RegClass);
12374 bool IsFP = Constraint[2] ==
'f';
12375 std::pair<StringRef, StringRef> Temp = Constraint.
split(
'$');
12376 std::pair<unsigned, const TargetRegisterClass *>
R;
12381 unsigned RegNo =
R.first;
12382 if (LoongArch::F0 <= RegNo && RegNo <= LoongArch::F31) {
12383 if (Subtarget.hasBasicD() && (VT == MVT::f64 || VT == MVT::Other)) {
12384 unsigned DReg = RegNo - LoongArch::F0 + LoongArch::F0_64;
12385 return std::make_pair(DReg, &LoongArch::FPR64RegClass);
12395void LoongArchTargetLowering::LowerAsmOperandForConstraint(
12399 if (Constraint.
size() == 1) {
12400 switch (Constraint[0]) {
12407 Subtarget.getGRLenVT()));
12416 Subtarget.getGRLenVT()));
12422 if (
C->getZExtValue() == 0)
12442#define GET_REGISTER_MATCHER
12443#include "LoongArchGenAsmMatcher.inc"
12449 std::string NewRegName = Name.second.str();
12455 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
12456 if (!ReservedRegs.
test(Reg))
12473 const APInt &
Imm = ConstNode->getAPIntValue();
12475 if ((
Imm + 1).isPowerOf2() || (
Imm - 1).isPowerOf2() ||
12476 (1 -
Imm).isPowerOf2() || (-1 -
Imm).isPowerOf2())
12479 if (ConstNode->hasOneUse() &&
12480 ((
Imm - 2).isPowerOf2() || (
Imm - 4).isPowerOf2() ||
12481 (
Imm - 8).isPowerOf2() || (
Imm - 16).isPowerOf2()))
12487 if (ConstNode->hasOneUse() && !(
Imm.sge(-2048) &&
Imm.sle(4095))) {
12488 unsigned Shifts =
Imm.countr_zero();
12495 if (ImmPop == 3 || ImmPop == 5 || ImmPop == 9 || ImmPop == 17)
12499 APInt ImmSmall =
APInt(
Imm.getBitWidth(), 1ULL << Shifts,
true);
12500 if ((
Imm - ImmSmall).isPowerOf2() || (
Imm + ImmSmall).isPowerOf2() ||
12501 (ImmSmall -
Imm).isPowerOf2())
12511 Type *Ty,
unsigned AS,
12530 switch (AM.
Scale) {
12566 EVT MemVT = LD->getMemoryVT();
12567 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
12578 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
12587 if (
Y.getValueType().isVector())
12599 Type *Ty,
bool IsSigned)
const {
12600 if (Subtarget.is64Bit() && Ty->isIntegerTy(32))
12609 if (Subtarget.isSoftFPABI() && (
Type.isFloatingPoint() && !
Type.isVector() &&
12610 Type.getSizeInBits() < Subtarget.getGRLen()))
12620 Align &PrefAlign)
const {
12624 if (Subtarget.is64Bit()) {
12626 PrefAlign =
Align(8);
12629 PrefAlign =
Align(4);
12644bool LoongArchTargetLowering::splitValueIntoRegisterParts(
12646 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
12647 bool IsABIRegCopy =
CC.has_value();
12650 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
12651 PartVT == MVT::f32) {
12666SDValue LoongArchTargetLowering::joinRegisterPartsIntoValue(
12668 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
12669 bool IsABIRegCopy =
CC.has_value();
12671 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
12672 PartVT == MVT::f32) {
12689 if (VT == MVT::f16 && Subtarget.hasBasicF())
12695unsigned LoongArchTargetLowering::getNumRegistersForCallingConv(
12698 if (VT == MVT::f16 && Subtarget.hasBasicF())
12707 unsigned Opc =
Op.getOpcode();
12712 case LoongArchISD::VANYNONZERO:
12713 case LoongArchISD::VALLZERO:
12714 case LoongArchISD::VALL_ZERO:
12715 case LoongArchISD::VANY_ZERO:
12716 case LoongArchISD::VALL_NONZERO:
12717 case LoongArchISD::VANY_NONZERO: {
12719 Known.Zero.setBitsFrom(1);
12722 case LoongArchISD::VPICK_ZEXT_ELT: {
12726 assert(
Known.getBitWidth() >= VTBits &&
"Unexpected width!");
12727 Known.Zero.setBitsFrom(VTBits);
12730 case LoongArchISD::BSTRINS: {
12732 unsigned Msb =
Op.getConstantOperandVal(2);
12733 unsigned Lsb =
Op.getConstantOperandVal(3);
12736 Known.insertBits(Known2.
trunc(Msb - Lsb + 1), Lsb);
12739 case LoongArchISD::BSTRPICK: {
12742 unsigned Msb =
Op.getConstantOperandVal(1);
12743 unsigned Lsb =
Op.getConstantOperandVal(2);
12753 unsigned Depth)
const {
12754 switch (
Op.getOpcode()) {
12757 case LoongArchISD::BSTRINS: {
12758 unsigned BitWidth =
Op.getScalarValueSizeInBits();
12759 unsigned Msb =
Op.getConstantOperandVal(2);
12760 unsigned Lsb =
Op.getConstantOperandVal(3);
12764 return std::min(Tmp,
BitWidth - 1 - Msb);
12768 return Tmp > Lsb ? Tmp - Lsb : 1;
12777 unsigned Depth)
const {
12778 EVT VT =
Op.getValueType();
12780 unsigned Opc =
Op.getOpcode();
12784 case LoongArchISD::CRC_W_B_W:
12785 case LoongArchISD::CRC_W_H_W:
12786 case LoongArchISD::CRCC_W_B_W:
12787 case LoongArchISD::CRCC_W_H_W: {
12789 APInt DemandedSrcBits =
12791 Opc == LoongArchISD::CRCC_W_B_W)
12795 OriginalDemandedElts, KnownSrc, TLO,
Depth + 1);
12797 case LoongArchISD::VMSKLTZ:
12798 case LoongArchISD::XVMSKLTZ: {
12800 MVT SrcVT = Src.getSimpleValueType();
12805 if (OriginalDemandedBits.
countr_zero() >= NumElts)
12809 APInt KnownUndef, KnownZero;
12825 if (KnownSrc.
One[SrcBits - 1])
12826 Known.One.setLowBits(NumElts);
12827 else if (KnownSrc.
Zero[SrcBits - 1])
12828 Known.Zero.setLowBits(NumElts);
12832 Src, DemandedSrcBits, DemandedElts, TLO.
DAG,
Depth + 1))
12839 Op, OriginalDemandedBits, OriginalDemandedElts,
Known, TLO,
Depth);
12863 unsigned Index)
const {
12874 unsigned Index)
const {
12878 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
12893 Align StackAlign)
const {
12897 unsigned StackProbeSize =
12901 return StackProbeSize ? StackProbeSize : StackAlign.
value();
12905LoongArchTargetLowering::lowerDYNAMIC_STACKALLOC(
SDValue Op,
12919 const EVT VT =
Op.getValueType();
12930 Chain = DAG.
getNode(LoongArchISD::PROBED_ALLOCA, dl, MVT::Other, Chain, SP);
12940 const Register TargetReg =
MI.getOperand(0).getReg();
12943 const bool IsLA64 = Subtarget.is64Bit();
12944 const Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
12946 const uint64_t ProbeSize =
TLI->getStackProbeSize(MF, StackAlign);
12951 MF.
insert(MBBInsertPoint, LoopTestMBB);
12954 MF.
insert(MBBInsertPoint, ExitMBB);
12965 TII->get(IsLA64 ? LoongArch::SUB_D : LoongArch::SUB_W),
SPReg)
12971 TII->get(IsLA64 ? LoongArch::ST_D : LoongArch::ST_W))
12977 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(LoongArch::BLTU))
12992 MBB->addSuccessor(LoopTestMBB);
12994 MI.eraseFromParent();
12996 return ExitMBB->
begin()->getParent();
static MCRegister MatchRegisterName(StringRef Name)
static SDValue performSHLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
If the operand is a bitwise AND with a constant RHS, and the shift has a constant RHS and is the only...
static bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSELECT_CCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo & TII
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static uint64_t getConstant(const Value *IndexValue)
static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
bpf unwind BPF Unwind Lower
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
static bool isSigned(unsigned Opcode)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static SDValue performINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
const MCPhysReg ArgFPR32s[]
static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Dispatching routine to lower various 128-bit LoongArch vector shuffles.
static SDValue lowerVECTOR_SHUFFLE_XVSHUF4I(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVSHUF4I (if possible).
static SDValue lowerVECTOR_SHUFFLE_VPICKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPICKEV (if possible).
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_XVPICKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPICKOD (if possible).
static SDValue unpackF64OnLA32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static bool fitsRegularPattern(typename SmallVectorImpl< ValType >::const_iterator Begin, unsigned CheckStride, typename SmallVectorImpl< ValType >::const_iterator End, ValType ExpectedIndex, unsigned ExpectedIndexStride)
Determine whether a range fits a regular pattern of values.
static SDValue lowerVECTOR_SHUFFLE_IsReverse(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE whose result is the reversed source vector.
static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, unsigned Depth)
static SDValue performUINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performHorizWideningCombine(SDNode *N, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static std::pair< SDValue, EVT > stripSignExtendInReg(SDValue V)
static SDValue emitIntrinsicErrorMessage(SDValue Op, StringRef ErrorMsg, SelectionDAG &DAG)
static SDValue ExtendSrcToDst(SDNode *N, SelectionDAG &DAG, unsigned ExtendOp)
static cl::opt< bool > ZeroDivCheck("loongarch-check-zero-division", cl::Hidden, cl::desc("Trap on integer division by zero."), cl::init(false))
static SDValue lowerVECTOR_SHUFFLE_XVPERMI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVPERMI (if possible).
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VSHUF(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VSHUF.
static int getEstimateRefinementSteps(EVT VT, const LoongArchSubtarget &Subtarget)
static bool isSupportedReciprocalEstimateType(EVT VT, const LoongArchSubtarget &Subtarget)
static void emitErrorAndReplaceIntrinsicResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, StringRef ErrorMsg, bool WithChain=true)
static SDValue lowerVECTOR_SHUFFLEAsByteRotate(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE as byte rotate (if possible).
static SDValue checkIntrinsicImmArg(SDValue Op, unsigned ImmOp, SelectionDAG &DAG, bool IsSigned=false)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_XVINSVE0(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVINSVE0 (if possible).
static SDValue performMOVFR2GR_SCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VILVH(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VILVH (if possible).
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performDemandedBitsCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI, unsigned ValNo, MVT ValVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsRet, Type *OrigTy)
static std::tuple< unsigned, SDValue, EVT > matchBinOpWithSharedOperand(SDValue BinV, SDValue X)
static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue performSPLIT_PAIR_F64Combine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static MachineBasicBlock * emitSplitPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue lowerVectorBitSetImm(SDNode *Node, SelectionDAG &DAG)
static SDValue performSETCC_BITCASTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performEXTENDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_XVPACKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPACKOD (if possible).
static bool buildVPERMIInfo(ArrayRef< int > Mask, SDValue V1, SDValue V2, SmallVectorImpl< SDValue > &SrcVec, unsigned &MaskImm)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue combineAndNotIntoVANDN(SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
Try to fold: (and (xor X, -1), Y) -> (vandn X, Y).
static SDValue lowerBUILD_VECTORAsBroadCastLoad(BuildVectorSDNode *BVOp, const SDLoc &DL, SelectionDAG &DAG)
#define CRC_CASE_EXT_BINARYOP(NAME, NODE)
static SDValue lowerVectorBitRevImm(SDNode *Node, SelectionDAG &DAG)
static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size, unsigned Depth)
static bool isConstantSplatVector(SDValue N, APInt &SplatValue, unsigned MinSizeInBits)
static SDValue lowerVECTOR_SHUFFLEAsShift(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, const APInt &Zeroable)
Lower VECTOR_SHUFFLE as shift (if possible).
static SDValue lowerVECTOR_SHUFFLE_VSHUF4I(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VSHUF4I (if possible).
static SDValue truncateVecElts(SDNode *Node, SelectionDAG &DAG)
static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
static MachineBasicBlock * insertDivByZeroTrap(MachineInstr &MI, MachineBasicBlock *MBB)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE_VEXTRINS(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VEXTRINS (if possible).
static SDValue lowerVectorBitClear(SDNode *Node, SelectionDAG &DAG)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VPACKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPACKEV (if possible).
static MachineBasicBlock * emitPseudoVMSKCOND(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue performSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performVANDNCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
Do target-specific dag combines on LoongArchISD::VANDN nodes.
static void replaceVPICKVE2GRResults(SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, unsigned ResOp)
static SDValue lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const APInt &Zeroable)
Lower VECTOR_SHUFFLE as ZERO_EXTEND Or ANY_EXTEND (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVPERMIQ(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVPERMI_Q (if possible).
static SDValue legalizeIntrinsicImmArg(SDNode *Node, unsigned ImmOp, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, bool IsSigned=false)
static cl::opt< MaterializeFPImm > MaterializeFPImmInsNum("loongarch-materialize-float-imm", cl::Hidden, cl::desc("Maximum number of instructions used (including code sequence " "to generate the value and moving the value to FPR) when " "materializing floating-point immediates (default = 3)"), cl::init(MaterializeFPImm3Ins), cl::values(clEnumValN(NoMaterializeFPImm, "0", "Use constant pool"), clEnumValN(MaterializeFPImm2Ins, "2", "Materialize FP immediate within 2 instructions"), clEnumValN(MaterializeFPImm3Ins, "3", "Materialize FP immediate within 3 instructions"), clEnumValN(MaterializeFPImm4Ins, "4", "Materialize FP immediate within 4 instructions"), clEnumValN(MaterializeFPImm5Ins, "5", "Materialize FP immediate within 5 instructions"), clEnumValN(MaterializeFPImm6Ins, "6", "Materialize FP immediate within 6 instructions " "(behaves same as 5 on loongarch64)")))
static SDValue emitIntrinsicWithChainErrorMessage(SDValue Op, StringRef ErrorMsg, SelectionDAG &DAG)
static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2)
static unsigned getLoongArchWOpcode(unsigned Opcode)
const MCPhysReg ArgFPR64s[]
static MachineBasicBlock * emitPseudoCTPOP(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue performMOVGR2FR_WCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
#define IOCSRWR_CASE(NAME, NODE)
#define CRC_CASE_EXT_UNARYOP(NAME, NODE)
static SDValue lowerVECTOR_SHUFFLE_VPACKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPACKOD (if possible).
static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT, SDValue Src, const SDLoc &DL)
static SDValue isNOT(SDValue V, SelectionDAG &DAG)
static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Dispatching routine to lower various 256-bit LoongArch vector shuffles.
static SDValue lowerVECTOR_SHUFFLE_VREPLVEI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VREPLVEI (if possible).
static MachineBasicBlock * emitPseudoXVINSGR2VR(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
const MCPhysReg PreserveNoneArgGPRs[]
static void fillVector(ArrayRef< SDValue > Ops, SelectionDAG &DAG, SDLoc DL, const LoongArchSubtarget &Subtarget, SDValue &Vector, EVT ResTy)
static SDValue fillSubVectorFromBuildVector(BuildVectorSDNode *Node, SelectionDAG &DAG, SDLoc DL, const LoongArchSubtarget &Subtarget, EVT ResTy, unsigned first)
static bool isSelectPseudo(MachineInstr &MI)
static SDValue foldVMskZeroTest(SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue lowerVectorSplatImm(SDNode *Node, unsigned ImmOp, SelectionDAG &DAG, bool IsSigned=false)
const MCPhysReg ArgGPRs[]
static SDValue lowerVECTOR_SHUFFLE_XVPERM(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVPERM (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVILVL(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVILVL (if possible).
static SDValue performVNi128AddSubCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VPERMI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VPERMI (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVEXTRINS(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVEXTRINS (if possible).
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static void replaceVecCondBranchResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, unsigned ResOp)
#define ASRT_LE_GT_CASE(NAME)
static SDValue lowerVECTOR_SHUFFLE_XVPACKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPACKEV (if possible).
static SDValue matchDeinterleaveBuildVector(SDValue N, unsigned &StartIndex)
static SDValue performBR_CCCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static void computeZeroableShuffleElements(ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt &KnownUndef, APInt &KnownZero)
Compute whether each element of a shuffle is zeroable.
static SDValue combineFP_ROUND(SDValue N, const SDLoc &DL, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static bool isCheapVNi128VectorOperand(SDValue V, MVT PartVT, SelectionDAG &DAG)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue widenShuffleMask(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
static bool canonicalizeShuffleVectorByLane(const SDLoc &DL, MutableArrayRef< int > Mask, MVT VT, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Shuffle vectors by lane to generate more optimized instructions.
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE_XVILVH(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVILVH (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVSHUF(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVSHUF (if possible).
static void replaceCMP_XCHG_128Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static SDValue lowerVectorPickVE2GR(SDNode *N, SelectionDAG &DAG, unsigned ResOp)
static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static bool isVectorRegisterVT(EVT VT, SelectionDAG &DAG)
static SDValue matchHalfOf128BitLanes(SDValue N, bool isLow)
#define IOCSRRD_CASE(NAME, NODE)
static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
Attempts to match vector shuffle as byte rotation.
static SDValue lowerVECTOR_SHUFFLE_XVREPLVE0(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVREPLVE0 (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVPICKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPICKEV (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVREPLVEI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVREPLVEI (if possible).
static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable)
Attempts to match a shuffle mask against the VBSLL, VBSRL, VSLLI and VSRLI instruction.
static SDValue lowerVECTOR_SHUFFLE_VILVL(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VILVL (if possible).
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVectorBitClearImm(SDNode *Node, SelectionDAG &DAG)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE as lane permute and then shuffle (if possible).
static bool hasOnlyVectorUsers(LoadSDNode *Ld, SelectionDAG &DAG)
static void replaceINTRINSIC_WO_CHAINResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue MergeBlocksConvert(SDNode *N, SelectionDAG &DAG, unsigned Opcode, unsigned BlockBits)
static SDValue lowerVECTOR_SHUFFLE_VPICKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPICKOD (if possible).
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static Register allocateArgGPR(CCState &State)
static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
Test whether a shuffle mask is equivalent within each sub-lane.
static SDValue convertRMEncoding(SelectionDAG &DAG, const SDLoc &DL, MVT GRLenVT, SDValue RMValue)
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static constexpr MCPhysReg SPReg
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
MachineRegisterInfo & MRI
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool inRange(const MCExpr *Expr, int64_t MinValue, int64_t MaxValue, bool AllowSymbol=false)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static bool isSequentialOrUndefInRange(ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
Return true if every element in Mask, beginning from position Pos and ending in Pos + Size,...
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
This class represents an incoming formal argument to a Function.
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
AtomicOrdering getFailureOrdering() const
Returns the failure ordering constraint of this cmpxchg instruction.
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
Value * getPointerOperand()
bool isFloatingPointOperation() const
BinOp getOperation() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
LLVM_ABI StringRef getValueAsString() const
Return the attribute's value as a string.
LLVM Basic Block Representation.
bool test(unsigned Idx) const
Returns true if bit Idx is set.
size_type count() const
Returns the number of bits which are set.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
LLVM_ABI void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getReg(unsigned ValNo, MVT ValVT, MCRegister Reg, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, MCRegister Reg, MVT LocVT, LocInfo HTP)
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
This class represents a function call, abstracting a target machine's calling convention.
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
The size in bits of the pointer representation in a given address space.
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Argument * getArg(unsigned i) const
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Common base class shared among various IRBuilders.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
LoongArchMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private Lo...
void setIncomingIndirectArg(unsigned ArgIndex, Register Reg)
void addSExt32Register(Register Reg)
Register getIncomingIndirectArg(unsigned ArgIndex) const
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const override
Hooks for building estimates in place of slower divisions and square roots.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool isFPImmCheapInGPR(const APFloat &Imm, EVT VT) const
Returns true if building Imm in a GPR and moving it to a float register is cheaper than loading it fr...
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
std::pair< bool, uint64_t > isImmVLDILegalForMode1(const APInt &SplatValue, const unsigned SplatBitSize) const
Check if a constant splat can be generated using [x]vldi, where imm[12] is 1.
void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool hasInlineStackProbe(const MachineFunction &MF) const override
True if stack clash protection is enabled for this function.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Determine if the target supports unaligned memory accesses.
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return the cost of extracting a subvector of type ResVT from a vector of type SrcVT,...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
MachineBasicBlock * emitDynamicProbedAlloc(MachineInstr &MI, MachineBasicBlock *MBB) const
bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isFPImmVLDILegal(const APFloat &Imm, EVT VT) const
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
unsigned getStackProbeSize(const MachineFunction &MF, Align StackAlign) const
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
void emitExpandAtomicRMW(AtomicRMWInst *AI) const override
Perform a atomicrmw expansion using a target-specific way.
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
LoongArchTargetLowering(const TargetMachine &TM, const LoongArchSubtarget &STI)
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const override
Return a reciprocal estimate value for the input operand.
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context, const Type *RetTy) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
Wrapper class representing physical registers. Should be passed by value.
bool hasFeature(unsigned Feature) const
static MVT getFloatingPointVT(unsigned BitWidth)
bool is128BitVector() const
Return true if this is a 128-bit vector type.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
bool is256BitVector() const
Return true if this is a 256-bit vector type.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
void push_back(MachineInstr *MI)
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
LLVM_ABI void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
void setIsKill(bool Val=true)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Class to represent pointers.
unsigned getAddressSpace() const
Return the address space of the Pointer type.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
LLVM_ABI bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
size_t use_size() const
Return the number of uses of this node.
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl, SDNodeFlags Flags={})
Constant fold a setcc to true or false.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getExtractSubvector(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Return the VT typed sub-vector of Vec at Idx.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getInsertSubvector(const SDLoc &DL, SDValue Vec, SDValue SubVec, unsigned Idx)
Insert SubVec at the Idx element of Vec.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
LLVM_ABI SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
LLVM_ABI bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static LLVM_ABI bool isZeroEltSplatMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses all elements with the same value as the first element of exa...
static LLVM_ABI bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
ArrayRef< int > getMask() const
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
typename SuperClass::const_iterator const_iterator
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr size_t size() const
Get the string size.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setMaxBytesForAlignment(unsigned MaxBytes)
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
ExtractSubvectorCost
Enum that specifies how expensive lowering an EXTRACT_SUBVECTOR is.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
FPImmAction
How the legalizer should materialize a floating-point immediate.
@ ConstantPool
Load the immediate from the constant pool.
@ Legal
The target can instruction select the immediate natively.
@ Bitcast
Build the immediate as a bitcast of an integer constant with the same bits.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
TargetLowering(const TargetLowering &)=delete
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Primary interface to the complete machine description for the target machine.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
bool shouldAssumeDSOLocal(const GlobalValue *GV) const
CodeModel::Model getCodeModel() const
Returns the code model.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetInstrInfo * getInstrInfo() const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI unsigned getIntegerBitWidth() const
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Value * getOperand(unsigned i) const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ Fast
Attempts to make calls as fast as possible (e.g.
@ PreserveNone
Used for runtime calls that preserves none general registers.
@ C
The default llvm calling convention, compatible with C.
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
LLVM_ABI bool isConstantSplatVectorAllZeros(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 ...
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ POISON
POISON - A poison node.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isExtVecInRegOpcode(unsigned Opcode)
LLVM_ABI bool isConstantSplatVectorAllOnes(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0...
LLVM_ABI bool isFreezeUndef(SDValue V)
Return true if the specified value is FREEZE(UNDEF).
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
LLVM_ABI bool isBuildVectorAllOnes(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are ~0 or undef.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LLVM_ABI bool isBuildVectorOfConstantSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantSDNode or undef.
LLVM_ABI NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
ABI getTargetABI(StringRef ABIName)
InstSeq generateInstSeq(int64_t Val)
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
constexpr RegState getKillRegState(bool B)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
LLVM_ABI bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr bool isShiftedInt(int64_t x)
Checks if a signed integer is an N bit number shifted left by S.
constexpr unsigned BitWidth
std::string join_items(Sep Separator, Args &&... Items)
Joins the strings in the parameter pack Items, adding Separator between the elements....
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
PointerUnion< const Value *, const PseudoSourceValue * > ValueType
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
@ End
Run the machine verifier once, at the end of the pipeline.
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
MCRegisterClass TargetRegisterClass
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
Align getNonZeroOrigAlign() const
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isBeforeLegalizeOps() const
bool isAfterLegalizeDAG() const
bool isBeforeLegalize() const
LLVM_ABI SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)