13 #include "llvm/IR/IntrinsicsHexagon.h"
20 cl::desc(
"Lower threshold (in bytes) for widening to HVX vectors"));
28 HexagonTargetLowering::initializeHVXLowering() {
73 auto setPromoteTo = [
this] (
unsigned Opc,
MVT FromTy,
MVT ToTy) {
97 for (
MVT T : FloatV) {
132 for (
MVT P : FloatW) {
162 for (
MVT T : LegalV) {
237 for (
MVT T : LegalW) {
324 for (
MVT T : LegalW) {
341 for (
MVT T : LegalV) {
368 int ElemWidth = ElemTy.getFixedSizeInBits();
369 int MaxElems = (8*HwLen) / ElemWidth;
370 for (
int N = 2;
N < MaxElems;
N *= 2) {
393 HexagonTargetLowering::getPreferredHvxVectorAction(
MVT VecTy)
const {
399 if (ElemTy ==
MVT::i1 && VecLen > HwLen)
423 unsigned HwWidth = 8*HwLen;
424 if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
442 HexagonTargetLowering::typeJoin(
const TypePair &Tys)
const {
443 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
447 Tys.second.getVectorNumElements());
450 HexagonTargetLowering::TypePair
451 HexagonTargetLowering::typeSplit(
MVT VecTy)
const {
454 assert((NumElem % 2) == 0 &&
"Expecting even-sized vector type");
456 return { HalfTy, HalfTy };
460 HexagonTargetLowering::typeExtElem(
MVT VecTy,
unsigned Factor)
const {
467 HexagonTargetLowering::typeTruncElem(
MVT VecTy,
unsigned Factor)
const {
474 HexagonTargetLowering::opCastElem(
SDValue Vec,
MVT ElemTy,
483 HexagonTargetLowering::opJoin(
const VectorPair &Ops,
const SDLoc &dl,
486 Ops.second, Ops.first);
489 HexagonTargetLowering::VectorPair
490 HexagonTargetLowering::opSplit(
SDValue Vec,
const SDLoc &dl,
492 TypePair Tys = typeSplit(ty(Vec));
495 return DAG.
SplitVector(Vec, dl, Tys.first, Tys.second);
499 HexagonTargetLowering::isHvxSingleTy(
MVT Ty)
const {
505 HexagonTargetLowering::isHvxPairTy(
MVT Ty)
const {
511 HexagonTargetLowering::isHvxBoolTy(
MVT Ty)
const {
516 bool HexagonTargetLowering::allowsHvxMemoryAccess(
531 bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
542 HexagonTargetLowering::convertToByteIndex(
SDValue ElemIdx,
MVT ElemTy,
551 unsigned L =
Log2_32(ElemWidth/8);
552 const SDLoc &dl(ElemIdx);
558 HexagonTargetLowering::getIndexInWord32(
SDValue Idx,
MVT ElemTy,
561 assert(ElemWidth >= 8 && ElemWidth <= 32);
567 const SDLoc &dl(Idx);
574 HexagonTargetLowering::getByteShuffle(
const SDLoc &dl,
SDValue Op0,
591 for (
unsigned I = 0;
I != ElemSize; ++
I)
592 ByteMask.push_back(-1);
594 int NewM =
M*ElemSize;
595 for (
unsigned I = 0;
I != ElemSize; ++
I)
596 ByteMask.push_back(NewM+
I);
601 opCastElem(Op1,
MVT::i8, DAG), ByteMask);
608 unsigned VecLen = Values.
size();
614 unsigned ElemSize = ElemWidth / 8;
615 assert(ElemSize*VecLen == HwLen);
621 assert((ElemSize == 1 || ElemSize == 2) &&
"Invalid element size");
622 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
624 for (
unsigned i = 0;
i != VecLen;
i += OpsPerWord) {
625 SDValue W = buildVector32(Values.
slice(
i, OpsPerWord), dl, PartVT, DAG);
633 unsigned NumValues = Values.
size();
636 for (
unsigned i = 0;
i != NumValues; ++
i) {
637 if (Values[
i].isUndef())
640 if (!SplatV.getNode())
642 else if (SplatV != Values[
i])
650 unsigned NumWords = Words.size();
652 bool IsSplat =
isSplat(Words, SplatV);
653 if (IsSplat && isUndef(SplatV))
657 auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.
getNode());
658 if (IdxN && IdxN->isZero())
659 return getZero(dl, VecTy, DAG);
668 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
687 auto IsBuildFromExtracts = [
this,&Values] (
SDValue &SrcVec,
692 SrcIdx.push_back(-1);
705 int I =
C->getSExtValue();
706 assert(
I >= 0 &&
"Negative element index");
715 if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
716 MVT ExtTy = ty(ExtVec);
718 if (ExtLen == VecLen || ExtLen == 2*VecLen) {
725 for (
int M : ExtIdx) {
735 for (
unsigned I = 0;
I != ExtLen; ++
I) {
736 if (
Mask.size() == ExtLen)
744 if (ExtLen == VecLen)
757 for (
unsigned i = 0;
i != NumWords; ++
i) {
759 if (Words[
i].isUndef())
761 for (
unsigned j =
i;
j != NumWords; ++
j)
762 if (Words[
i] == Words[
j])
765 if (VecHist[
i] > VecHist[
n])
769 SDValue HalfV = getZero(dl, VecTy, DAG);
770 if (VecHist[
n] > 1) {
785 for (
unsigned i = 0;
i != NumWords/2; ++
i) {
787 if (Words[
i] != Words[
n] || VecHist[
n] <= 1) {
794 if (Words[
i+NumWords/2] != Words[
n] || VecHist[
n] <= 1) {
798 {HalfV1, Words[
i+NumWords/2]});
821 HexagonTargetLowering::createHvxPrefixPred(
SDValue PredV,
const SDLoc &dl,
822 unsigned BitBytes,
bool ZeroFill,
SelectionDAG &DAG)
const {
823 MVT PredTy = ty(PredV);
842 for (
unsigned i = 0;
i != HwLen; ++
i) {
843 unsigned Num =
i % Scale;
844 unsigned Off =
i / Scale;
853 assert(BlockLen < HwLen &&
"vsetq(v1) prerequisite");
855 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
868 auto Lo32 = [&DAG, &dl] (
SDValue P) {
871 auto Hi32 = [&DAG, &dl] (
SDValue P) {
878 Words[IdxW].push_back(Hi32(W0));
879 Words[IdxW].push_back(Lo32(W0));
881 while (Bytes < BitBytes) {
886 for (
const SDValue &
W : Words[IdxW ^ 1]) {
888 Words[IdxW].push_back(Hi32(T));
889 Words[IdxW].push_back(Lo32(T));
892 for (
const SDValue &
W : Words[IdxW ^ 1]) {
893 Words[IdxW].push_back(
W);
894 Words[IdxW].push_back(
W);
900 assert(Bytes == BitBytes);
902 SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
904 for (
const SDValue &
W : Words[IdxW]) {
918 unsigned VecLen = Values.
size();
920 assert(VecLen <= HwLen || VecLen == 8*HwLen);
922 bool AllT =
true, AllF =
true;
925 if (
const auto *
N = dyn_cast<ConstantSDNode>(V.getNode()))
929 auto IsFalse = [] (
SDValue V) {
930 if (
const auto *
N = dyn_cast<ConstantSDNode>(V.getNode()))
935 if (VecLen <= HwLen) {
939 assert(HwLen % VecLen == 0);
940 unsigned BitBytes = HwLen / VecLen;
947 for (
unsigned B = 0;
B != BitBytes; ++
B)
948 Bytes.push_back(
Ext);
954 for (
unsigned I = 0;
I != VecLen;
I += 8) {
957 for (;
B != 8; ++
B) {
958 if (!Values[
I+
B].isUndef())
967 Bytes.push_back(
Ext);
981 SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
986 HexagonTargetLowering::extractHvxElementReg(
SDValue VecV,
SDValue IdxV,
991 assert(ElemWidth >= 8 && ElemWidth <= 32);
994 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
1003 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
1006 return extractVector(ExVec, SubIdx, dl, ElemTy,
MVT::i32, DAG);
1010 HexagonTargetLowering::extractHvxElementPred(
SDValue VecV,
SDValue IdxV,
1023 SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl,
MVT::i32, DAG);
1025 return getInstr(Hexagon::C2_cmpgtui, dl,
MVT::i1, {ExtB, Zero}, DAG);
1029 HexagonTargetLowering::insertHvxElementReg(
SDValue VecV,
SDValue IdxV,
1034 assert(ElemWidth >= 8 && ElemWidth <= 32);
1039 MVT VecTy = ty(VecV);
1051 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
1053 return InsertWord(VecV, ValV, ByteIdx);
1064 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
1065 MVT SubVecTy = tyVector(ty(
Ext), ElemTy);
1067 ValV, SubIdx, dl, ElemTy, DAG);
1070 return InsertWord(VecV,
Ins, ByteIdx);
1074 HexagonTargetLowering::insertHvxElementPred(
SDValue VecV,
SDValue IdxV,
1085 SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
1090 HexagonTargetLowering::extractHvxSubvectorReg(
SDValue VecV,
SDValue IdxV,
1092 MVT VecTy = ty(VecV);
1094 unsigned Idx = cast<ConstantSDNode>(IdxV.
getNode())->getZExtValue();
1101 if (isHvxPairTy(VecTy)) {
1103 if (Idx * ElemWidth >= 8*HwLen) {
1104 SubIdx = Hexagon::vsub_hi;
1107 SubIdx = Hexagon::vsub_lo;
1109 VecTy = typeSplit(VecTy).first;
1121 unsigned WordIdx = (Idx*ElemWidth) / 32;
1135 HexagonTargetLowering::extractHvxSubvectorPred(
SDValue VecV,
SDValue IdxV,
1137 MVT VecTy = ty(VecV);
1142 unsigned Idx = cast<ConstantSDNode>(IdxV.
getNode())->getZExtValue();
1146 unsigned Offset = Idx * BitBytes;
1157 for (
unsigned i = 0;
i != HwLen/Rep; ++
i) {
1158 for (
unsigned j = 0;
j != Rep; ++
j)
1159 Mask.push_back(
i + Offset);
1175 unsigned Rep = 8 / ResLen;
1178 for (
unsigned r = 0; r != HwLen/ResLen; ++r) {
1180 for (
unsigned i = 0;
i != ResLen; ++
i) {
1181 for (
unsigned j = 0;
j != Rep; ++
j)
1182 Mask.push_back(Offset +
i*BitBytes);
1194 return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
1199 HexagonTargetLowering::insertHvxSubvectorReg(
SDValue VecV,
SDValue SubV,
1201 MVT VecTy = ty(VecV);
1202 MVT SubTy = ty(SubV);
1207 bool IsPair = isHvxPairTy(VecTy);
1221 if (isHvxSingleTy(SubTy)) {
1222 if (
const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.
getNode())) {
1223 unsigned Idx = CN->getZExtValue();
1225 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
1247 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.
getNode());
1248 if (!IdxN || !IdxN->isZero()) {
1256 unsigned RolBase = HwLen;
1271 if (RolBase != 4 || !IdxN || !IdxN->isZero()) {
1286 HexagonTargetLowering::insertHvxSubvectorPred(
SDValue VecV,
SDValue SubV,
1288 MVT VecTy = ty(VecV);
1289 MVT SubTy = ty(SubV);
1296 assert(HwLen % VecLen == 0 &&
"Unexpected vector type");
1299 unsigned BitBytes = HwLen / VecLen;
1300 unsigned BlockLen = HwLen / Scale;
1304 SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes,
false, DAG);
1307 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.
getNode());
1308 if (!IdxN || !IdxN->isZero()) {
1317 assert(BlockLen < HwLen &&
"vsetq(v1) prerequisite");
1319 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1321 ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
1323 if (!IdxN || !IdxN->isZero()) {
1332 HexagonTargetLowering::extendHvxVectorPred(
SDValue VecV,
const SDLoc &dl,
1344 SDValue False = getZero(dl, ResTy, DAG);
1345 return DAG.
getSelect(dl, ResTy, VecV, True, False);
1349 HexagonTargetLowering::compressHvxPred(
SDValue VecQ,
const SDLoc &dl,
1359 MVT PredTy = ty(VecQ);
1361 assert(HwLen % PredLen == 0);
1368 for (
unsigned i = 0;
i != HwLen/8; ++
i) {
1369 for (
unsigned j = 0;
j != 8; ++
j)
1382 getZero(dl, VecTy, DAG));
1388 SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG);
1390 SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy,
1398 for (
unsigned i = 0;
i != HwLen; ++
i)
1399 Mask.push_back((8*
i) % HwLen +
i/(HwLen/8));
1411 unsigned Size =
Op.getNumOperands();
1413 for (
unsigned i = 0;
i !=
Size; ++
i)
1414 Ops.push_back(
Op.getOperand(
i));
1421 MVT SingleTy = typeSplit(VecTy).first;
1422 SDValue V0 = buildHvxVectorReg(
A.take_front(Size/2), dl, SingleTy, DAG);
1423 SDValue V1 = buildHvxVectorReg(
A.drop_front(Size/2), dl, SingleTy, DAG);
1428 return buildHvxVectorPred(Ops, dl, VecTy, DAG);
1435 for (
unsigned i = 0;
i !=
Size;
i++)
1439 tyVector(VecTy,
MVT::i16), NewOps);
1443 return buildHvxVectorReg(Ops, dl, VecTy, DAG);
1451 MVT ArgTy = ty(
Op.getOperand(0));
1472 unsigned NumOp =
Op.getNumOperands();
1485 for (
unsigned i = 0,
e = Elems.size();
i !=
e; ++
i) {
1532 MVT HalfTy = typeSplit(VecTy).first;
1534 Ops.take_front(NumOp/2));
1536 Ops.take_back(NumOp/2));
1545 for (
SDValue V :
Op.getNode()->op_values()) {
1546 SDValue P = createHvxPrefixPred(V, dl, BitBytes,
true, DAG);
1547 Prefixes.push_back(
P);
1553 SDValue Res = getZero(dl, ByteTy, DAG);
1554 for (
unsigned i = 0,
e = Prefixes.size();
i !=
e; ++
i) {
1570 return extractHvxElementPred(VecV, IdxV, dl, ty(
Op), DAG);
1572 return extractHvxElementReg(VecV, IdxV, dl, ty(
Op), DAG);
1585 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1595 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1602 MVT SrcTy = ty(SrcV);
1605 unsigned Idx = cast<ConstantSDNode>(IdxV.
getNode())->getZExtValue();
1612 return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1614 return extractHvxSubvectorReg(SrcV, IdxV, dl, DstTy, DAG);
1626 MVT VecTy = ty(VecV);
1629 return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1631 return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1644 return LowerHvxSignExt(
Op, DAG);
1654 return extendHvxVectorPred(InpV,
SDLoc(
Op), ty(
Op),
false, DAG);
1664 return extendHvxVectorPred(InpV,
SDLoc(
Op), ty(
Op),
true, DAG);
1675 assert(ResTy == ty(InpV));
1718 MVT ExtTy = typeExtElem(ResTy, 2);
1719 unsigned MpyOpc = ElemTy ==
MVT::i8
1720 ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv)
1721 : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv);
1722 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1725 for (
unsigned I = 0;
I < VecLen;
I += 2) {
1726 ShuffMask.push_back(
I+1);
1727 ShuffMask.push_back(
I+VecLen+1);
1729 VectorPair
P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1730 SDValue BS = getByteShuffle(dl,
P.first,
P.second, ShuffMask, DAG);
1752 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG);
1754 SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG);
1755 SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1758 SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {
T1, S16}, DAG);
1760 SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG);
1768 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh_64, dl,
PairTy, {Vs, Vt}, DAG);
1771 return opSplit(
T1, dl, DAG).second;
1776 return MulHS_V62(Vs, Vt);
1777 return MulHS_V60(Vs, Vt);
1782 auto LoVec = [&DAG,ResTy,dl] (
SDValue Pair) {
1785 auto HiVec = [&DAG,ResTy,dl] (
SDValue Pair) {
1790 SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy,
1795 SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl,
PairTy, {Vs, Vt}, DAG);
1799 SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG);
1802 SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt,
P}, DAG);
1803 SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl,
PairTy, {Vs, D0}, DAG);
1809 {LoVec(T2), HiVec(T2)}, DAG);
1812 SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG);
1822 MVT ValTy = ty(Val);
1828 SDValue VQ = compressHvxPred(Val, dl, WordTy, DAG);
1849 assert(Words.size() % 2 == 0);
1850 for (
unsigned i = 0,
e = Words.size();
i <
e;
i += 2) {
1853 Combines.push_back(
C);
1876 for (
unsigned I = 0;
I != HwLen / 8; ++
I) {
1880 for (
unsigned J = 0; J != 8; ++J) {
1881 Bytes.push_back(Byte);
1888 SDValue I2V = buildHvxVectorReg(Bytes, dl, ConstantVecTy, DAG);
1915 assert(HwLen % VecLen == 0);
1916 unsigned ElemSize = HwLen / VecLen;
1928 if (
SDValue S = getVectorShiftByInt(
Op, DAG))
1938 unsigned IntNo = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
1940 unsigned IntPredCast = Use64b ? Intrinsic::hexagon_V6_pred_typecast
1941 : Intrinsic::hexagon_V6_pred_typecast_128B;
1942 if (IntNo == IntPredCast) {
1945 if (isHvxBoolTy(ResTy) && isHvxBoolTy(OpTy)) {
1960 auto *MaskN = cast<MaskedLoadStoreSDNode>(
Op.getNode());
1962 SDValue Chain = MaskN->getChain();
1966 unsigned Opc =
Op->getOpcode();
1972 SDValue Thru = cast<MaskedLoadSDNode>(MaskN)->getPassThru();
1983 unsigned StoreOpc = Hexagon::V6_vS32b_qpred_ai;
1984 SDValue Value = cast<MaskedStoreSDNode>(MaskN)->getValue();
1987 if (MaskN->getAlign().value() % HwLen == 0) {
1996 SDValue Z = getZero(dl, ty(V), DAG);
2000 SDValue LoV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {V,
Z,
A}, DAG);
2001 SDValue HiV = getInstr(Hexagon::V6_vlalignb, dl, ty(V), {
Z, V,
A}, DAG);
2002 return std::make_pair(LoV, HiV);
2008 VectorPair Tmp = StoreAlign(MaskV,
Base);
2011 VectorPair ValueU = StoreAlign(
Value,
Base);
2016 {MaskU.first,
Base, Offset0, ValueU.first, Chain}, DAG);
2019 {MaskU.second,
Base, Offset1, ValueU.second, Chain}, DAG);
2021 DAG.
setNodeMemRefs(cast<MachineSDNode>(StoreHi.getNode()), {MemOp});
2033 MVT ArgTy = ty(
Op.getOperand(0));
2044 getInstr(Hexagon::V6_vmpy_qf32_hf, dl, VecTy, {F16Vec, Fp16Ones}, DAG);
2046 MVT HalfTy = typeSplit(VecTy).first;
2047 VectorPair Pair = opSplit(VmpyVec, dl, DAG);
2049 getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.first}, DAG);
2051 getInstr(Hexagon::V6_vconv_sf_qf32, dl, HalfTy, {Pair.second}, DAG);
2054 getInstr(Hexagon::V6_vshuffvdd, dl, VecTy,
2066 unsigned Opc =
Op.getOpcode();
2105 auto SplitVTNode = [&DAG,
this] (
const VTSDNode *
N) {
2106 MVT Ty = typeSplit(
N->getVT().getSimpleVT()).first;
2108 return std::make_pair(TV, TV);
2113 ? opSplit(A, dl, DAG)
2114 :
std::make_pair(
A,
A);
2117 if (
const auto *
N = dyn_cast<const VTSDNode>(
A.getNode()))
2120 OpsL.push_back(
P.first);
2121 OpsH.push_back(
P.second);
2125 MVT HalfTy = typeSplit(ResTy).first;
2134 auto *MemN = cast<MemSDNode>(
Op.getNode());
2136 MVT MemTy = MemN->getMemoryVT().getSimpleVT();
2137 if (!isHvxPairTy(MemTy))
2142 MVT SingleTy = typeSplit(MemTy).first;
2143 SDValue Chain = MemN->getChain();
2144 SDValue Base0 = MemN->getBasePtr();
2159 assert(cast<LoadSDNode>(
Op)->isUnindexed());
2168 assert(cast<StoreSDNode>(
Op)->isUnindexed());
2169 VectorPair Vals = opSplit(cast<StoreSDNode>(
Op)->getValue(), dl, DAG);
2177 auto MaskN = cast<MaskedLoadStoreSDNode>(
Op);
2178 assert(MaskN->isUnindexed());
2179 VectorPair Masks = opSplit(MaskN->getMask(), dl, DAG);
2184 opSplit(cast<MaskedLoadSDNode>(
Op)->getPassThru(), dl, DAG);
2186 DAG.
getMaskedLoad(SingleTy, dl, Chain, Base0, Offset, Masks.first,
2190 DAG.
getMaskedLoad(SingleTy, dl, Chain, Base1, Offset, Masks.second,
2199 VectorPair Vals = opSplit(cast<MaskedStoreSDNode>(
Op)->getValue(), dl, DAG);
2201 Masks.first, SingleTy, MOp0,
2204 Masks.second, SingleTy, MOp1,
2209 std::string
Name =
"Unexpected operation: " +
Op->getOperationName(&DAG);
2216 auto *LoadN = cast<LoadSDNode>(
Op.getNode());
2217 assert(LoadN->isUnindexed() &&
"Not widening indexed loads yet");
2218 assert(LoadN->getMemoryVT().getVectorElementType() !=
MVT::i1 &&
2219 "Not widening loads of i1 yet");
2221 SDValue Chain = LoadN->getChain();
2228 assert(ResLen < HwLen &&
"vsetq(v1) prerequisite");
2231 SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
2248 auto *StoreN = cast<StoreSDNode>(
Op.getNode());
2249 assert(StoreN->isUnindexed() &&
"Not widening indexed stores yet");
2250 assert(StoreN->getMemoryVT().getVectorElementType() !=
MVT::i1 &&
2251 "Not widening stores of i1 yet");
2253 SDValue Chain = StoreN->getChain();
2263 for (
unsigned Len = ValueLen; Len < HwLen; ) {
2267 assert(ty(
Value).getVectorNumElements() == HwLen);
2269 assert(ValueLen < HwLen &&
"vsetq(v1) prerequisite");
2271 SDValue Mask = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
2274 auto *
MemOp = MF.getMachineMemOperand(StoreN->getMemOperand(), 0, HwLen);
2282 SDValue Op0 =
Op.getOperand(0), Op1 =
Op.getOperand(1);
2292 SDValue WideOp0 = appendUndef(Op0, WideOpTy, DAG);
2293 SDValue WideOp1 = appendUndef(Op1, WideOpTy, DAG);
2297 {WideOp0, WideOp1,
Op.getOperand(2)});
2301 {SetCC, getZero(dl,
MVT::i32, DAG)});
2320 auto getFactor = [HwWidth](
MVT Ty) {
2322 return HwWidth >
Width ? HwWidth /
Width : 1;
2325 auto getWideTy = [getFactor](
MVT Ty) {
2332 SDValue WideOp = appendUndef(Op0, getWideTy(OpTy), DAG);
2333 SDValue WideRes = DAG.
getNode(Opcode, dl, getWideTy(ResTy), WideOp);
2353 auto getFactor = [HwWidth](
MVT Ty) {
2356 return HwWidth /
Width;
2359 auto getWideTy = [getFactor](
MVT Ty) {
2369 SDValue WideOp = appendUndef(Op0, getWideTy(OpTy), DAG);
2374 if (shouldWidenToHvx(ResTy, DAG))
2383 {WideRes, getZero(dl,
MVT::i32, DAG)});
2388 unsigned Opc =
Op.getOpcode();
2389 bool IsPairOp = isHvxPairTy(ty(
Op)) ||
2391 return isHvxPairTy(ty(V));
2402 return SplitHvxMemOp(
Op, DAG);
2408 return SplitHvxPairOp(
Op, DAG);
2437 return SplitHvxPairOp(
Op, DAG);
2459 case ISD::SRL:
return LowerHvxShift(
Op, DAG);
2483 HexagonTargetLowering::LowerHvxOperationWrapper(
SDNode *
N,
2485 unsigned Opc =
N->getOpcode();
2492 if (shouldWidenToHvx(ty(
Op.getOperand(0)), DAG)) {
2493 if (
SDValue T = WidenHvxExtend(
Op, DAG))
2498 if (shouldWidenToHvx(ty(
Op.getOperand(0)), DAG)) {
2504 if (shouldWidenToHvx(ty(
Op.getOperand(0)), DAG)) {
2505 if (
SDValue T = WidenHvxTruncate(
Op, DAG))
2510 if (shouldWidenToHvx(ty(cast<StoreSDNode>(
N)->getValue()), DAG)) {
2517 if (isHvxPairTy(ty(
Op))) {
2520 Results.push_back(
S.getOperand(0));
2521 Results.push_back(
S.getOperand(1));
2525 if (isHvxPairTy(ty(
Op->getOperand(1)))) {
2536 HexagonTargetLowering::ReplaceHvxNodeResults(
SDNode *
N,
2538 unsigned Opc =
N->getOpcode();
2544 if (shouldWidenToHvx(ty(
Op), DAG)) {
2545 if (
SDValue T = WidenHvxExtend(
Op, DAG))
2550 if (shouldWidenToHvx(ty(
Op), DAG)) {
2556 if (shouldWidenToHvx(ty(
Op), DAG)) {
2557 if (
SDValue T = WidenHvxTruncate(
Op, DAG))
2562 if (shouldWidenToHvx(ty(
Op), DAG)) {
2571 if (isHvxBoolTy(ty(
N->getOperand(0)))) {
2583 HexagonTargetLowering::PerformHvxDAGCombine(
SDNode *
N, DAGCombinerInfo &DCI)
2588 unsigned Opc =
Op.getOpcode();
2589 if (DCI.isBeforeLegalizeOps())
2607 if (
const auto *
C = dyn_cast<ConstantSDNode>(Ops[0].getOperand(0)))
2617 return getZero(dl, ty(
Op), DAG);
2620 if (isUndef(Ops[1]))
2625 SDValue Vec = Ops[0].getOperand(0);
2626 SDValue Rot0 = Ops[1], Rot1 = Ops[0].getOperand(1);
2638 HexagonTargetLowering::shouldWidenToHvx(
MVT Ty,
SelectionDAG &DAG)
const {
2639 auto Action = getPreferredHvxVectorAction(Ty);
2654 auto IsHvxTy = [
this](
EVT Ty) {
2655 return Ty.isSimple() && Subtarget.
isHVXVectorType(Ty.getSimpleVT(),
true);
2658 return Op.getValueType().isSimple() &&
2665 auto IsWidenedToHvx = [
this, &DAG](
SDValue Op) {
2666 if (!
Op.getValueType().isSimple())
2669 return ValTy.
isVector() && shouldWidenToHvx(ValTy, DAG);
2672 for (
int i = 0,
e =
N->getNumValues();
i !=
e; ++
i) {