51#define DEBUG_TYPE "legalizevectorops"
55class VectorLegalizer {
67 LegalizedNodes.
insert(std::make_pair(
From, To));
70 LegalizedNodes.
insert(std::make_pair(To, To));
139 std::pair<SDValue, SDValue> ExpandLoad(
SDNode *
N);
197bool VectorLegalizer::Run() {
199 bool HasVectors =
false;
201 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I) {
221 DAG.AssignTopologicalOrder();
223 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I)
227 SDValue OldRoot = DAG.getRoot();
228 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
229 DAG.setRoot(LegalizedNodes[OldRoot]);
231 LegalizedNodes.clear();
234 DAG.RemoveDeadNodes();
241 "Unexpected number of results");
243 for (
unsigned i = 0, e =
Op->getNumValues(); i != e; ++i)
244 AddLegalizedOperand(
Op.getValue(i),
SDValue(Result, i));
249VectorLegalizer::RecursivelyLegalizeResults(
SDValue Op,
252 "Unexpected number of results");
254 for (
unsigned i = 0, e =
Results.
size(); i != e; ++i) {
256 AddLegalizedOperand(
Op.getValue(i),
Results[i]);
266 if (
I != LegalizedNodes.end())
return I->second;
270 for (
const SDValue &Oper :
Op->op_values())
273 SDNode *
Node = DAG.UpdateNodeOperands(
Op.getNode(), Ops);
275 bool HasVectorValueOrOp =
278 [](
SDValue O) { return O.getValueType().isVector(); });
279 if (!HasVectorValueOrOp)
280 return TranslateLegalizeResults(
Op,
Node);
284 switch (
Op.getOpcode()) {
286 return TranslateLegalizeResults(
Op,
Node);
290 EVT LoadedVT =
LD->getMemoryVT();
292 Action = TLI.getLoadExtAction(ExtType,
LD->getValueType(0), LoadedVT);
297 EVT StVT =
ST->getMemoryVT();
298 MVT ValVT =
ST->getValue().getSimpleValueType();
299 if (StVT.
isVector() &&
ST->isTruncatingStore())
300 Action = TLI.getTruncStoreAction(ValVT, StVT);
304 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
307 if (Action == TargetLowering::Legal)
308 Action = TargetLowering::Expand;
310#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
311 case ISD::STRICT_##DAGN:
312#include "llvm/IR/ConstrainedOps.def"
313 ValVT =
Node->getValueType(0);
316 ValVT =
Node->getOperand(1).getValueType();
319 MVT OpVT =
Node->getOperand(1).getSimpleValueType();
321 Action = TLI.getCondCodeAction(CCCode, OpVT);
322 if (Action == TargetLowering::Legal)
323 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
325 Action = TLI.getOperationAction(
Node->getOpcode(), ValVT);
332 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
333 TLI.getStrictFPOperationAction(
Node->getOpcode(), ValVT) ==
334 TargetLowering::Legal) {
336 if (TLI.getOperationAction(
Node->getOpcode(), EltVT)
337 == TargetLowering::Expand &&
338 TLI.getStrictFPOperationAction(
Node->getOpcode(), EltVT)
339 == TargetLowering::Legal)
340 Action = TargetLowering::Legal;
454 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
464 unsigned Scale =
Node->getConstantOperandVal(2);
465 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
466 Node->getValueType(0), Scale);
488 Action = TLI.getOperationAction(
Node->getOpcode(),
489 Node->getOperand(0).getValueType());
493 Action = TLI.getOperationAction(
Node->getOpcode(),
494 Node->getOperand(1).getValueType());
497 MVT OpVT =
Node->getOperand(0).getSimpleValueType();
499 Action = TLI.getCondCodeAction(CCCode, OpVT);
500 if (Action == TargetLowering::Legal)
501 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
505#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
507 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
508 : Node->getOperand(LEGALPOS).getValueType(); \
509 if (ISD::VPID == ISD::VP_SETCC) { \
510 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
511 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
512 if (Action != TargetLowering::Legal) \
516 if (!Node->getValueType(0).isVector() && \
517 Node->getValueType(0) != MVT::Other) { \
518 Action = TargetLowering::Legal; \
521 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
523#include "llvm/IR/VPIntrinsics.def"
531 case TargetLowering::Promote:
533 "This action is not supported yet!");
535 Promote(
Node, ResultVals);
536 assert(!ResultVals.
empty() &&
"No results for promotion?");
538 case TargetLowering::Legal:
541 case TargetLowering::Custom:
543 if (LowerOperationWrapper(
Node, ResultVals))
547 case TargetLowering::Expand:
549 Expand(
Node, ResultVals);
553 if (ResultVals.
empty())
554 return TranslateLegalizeResults(
Op,
Node);
557 return RecursivelyLegalizeResults(
Op, ResultVals);
562bool VectorLegalizer::LowerOperationWrapper(
SDNode *
Node,
574 if (
Node->getNumValues() == 1) {
582 "Lowering returned the wrong number of results!");
585 for (
unsigned I = 0, E =
Node->getNumValues();
I != E; ++
I)
591void VectorLegalizer::PromoteSETCC(
SDNode *
Node,
593 MVT VecVT =
Node->getOperand(0).getSimpleValueType();
594 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
601 Operands[0] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(0));
602 Operands[1] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(1));
605 if (
Node->getOpcode() == ISD::VP_SETCC) {
616void VectorLegalizer::PromoteSTRICT(
SDNode *
Node,
618 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
619 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
627 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
628 if (
Node->getOperand(j).getValueType().isVector() &&
635 {
Node->getOperand(0),
Node->getOperand(j)});
641 SDVTList VTs = DAG.getVTList(NewVecVT,
Node->getValueType(1));
651 DAG.getIntPtrConstant(0,
DL,
true)});
660 switch (
Node->getOpcode()) {
701 "Can't promote a vector with multiple results!");
702 MVT VT =
Node->getSimpleValueType(0);
703 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
707 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
711 if (
Node->getOperand(j).getValueType().isVector() && !SkipPromote)
712 if (
Node->getOperand(j)
714 .getVectorElementType()
715 .isFloatingPoint() &&
731 DAG.getIntPtrConstant(0, dl,
true));
738void VectorLegalizer::PromoteINT_TO_FP(
SDNode *
Node,
742 bool IsStrict =
Node->isStrictFPOpcode();
743 MVT VT =
Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
744 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
746 "Vectors have different number of elements!");
755 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
756 if (
Node->getOperand(j).getValueType().isVector())
757 Operands[j] = DAG.getNode(Opc, dl, NVT,
Node->getOperand(j));
779void VectorLegalizer::PromoteFP_TO_INT(
SDNode *
Node,
781 MVT VT =
Node->getSimpleValueType(0);
782 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
783 bool IsStrict =
Node->isStrictFPOpcode();
785 "Vectors have different number of elements!");
787 unsigned NewOpc =
Node->getOpcode();
801 Promoted = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
802 {
Node->getOperand(0),
Node->getOperand(1)});
805 Promoted = DAG.
getNode(NewOpc, dl, NVT,
Node->getOperand(0));
816 Promoted = DAG.
getNode(NewOpc, dl, NVT, Promoted,
824std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(
SDNode *
N) {
826 return TLI.scalarizeVectorLoad(LD, DAG);
831 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
836 switch (
Node->getOpcode()) {
838 std::pair<SDValue, SDValue> Tmp = ExpandLoad(
Node);
847 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
854 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(
Node));
857 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(
Node));
860 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(
Node));
885 if (
Node->getValueType(0).isScalableVector()) {
886 EVT CondVT = TLI.getSetCCResultType(
887 DAG.getDataLayout(), *DAG.getContext(),
Node->getValueType(0));
890 Node->getOperand(1),
Node->getOperand(4));
893 Node->getOperand(3)));
939 case ISD::VP_BITREVERSE:
940 if (
SDValue Expanded = TLI.expandVPBITREVERSE(
Node, DAG)) {
946 if (
SDValue Expanded = TLI.expandCTPOP(
Node, DAG)) {
952 if (
SDValue Expanded = TLI.expandVPCTPOP(
Node, DAG)) {
959 if (
SDValue Expanded = TLI.expandCTLZ(
Node, DAG)) {
965 case ISD::VP_CTLZ_ZERO_UNDEF:
966 if (
SDValue Expanded = TLI.expandVPCTLZ(
Node, DAG)) {
973 if (
SDValue Expanded = TLI.expandCTTZ(
Node, DAG)) {
979 case ISD::VP_CTTZ_ZERO_UNDEF:
980 if (
SDValue Expanded = TLI.expandVPCTTZ(
Node, DAG)) {
989 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG)) {
996 if (
SDValue Expanded = TLI.expandROT(
Node,
false , DAG)) {
1003 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG)) {
1010 Results.push_back(TLI.expandFMINIMUM_FMAXIMUM(
Node, DAG));
1016 if (
SDValue Expanded = TLI.expandIntMINMAX(
Node, DAG)) {
1037 if (
SDValue Expanded = TLI.expandAddSubSat(
Node, DAG)) {
1044 if (
SDValue Expanded = TLI.expandShlSat(
Node, DAG)) {
1052 if (
Node->getValueType(0).isScalableVector()) {
1053 if (
SDValue Expanded = TLI.expandFP_TO_INT_SAT(
Node, DAG)) {
1061 if (
SDValue Expanded = TLI.expandFixedPointMul(
Node, DAG)) {
1080#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1081 case ISD::STRICT_##DAGN:
1082#include "llvm/IR/ConstrainedOps.def"
1100 Results.push_back(TLI.expandVecReduce(
Node, DAG));
1104 Results.push_back(TLI.expandVecReduceSeq(
Node, DAG));
1114 if (tryExpandVecMathCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
1115 RTLIB::REM_F80, RTLIB::REM_F128,
1123 if (
Node->getNumValues() == 1) {
1127 "VectorLegalizer Expand returned wrong number of results!");
1137 EVT VT =
Node->getValueType(0);
1155 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1156 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1157 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand ||
1160 VT) == TargetLowering::Expand)
1161 return DAG.UnrollVectorOp(
Node);
1169 Mask = DAG.getSelect(
DL, BitTy, Mask, DAG.getAllOnesConstant(
DL, BitTy),
1170 DAG.getConstant(0,
DL, BitTy));
1173 Mask = DAG.getSplat(MaskTy,
DL, Mask);
1181 SDValue NotMask = DAG.getNOT(
DL, Mask, MaskTy);
1190 EVT VT =
Node->getValueType(0);
1193 if (TLI.getOperationAction(
ISD::SRA, VT) == TargetLowering::Expand ||
1194 TLI.getOperationAction(
ISD::SHL, VT) == TargetLowering::Expand)
1195 return DAG.UnrollVectorOp(
Node);
1198 EVT OrigTy = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1202 SDValue ShiftSz = DAG.getConstant(BW - OrigBW,
DL, VT);
1212 EVT VT =
Node->getValueType(0);
1215 EVT SrcVT = Src.getValueType();
1222 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1227 Src, DAG.getVectorIdxConstant(0,
DL));
1232 ShuffleMask.
resize(NumSrcElements, -1);
1235 int ExtLaneScale = NumSrcElements / NumElements;
1236 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1237 for (
int i = 0; i < NumElements; ++i)
1238 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1242 DAG.getVectorShuffle(SrcVT,
DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1247 EVT VT =
Node->getValueType(0);
1249 EVT SrcVT = Src.getValueType();
1260 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth,
DL, VT);
1271 EVT VT =
Node->getValueType(0);
1274 EVT SrcVT = Src.getValueType();
1281 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1286 Src, DAG.getVectorIdxConstant(0,
DL));
1294 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1296 int ExtLaneScale = NumSrcElements / NumElements;
1297 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1298 for (
int i = 0; i < NumElements; ++i)
1299 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1302 DAG.getVectorShuffle(SrcVT,
DL, Zero, Src, ShuffleMask));
1308 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
1309 ShuffleMask.
push_back((
I * ScalarSizeInBytes) + J);
1313 EVT VT =
Node->getValueType(0);
1317 return TLI.expandBSWAP(
Node, DAG);
1325 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1328 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1334 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1335 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1336 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1337 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT))
1338 return TLI.expandBSWAP(
Node, DAG);
1341 return DAG.UnrollVectorOp(
Node);
1344void VectorLegalizer::ExpandBITREVERSE(
SDNode *
Node,
1346 EVT VT =
Node->getValueType(0);
1350 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1365 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1370 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1372 (TLI.isOperationLegalOrCustom(
ISD::SHL, ByteVT) &&
1373 TLI.isOperationLegalOrCustom(
ISD::SRL, ByteVT) &&
1374 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, ByteVT) &&
1375 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, ByteVT)))) {
1378 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT),
1389 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1390 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1391 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1392 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT)) {
1393 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1417 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1418 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1419 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand)
1420 return DAG.UnrollVectorOp(
Node);
1426 auto BoolContents = TLI.getBooleanContents(Op1.
getValueType());
1427 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1428 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1430 return DAG.UnrollVectorOp(
Node);
1436 return DAG.UnrollVectorOp(
Node);
1444 SDValue NotMask = DAG.getNOT(
DL, Mask, VT);
1466 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1467 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1468 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1469 return DAG.UnrollVectorOp(
Node);
1473 return DAG.UnrollVectorOp(
Node);
1475 SDValue Ones = DAG.getAllOnesConstant(
DL, VT);
1478 Op1 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op1, Mask, Ones, EVL);
1479 Op2 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op2, NotMask, Ones, EVL);
1480 return DAG.getNode(ISD::VP_OR,
DL, VT, Op1, Op2, Ones, EVL);
1495 EVT MaskVT =
Mask.getValueType();
1507 return DAG.UnrollVectorOp(
Node);
1511 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1512 EVLVecVT) != MaskVT)
1513 return DAG.UnrollVectorOp(
Node);
1515 SDValue StepVec = DAG.getStepVector(
DL, EVLVecVT);
1516 SDValue SplatEVL = DAG.getSplat(EVLVecVT,
DL, EVL);
1518 DAG.getSetCC(
DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1521 return DAG.getSelect(
DL,
Node->getValueType(0), FullMask, Op1, Op2);
1526 EVT VT =
Node->getValueType(0);
1528 unsigned DivOpc =
Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1530 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1531 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1532 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1544 SDValue Mul = DAG.getNode(ISD::VP_MUL,
DL, VT, Divisor, Div, Mask, EVL);
1545 return DAG.getNode(ISD::VP_SUB,
DL, VT, Dividend,
Mul, Mask, EVL);
1548void VectorLegalizer::ExpandFP_TO_UINT(
SDNode *
Node,
1552 if (TLI.expandFP_TO_UINT(
Node, Result, Chain, DAG)) {
1554 if (
Node->isStrictFPOpcode())
1560 if (
Node->isStrictFPOpcode()) {
1568void VectorLegalizer::ExpandUINT_TO_FLOAT(
SDNode *
Node,
1570 bool IsStrict =
Node->isStrictFPOpcode();
1571 unsigned OpNo = IsStrict ? 1 : 0;
1573 EVT VT = Src.getValueType();
1579 if (TLI.expandUINT_TO_FP(
Node, Result, Chain, DAG)) {
1588 TargetLowering::Expand) ||
1590 TargetLowering::Expand)) ||
1591 TLI.getOperationAction(
ISD::SRL, VT) == TargetLowering::Expand) {
1602 assert((BW == 64 || BW == 32) &&
1603 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1605 SDValue HalfWord = DAG.getConstant(BW / 2,
DL, VT);
1610 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1611 SDValue HalfWordMask = DAG.getConstant(HWMask,
DL, VT);
1615 DAG.getConstantFP(1ULL << (BW / 2),
DL,
Node->getValueType(0));
1626 {
Node->getValueType(0), MVT::Other},
1627 {
Node->getOperand(0),
HI});
1631 {
Node->getValueType(0), MVT::Other},
1632 {
Node->getOperand(0),
LO});
1660 if (TLI.isOperationLegalOrCustom(
ISD::FSUB,
Node->getValueType(0))) {
1665 Node->getOperand(0));
1667 return DAG.UnrollVectorOp(
Node);
1670void VectorLegalizer::ExpandFSUB(
SDNode *
Node,
1675 EVT VT =
Node->getValueType(0);
1676 if (TLI.isOperationLegalOrCustom(
ISD::FNEG, VT) &&
1677 TLI.isOperationLegalOrCustom(
ISD::FADD, VT))
1684void VectorLegalizer::ExpandSETCC(
SDNode *
Node,
1686 bool NeedInvert =
false;
1687 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
1691 unsigned Offset = IsStrict ? 1 : 0;
1698 MVT OpVT =
LHS.getSimpleValueType();
1701 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1718 TLI.LegalizeSetCCCondCode(DAG,
Node->getValueType(0), LHS, RHS,
CC, Mask,
1719 EVL, NeedInvert, dl, Chain, IsSignaling);
1726 LHS = DAG.getNode(
Node->getOpcode(), dl,
Node->getVTList(),
1727 {Chain, LHS, RHS, CC},
Node->getFlags());
1728 Chain =
LHS.getValue(1);
1730 LHS = DAG.getNode(ISD::VP_SETCC, dl,
Node->getValueType(0),
1731 {LHS, RHS, CC, Mask, EVL},
Node->getFlags());
1742 LHS = DAG.getLogicalNOT(dl, LHS,
LHS->getValueType(0));
1744 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL,
LHS->getValueType(0));
1747 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
1751 EVT VT =
Node->getValueType(0);
1754 DAG.getBoolConstant(
true, dl, VT,
LHS.getValueType()),
1755 DAG.getBoolConstant(
false, dl, VT,
LHS.getValueType()),
CC);
1756 LHS->setFlags(
Node->getFlags());
1764void VectorLegalizer::ExpandUADDSUBO(
SDNode *
Node,
1767 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
1772void VectorLegalizer::ExpandSADDSUBO(
SDNode *
Node,
1775 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
1780void VectorLegalizer::ExpandMULO(
SDNode *
Node,
1783 if (!TLI.expandMULO(
Node, Result, Overflow, DAG))
1784 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(
Node);
1790void VectorLegalizer::ExpandFixedPointDiv(
SDNode *
Node,
1793 if (
SDValue Expanded = TLI.expandFixedPointDiv(
N->getOpcode(),
SDLoc(
N),
1794 N->getOperand(0),
N->getOperand(1),
N->getConstantOperandVal(2), DAG))
1798void VectorLegalizer::ExpandStrictFPOp(
SDNode *
Node,
1821 "Expected REM node");
1824 if (!TLI.expandREM(
Node, Result, DAG))
1839 assert(!
Node->isStrictFPOpcode() &&
"Unexpected strict fp operation!");
1841 const char *LCName = TLI.getLibcallName(LC);
1844 LLVM_DEBUG(
dbgs() <<
"Looking for vector variant of " << LCName <<
"\n");
1846 EVT VT =
Node->getValueType(0);
1864 for (
unsigned i = 0; i <
Node->getNumOperands(); ++i) {
1865 assert(
Node->getOperand(i).getValueType() == VT &&
1866 "Expected matching vector types!");
1869 FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys,
false);
1881 if (OptVFInfo->Shape.Parameters.size() !=
1890 Entry.IsSExt =
false;
1891 Entry.IsZExt =
false;
1894 for (
auto &VFParam : OptVFInfo->Shape.Parameters) {
1895 if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
1896 EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);
1897 Entry.Node = DAG.getBoolConstant(
true,
DL, MaskVT, VT);
1899 Args.push_back(Entry);
1904 if (VFParam.ParamKind != VFParamKind::Vector)
1909 Args.push_back(Entry);
1914 TLI.getPointerTy(DAG.getDataLayout()));
1917 .setChain(DAG.getEntryNode())
1920 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
1921 Results.push_back(CallResult.first);
1926bool VectorLegalizer::tryExpandVecMathCall(
1931 Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64,
1932 Call_F80, Call_F128, Call_PPCF128);
1934 if (LC == RTLIB::UNKNOWN_LIBCALL)
1940void VectorLegalizer::UnrollStrictFPOp(
SDNode *
Node,
1942 EVT VT =
Node->getValueType(0);
1945 unsigned NumOpers =
Node->getNumOperands();
1948 EVT TmpEltVT = EltVT;
1952 *DAG.getContext(), TmpEltVT);
1954 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1960 for (
unsigned i = 0; i < NumElems; ++i) {
1962 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1968 for (
unsigned j = 1;
j < NumOpers; ++
j) {
1985 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1986 DAG.getAllOnesConstant(dl, EltVT),
1987 DAG.getConstant(0, dl, EltVT));
2001 EVT VT =
Node->getValueType(0);
2007 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
2010 for (
unsigned i = 0; i < NumElems; ++i) {
2012 DAG.getVectorIdxConstant(i, dl));
2014 DAG.getVectorIdxConstant(i, dl));
2017 *DAG.getContext(), TmpEltVT),
2018 LHSElem, RHSElem,
CC);
2019 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
2020 DAG.getConstant(0, dl, EltVT));
2022 return DAG.getBuildVector(VT, dl, Ops);
2026 return VectorLegalizer(*this).Run();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
ilist< SDNode >::iterator allnodes_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Provides info so a possible vectorization of a function can be computed.
std::string getVectorFunctionABIVariantString() const
Returns a vector function ABI variant string on the form: ZGV<isa><mask><vlen><vparams><scalarname>(<...
StringRef getVectorFnName() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.