51#define DEBUG_TYPE "legalizevectorops"
55class VectorLegalizer {
67 LegalizedNodes.
insert(std::make_pair(
From, To));
70 LegalizedNodes.
insert(std::make_pair(To, To));
139 std::pair<SDValue, SDValue> ExpandLoad(
SDNode *
N);
204bool VectorLegalizer::Run() {
206 bool HasVectors =
false;
208 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I) {
228 DAG.AssignTopologicalOrder();
230 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I)
234 SDValue OldRoot = DAG.getRoot();
235 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
236 DAG.setRoot(LegalizedNodes[OldRoot]);
238 LegalizedNodes.clear();
241 DAG.RemoveDeadNodes();
248 "Unexpected number of results");
250 for (
unsigned i = 0, e =
Op->getNumValues(); i != e; ++i)
251 AddLegalizedOperand(
Op.getValue(i),
SDValue(Result, i));
256VectorLegalizer::RecursivelyLegalizeResults(
SDValue Op,
259 "Unexpected number of results");
261 for (
unsigned i = 0, e =
Results.
size(); i != e; ++i) {
263 AddLegalizedOperand(
Op.getValue(i),
Results[i]);
273 if (
I != LegalizedNodes.end())
return I->second;
277 for (
const SDValue &Oper :
Op->op_values())
280 SDNode *
Node = DAG.UpdateNodeOperands(
Op.getNode(), Ops);
282 bool HasVectorValueOrOp =
285 [](
SDValue O) { return O.getValueType().isVector(); });
286 if (!HasVectorValueOrOp)
287 return TranslateLegalizeResults(
Op,
Node);
291 switch (
Op.getOpcode()) {
293 return TranslateLegalizeResults(
Op,
Node);
297 EVT LoadedVT =
LD->getMemoryVT();
299 Action = TLI.getLoadExtAction(ExtType,
LD->getValueType(0), LoadedVT);
304 EVT StVT =
ST->getMemoryVT();
305 MVT ValVT =
ST->getValue().getSimpleValueType();
306 if (StVT.
isVector() &&
ST->isTruncatingStore())
307 Action = TLI.getTruncStoreAction(ValVT, StVT);
311 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
314 if (Action == TargetLowering::Legal)
315 Action = TargetLowering::Expand;
317#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
318 case ISD::STRICT_##DAGN:
319#include "llvm/IR/ConstrainedOps.def"
320 ValVT =
Node->getValueType(0);
323 ValVT =
Node->getOperand(1).getValueType();
326 MVT OpVT =
Node->getOperand(1).getSimpleValueType();
328 Action = TLI.getCondCodeAction(CCCode, OpVT);
329 if (Action == TargetLowering::Legal)
330 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
332 Action = TLI.getOperationAction(
Node->getOpcode(), ValVT);
339 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
340 TLI.getStrictFPOperationAction(
Node->getOpcode(), ValVT) ==
341 TargetLowering::Legal) {
343 if (TLI.getOperationAction(
Node->getOpcode(), EltVT)
344 == TargetLowering::Expand &&
345 TLI.getStrictFPOperationAction(
Node->getOpcode(), EltVT)
346 == TargetLowering::Legal)
347 Action = TargetLowering::Legal;
452 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
462 unsigned Scale =
Node->getConstantOperandVal(2);
463 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
464 Node->getValueType(0), Scale);
486 Action = TLI.getOperationAction(
Node->getOpcode(),
487 Node->getOperand(0).getValueType());
491 Action = TLI.getOperationAction(
Node->getOpcode(),
492 Node->getOperand(1).getValueType());
495 MVT OpVT =
Node->getOperand(0).getSimpleValueType();
497 Action = TLI.getCondCodeAction(CCCode, OpVT);
498 if (Action == TargetLowering::Legal)
499 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
503#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
505 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
506 : Node->getOperand(LEGALPOS).getValueType(); \
507 if (ISD::VPID == ISD::VP_SETCC) { \
508 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
509 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
510 if (Action != TargetLowering::Legal) \
513 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
515#include "llvm/IR/VPIntrinsics.def"
523 case TargetLowering::Promote:
525 "This action is not supported yet!");
527 Promote(
Node, ResultVals);
528 assert(!ResultVals.
empty() &&
"No results for promotion?");
530 case TargetLowering::Legal:
533 case TargetLowering::Custom:
535 if (LowerOperationWrapper(
Node, ResultVals))
539 case TargetLowering::Expand:
541 Expand(
Node, ResultVals);
545 if (ResultVals.
empty())
546 return TranslateLegalizeResults(
Op,
Node);
549 return RecursivelyLegalizeResults(
Op, ResultVals);
554bool VectorLegalizer::LowerOperationWrapper(
SDNode *
Node,
566 if (
Node->getNumValues() == 1) {
574 "Lowering returned the wrong number of results!");
577 for (
unsigned I = 0, E =
Node->getNumValues();
I != E; ++
I)
583void VectorLegalizer::PromoteReduction(
SDNode *
Node,
585 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
586 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
587 MVT ScalarVT =
Node->getSimpleValueType(0);
594 if (
Node->getOperand(0).getValueType().isFloatingPoint())
601 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
602 if (
Node->getOperand(j).getValueType().isVector() &&
606 if (
Node->getOperand(j).getValueType().isFloatingPoint())
620 DAG.getIntPtrConstant(0,
DL,
true));
627void VectorLegalizer::PromoteSETCC(
SDNode *
Node,
629 MVT VecVT =
Node->getOperand(0).getSimpleValueType();
630 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
637 Operands[0] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(0));
638 Operands[1] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(1));
641 if (
Node->getOpcode() == ISD::VP_SETCC) {
652void VectorLegalizer::PromoteSTRICT(
SDNode *
Node,
654 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
655 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
663 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
664 if (
Node->getOperand(j).getValueType().isVector() &&
671 {
Node->getOperand(0),
Node->getOperand(j)});
677 SDVTList VTs = DAG.getVTList(NewVecVT,
Node->getValueType(1));
687 DAG.getIntPtrConstant(0,
DL,
true)});
696 switch (
Node->getOpcode()) {
711 case ISD::VP_REDUCE_ADD:
712 case ISD::VP_REDUCE_MUL:
713 case ISD::VP_REDUCE_AND:
714 case ISD::VP_REDUCE_OR:
715 case ISD::VP_REDUCE_XOR:
716 case ISD::VP_REDUCE_SMAX:
717 case ISD::VP_REDUCE_SMIN:
718 case ISD::VP_REDUCE_UMAX:
719 case ISD::VP_REDUCE_UMIN:
720 case ISD::VP_REDUCE_FADD:
721 case ISD::VP_REDUCE_FMUL:
722 case ISD::VP_REDUCE_FMAX:
723 case ISD::VP_REDUCE_FMIN:
724 case ISD::VP_REDUCE_SEQ_FADD:
754 "Can't promote a vector with multiple results!");
755 MVT VT =
Node->getSimpleValueType(0);
756 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
760 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
764 if (
Node->getOperand(j).getValueType().isVector() && !SkipPromote)
765 if (
Node->getOperand(j)
767 .getVectorElementType()
768 .isFloatingPoint() &&
784 DAG.getIntPtrConstant(0, dl,
true));
791void VectorLegalizer::PromoteINT_TO_FP(
SDNode *
Node,
795 bool IsStrict =
Node->isStrictFPOpcode();
796 MVT VT =
Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
797 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
799 "Vectors have different number of elements!");
808 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
809 if (
Node->getOperand(j).getValueType().isVector())
810 Operands[j] = DAG.getNode(Opc, dl, NVT,
Node->getOperand(j));
832void VectorLegalizer::PromoteFP_TO_INT(
SDNode *
Node,
834 MVT VT =
Node->getSimpleValueType(0);
835 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
836 bool IsStrict =
Node->isStrictFPOpcode();
838 "Vectors have different number of elements!");
840 unsigned NewOpc =
Node->getOpcode();
854 Promoted = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
855 {
Node->getOperand(0),
Node->getOperand(1)});
858 Promoted = DAG.
getNode(NewOpc, dl, NVT,
Node->getOperand(0));
869 Promoted = DAG.
getNode(NewOpc, dl, NVT, Promoted,
877std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(
SDNode *
N) {
879 return TLI.scalarizeVectorLoad(LD, DAG);
884 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
889 switch (
Node->getOpcode()) {
891 std::pair<SDValue, SDValue> Tmp = ExpandLoad(
Node);
900 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
907 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(
Node));
910 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(
Node));
913 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(
Node));
938 if (
Node->getValueType(0).isScalableVector()) {
939 EVT CondVT = TLI.getSetCCResultType(
940 DAG.getDataLayout(), *DAG.getContext(),
Node->getValueType(0));
943 Node->getOperand(1),
Node->getOperand(4));
946 Node->getOperand(3)));
983 case ISD::VP_BITREVERSE:
984 if (
SDValue Expanded = TLI.expandVPBITREVERSE(
Node, DAG)) {
990 if (
SDValue Expanded = TLI.expandCTPOP(
Node, DAG)) {
996 if (
SDValue Expanded = TLI.expandVPCTPOP(
Node, DAG)) {
1003 if (
SDValue Expanded = TLI.expandCTLZ(
Node, DAG)) {
1009 case ISD::VP_CTLZ_ZERO_UNDEF:
1010 if (
SDValue Expanded = TLI.expandVPCTLZ(
Node, DAG)) {
1017 if (
SDValue Expanded = TLI.expandCTTZ(
Node, DAG)) {
1023 case ISD::VP_CTTZ_ZERO_UNDEF:
1024 if (
SDValue Expanded = TLI.expandVPCTTZ(
Node, DAG)) {
1033 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG)) {
1040 if (
SDValue Expanded = TLI.expandROT(
Node,
false , DAG)) {
1047 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG)) {
1056 if (
SDValue Expanded = TLI.expandIntMINMAX(
Node, DAG)) {
1077 if (
SDValue Expanded = TLI.expandAddSubSat(
Node, DAG)) {
1084 if (
SDValue Expanded = TLI.expandShlSat(
Node, DAG)) {
1092 if (
Node->getValueType(0).isScalableVector()) {
1093 if (
SDValue Expanded = TLI.expandFP_TO_INT_SAT(
Node, DAG)) {
1101 if (
SDValue Expanded = TLI.expandFixedPointMul(
Node, DAG)) {
1120#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1121 case ISD::STRICT_##DAGN:
1122#include "llvm/IR/ConstrainedOps.def"
1140 Results.push_back(TLI.expandVecReduce(
Node, DAG));
1144 Results.push_back(TLI.expandVecReduceSeq(
Node, DAG));
1154 if (tryExpandVecMathCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
1155 RTLIB::REM_F80, RTLIB::REM_F128,
1163 if (
Node->getNumValues() == 1) {
1167 "VectorLegalizer Expand returned wrong number of results!");
1177 EVT VT =
Node->getValueType(0);
1195 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1196 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1197 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand ||
1200 VT) == TargetLowering::Expand)
1201 return DAG.UnrollVectorOp(
Node);
1209 Mask = DAG.getSelect(
DL, BitTy, Mask, DAG.getAllOnesConstant(
DL, BitTy),
1210 DAG.getConstant(0,
DL, BitTy));
1213 Mask = DAG.getSplat(MaskTy,
DL, Mask);
1221 SDValue NotMask = DAG.getNOT(
DL, Mask, MaskTy);
1230 EVT VT =
Node->getValueType(0);
1233 if (TLI.getOperationAction(
ISD::SRA, VT) == TargetLowering::Expand ||
1234 TLI.getOperationAction(
ISD::SHL, VT) == TargetLowering::Expand)
1235 return DAG.UnrollVectorOp(
Node);
1238 EVT OrigTy = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1242 SDValue ShiftSz = DAG.getConstant(BW - OrigBW,
DL, VT);
1252 EVT VT =
Node->getValueType(0);
1255 EVT SrcVT = Src.getValueType();
1262 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1267 Src, DAG.getVectorIdxConstant(0,
DL));
1272 ShuffleMask.
resize(NumSrcElements, -1);
1275 int ExtLaneScale = NumSrcElements / NumElements;
1276 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1277 for (
int i = 0; i < NumElements; ++i)
1278 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1282 DAG.getVectorShuffle(SrcVT,
DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1287 EVT VT =
Node->getValueType(0);
1289 EVT SrcVT = Src.getValueType();
1300 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth,
DL, VT);
1311 EVT VT =
Node->getValueType(0);
1314 EVT SrcVT = Src.getValueType();
1321 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1326 Src, DAG.getVectorIdxConstant(0,
DL));
1334 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1336 int ExtLaneScale = NumSrcElements / NumElements;
1337 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1338 for (
int i = 0; i < NumElements; ++i)
1339 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1342 DAG.getVectorShuffle(SrcVT,
DL, Zero, Src, ShuffleMask));
1348 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
1349 ShuffleMask.
push_back((
I * ScalarSizeInBytes) + J);
1353 EVT VT =
Node->getValueType(0);
1357 return TLI.expandBSWAP(
Node, DAG);
1365 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1368 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1374 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1375 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1376 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1377 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT))
1378 return TLI.expandBSWAP(
Node, DAG);
1381 return DAG.UnrollVectorOp(
Node);
1384void VectorLegalizer::ExpandBITREVERSE(
SDNode *
Node,
1386 EVT VT =
Node->getValueType(0);
1390 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1405 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1410 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1412 (TLI.isOperationLegalOrCustom(
ISD::SHL, ByteVT) &&
1413 TLI.isOperationLegalOrCustom(
ISD::SRL, ByteVT) &&
1414 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, ByteVT) &&
1415 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, ByteVT)))) {
1418 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT),
1429 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1430 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1431 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1432 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT)) {
1433 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1457 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1458 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1459 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand)
1460 return DAG.UnrollVectorOp(
Node);
1466 auto BoolContents = TLI.getBooleanContents(Op1.
getValueType());
1467 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1468 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1470 return DAG.UnrollVectorOp(
Node);
1476 return DAG.UnrollVectorOp(
Node);
1484 SDValue NotMask = DAG.getNOT(
DL, Mask, VT);
1506 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1507 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1508 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1509 return DAG.UnrollVectorOp(
Node);
1513 return DAG.UnrollVectorOp(
Node);
1515 SDValue Ones = DAG.getAllOnesConstant(
DL, VT);
1518 Op1 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op1, Mask, Ones, EVL);
1519 Op2 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op2, NotMask, Ones, EVL);
1520 return DAG.getNode(ISD::VP_OR,
DL, VT, Op1, Op2, Ones, EVL);
1535 EVT MaskVT =
Mask.getValueType();
1547 return DAG.UnrollVectorOp(
Node);
1551 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1552 EVLVecVT) != MaskVT)
1553 return DAG.UnrollVectorOp(
Node);
1555 SDValue StepVec = DAG.getStepVector(
DL, EVLVecVT);
1556 SDValue SplatEVL = DAG.getSplat(EVLVecVT,
DL, EVL);
1558 DAG.getSetCC(
DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1561 return DAG.getSelect(
DL,
Node->getValueType(0), FullMask, Op1, Op2);
1566 EVT VT =
Node->getValueType(0);
1568 unsigned DivOpc =
Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1570 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1571 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1572 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1584 SDValue Mul = DAG.getNode(ISD::VP_MUL,
DL, VT, Divisor, Div, Mask, EVL);
1585 return DAG.getNode(ISD::VP_SUB,
DL, VT, Dividend,
Mul, Mask, EVL);
1588void VectorLegalizer::ExpandFP_TO_UINT(
SDNode *
Node,
1592 if (TLI.expandFP_TO_UINT(
Node, Result, Chain, DAG)) {
1594 if (
Node->isStrictFPOpcode())
1600 if (
Node->isStrictFPOpcode()) {
1608void VectorLegalizer::ExpandUINT_TO_FLOAT(
SDNode *
Node,
1610 bool IsStrict =
Node->isStrictFPOpcode();
1611 unsigned OpNo = IsStrict ? 1 : 0;
1613 EVT VT = Src.getValueType();
1619 if (TLI.expandUINT_TO_FP(
Node, Result, Chain, DAG)) {
1628 TargetLowering::Expand) ||
1630 TargetLowering::Expand)) ||
1631 TLI.getOperationAction(
ISD::SRL, VT) == TargetLowering::Expand) {
1642 assert((BW == 64 || BW == 32) &&
1643 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1645 SDValue HalfWord = DAG.getConstant(BW / 2,
DL, VT);
1650 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1651 SDValue HalfWordMask = DAG.getConstant(HWMask,
DL, VT);
1655 DAG.getConstantFP(1ULL << (BW / 2),
DL,
Node->getValueType(0));
1666 {
Node->getValueType(0), MVT::Other},
1667 {
Node->getOperand(0),
HI});
1671 {
Node->getValueType(0), MVT::Other},
1672 {
Node->getOperand(0),
LO});
1700 if (TLI.isOperationLegalOrCustom(
ISD::FSUB,
Node->getValueType(0))) {
1705 Node->getOperand(0));
1707 return DAG.UnrollVectorOp(
Node);
1710void VectorLegalizer::ExpandFSUB(
SDNode *
Node,
1715 EVT VT =
Node->getValueType(0);
1716 if (TLI.isOperationLegalOrCustom(
ISD::FNEG, VT) &&
1717 TLI.isOperationLegalOrCustom(
ISD::FADD, VT))
1724void VectorLegalizer::ExpandSETCC(
SDNode *
Node,
1726 bool NeedInvert =
false;
1727 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
1731 unsigned Offset = IsStrict ? 1 : 0;
1738 MVT OpVT =
LHS.getSimpleValueType();
1741 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1758 TLI.LegalizeSetCCCondCode(DAG,
Node->getValueType(0), LHS, RHS,
CC, Mask,
1759 EVL, NeedInvert, dl, Chain, IsSignaling);
1766 LHS = DAG.getNode(
Node->getOpcode(), dl,
Node->getVTList(),
1767 {Chain, LHS, RHS, CC},
Node->getFlags());
1768 Chain =
LHS.getValue(1);
1770 LHS = DAG.getNode(ISD::VP_SETCC, dl,
Node->getValueType(0),
1771 {LHS, RHS, CC, Mask, EVL},
Node->getFlags());
1782 LHS = DAG.getLogicalNOT(dl, LHS,
LHS->getValueType(0));
1784 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL,
LHS->getValueType(0));
1787 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
1791 EVT VT =
Node->getValueType(0);
1794 DAG.getBoolConstant(
true, dl, VT,
LHS.getValueType()),
1795 DAG.getBoolConstant(
false, dl, VT,
LHS.getValueType()),
CC);
1796 LHS->setFlags(
Node->getFlags());
1804void VectorLegalizer::ExpandUADDSUBO(
SDNode *
Node,
1807 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
1812void VectorLegalizer::ExpandSADDSUBO(
SDNode *
Node,
1815 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
1820void VectorLegalizer::ExpandMULO(
SDNode *
Node,
1823 if (!TLI.expandMULO(
Node, Result, Overflow, DAG))
1824 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(
Node);
1830void VectorLegalizer::ExpandFixedPointDiv(
SDNode *
Node,
1833 if (
SDValue Expanded = TLI.expandFixedPointDiv(
N->getOpcode(),
SDLoc(
N),
1834 N->getOperand(0),
N->getOperand(1),
N->getConstantOperandVal(2), DAG))
1838void VectorLegalizer::ExpandStrictFPOp(
SDNode *
Node,
1861 "Expected REM node");
1864 if (!TLI.expandREM(
Node, Result, DAG))
1879 assert(!
Node->isStrictFPOpcode() &&
"Unexpected strict fp operation!");
1881 const char *LCName = TLI.getLibcallName(LC);
1884 LLVM_DEBUG(
dbgs() <<
"Looking for vector variant of " << LCName <<
"\n");
1886 EVT VT =
Node->getValueType(0);
1904 for (
unsigned i = 0; i <
Node->getNumOperands(); ++i) {
1905 assert(
Node->getOperand(i).getValueType() == VT &&
1906 "Expected matching vector types!");
1909 FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys,
false);
1921 if (OptVFInfo->Shape.Parameters.size() !=
1930 Entry.IsSExt =
false;
1931 Entry.IsZExt =
false;
1934 for (
auto &VFParam : OptVFInfo->Shape.Parameters) {
1935 if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
1936 EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);
1937 Entry.Node = DAG.getBoolConstant(
true,
DL, MaskVT, VT);
1939 Args.push_back(Entry);
1944 if (VFParam.ParamKind != VFParamKind::Vector)
1947 Entry.Node =
Node->getOperand(OpNum++);
1949 Args.push_back(Entry);
1954 TLI.getPointerTy(DAG.getDataLayout()));
1957 .setChain(DAG.getEntryNode())
1960 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
1961 Results.push_back(CallResult.first);
1966bool VectorLegalizer::tryExpandVecMathCall(
1971 Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64,
1972 Call_F80, Call_F128, Call_PPCF128);
1974 if (LC == RTLIB::UNKNOWN_LIBCALL)
1980void VectorLegalizer::UnrollStrictFPOp(
SDNode *
Node,
1982 EVT VT =
Node->getValueType(0);
1985 unsigned NumOpers =
Node->getNumOperands();
1988 EVT TmpEltVT = EltVT;
1992 *DAG.getContext(), TmpEltVT);
1994 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
2000 for (
unsigned i = 0; i < NumElems; ++i) {
2002 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
2008 for (
unsigned j = 1;
j < NumOpers; ++
j) {
2025 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
2026 DAG.getAllOnesConstant(dl, EltVT),
2027 DAG.getConstant(0, dl, EltVT));
2041 EVT VT =
Node->getValueType(0);
2047 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
2050 for (
unsigned i = 0; i < NumElems; ++i) {
2052 DAG.getVectorIdxConstant(i, dl));
2054 DAG.getVectorIdxConstant(i, dl));
2057 *DAG.getContext(), TmpEltVT),
2058 LHSElem, RHSElem,
CC);
2059 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
2060 DAG.getConstant(0, dl, EltVT));
2062 return DAG.getBuildVector(VT, dl, Ops);
2066 return VectorLegalizer(*this).Run();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
ilist< SDNode >::iterator allnodes_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Provides info so a possible vectorization of a function can be computed.
std::string getVectorFunctionABIVariantString() const
Returns a vector function ABI variant string on the form: ZGV<isa><mask><vlen><vparams><scalarname>(<...
StringRef getVectorFnName() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.