51#define DEBUG_TYPE "legalizevectorops"
55class VectorLegalizer {
67 LegalizedNodes.
insert(std::make_pair(
From, To));
70 LegalizedNodes.
insert(std::make_pair(To, To));
139 std::pair<SDValue, SDValue> ExpandLoad(
SDNode *
N);
204bool VectorLegalizer::Run() {
206 bool HasVectors =
false;
208 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I) {
228 DAG.AssignTopologicalOrder();
230 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I)
234 SDValue OldRoot = DAG.getRoot();
235 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
236 DAG.setRoot(LegalizedNodes[OldRoot]);
238 LegalizedNodes.clear();
241 DAG.RemoveDeadNodes();
248 "Unexpected number of results");
250 for (
unsigned i = 0, e =
Op->getNumValues(); i != e; ++i)
251 AddLegalizedOperand(
Op.getValue(i),
SDValue(Result, i));
256VectorLegalizer::RecursivelyLegalizeResults(
SDValue Op,
259 "Unexpected number of results");
261 for (
unsigned i = 0, e =
Results.
size(); i != e; ++i) {
263 AddLegalizedOperand(
Op.getValue(i),
Results[i]);
273 if (
I != LegalizedNodes.end())
return I->second;
277 for (
const SDValue &Oper :
Op->op_values())
280 SDNode *
Node = DAG.UpdateNodeOperands(
Op.getNode(), Ops);
282 bool HasVectorValueOrOp =
285 [](
SDValue O) { return O.getValueType().isVector(); });
286 if (!HasVectorValueOrOp)
287 return TranslateLegalizeResults(
Op,
Node);
291 switch (
Op.getOpcode()) {
293 return TranslateLegalizeResults(
Op,
Node);
297 EVT LoadedVT =
LD->getMemoryVT();
299 Action = TLI.getLoadExtAction(ExtType,
LD->getValueType(0), LoadedVT);
304 EVT StVT =
ST->getMemoryVT();
305 MVT ValVT =
ST->getValue().getSimpleValueType();
306 if (StVT.
isVector() &&
ST->isTruncatingStore())
307 Action = TLI.getTruncStoreAction(ValVT, StVT);
311 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
314 if (Action == TargetLowering::Legal)
315 Action = TargetLowering::Expand;
317#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
318 case ISD::STRICT_##DAGN:
319#include "llvm/IR/ConstrainedOps.def"
320 ValVT =
Node->getValueType(0);
323 ValVT =
Node->getOperand(1).getValueType();
326 MVT OpVT =
Node->getOperand(1).getSimpleValueType();
328 Action = TLI.getCondCodeAction(CCCode, OpVT);
329 if (Action == TargetLowering::Legal)
330 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
332 Action = TLI.getOperationAction(
Node->getOpcode(), ValVT);
339 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
340 TLI.getStrictFPOperationAction(
Node->getOpcode(), ValVT) ==
341 TargetLowering::Legal) {
343 if (TLI.getOperationAction(
Node->getOpcode(), EltVT)
344 == TargetLowering::Expand &&
345 TLI.getStrictFPOperationAction(
Node->getOpcode(), EltVT)
346 == TargetLowering::Legal)
347 Action = TargetLowering::Legal;
451 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
461 unsigned Scale =
Node->getConstantOperandVal(2);
462 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
463 Node->getValueType(0), Scale);
485 Action = TLI.getOperationAction(
Node->getOpcode(),
486 Node->getOperand(0).getValueType());
490 Action = TLI.getOperationAction(
Node->getOpcode(),
491 Node->getOperand(1).getValueType());
494 MVT OpVT =
Node->getOperand(0).getSimpleValueType();
496 Action = TLI.getCondCodeAction(CCCode, OpVT);
497 if (Action == TargetLowering::Legal)
498 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
502#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
504 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
505 : Node->getOperand(LEGALPOS).getValueType(); \
506 if (ISD::VPID == ISD::VP_SETCC) { \
507 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
508 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
509 if (Action != TargetLowering::Legal) \
512 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
514#include "llvm/IR/VPIntrinsics.def"
522 case TargetLowering::Promote:
524 "This action is not supported yet!");
526 Promote(
Node, ResultVals);
527 assert(!ResultVals.
empty() &&
"No results for promotion?");
529 case TargetLowering::Legal:
532 case TargetLowering::Custom:
534 if (LowerOperationWrapper(
Node, ResultVals))
538 case TargetLowering::Expand:
540 Expand(
Node, ResultVals);
544 if (ResultVals.
empty())
545 return TranslateLegalizeResults(
Op,
Node);
548 return RecursivelyLegalizeResults(
Op, ResultVals);
553bool VectorLegalizer::LowerOperationWrapper(
SDNode *
Node,
565 if (
Node->getNumValues() == 1) {
573 "Lowering returned the wrong number of results!");
576 for (
unsigned I = 0, E =
Node->getNumValues();
I != E; ++
I)
582void VectorLegalizer::PromoteReduction(
SDNode *
Node,
584 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
585 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
586 MVT ScalarVT =
Node->getSimpleValueType(0);
593 if (
Node->getOperand(0).getValueType().isFloatingPoint())
600 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
601 if (
Node->getOperand(j).getValueType().isVector() &&
605 if (
Node->getOperand(j).getValueType().isFloatingPoint())
619 DAG.getIntPtrConstant(0,
DL,
true));
626void VectorLegalizer::PromoteSETCC(
SDNode *
Node,
628 MVT VecVT =
Node->getOperand(0).getSimpleValueType();
629 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
636 Operands[0] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(0));
637 Operands[1] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(1));
640 if (
Node->getOpcode() == ISD::VP_SETCC) {
651void VectorLegalizer::PromoteSTRICT(
SDNode *
Node,
653 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
654 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
662 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
663 if (
Node->getOperand(j).getValueType().isVector() &&
670 {
Node->getOperand(0),
Node->getOperand(j)});
676 SDVTList VTs = DAG.getVTList(NewVecVT,
Node->getValueType(1));
686 DAG.getIntPtrConstant(0,
DL,
true)});
695 switch (
Node->getOpcode()) {
710 case ISD::VP_REDUCE_ADD:
711 case ISD::VP_REDUCE_MUL:
712 case ISD::VP_REDUCE_AND:
713 case ISD::VP_REDUCE_OR:
714 case ISD::VP_REDUCE_XOR:
715 case ISD::VP_REDUCE_SMAX:
716 case ISD::VP_REDUCE_SMIN:
717 case ISD::VP_REDUCE_UMAX:
718 case ISD::VP_REDUCE_UMIN:
719 case ISD::VP_REDUCE_FADD:
720 case ISD::VP_REDUCE_FMUL:
721 case ISD::VP_REDUCE_FMAX:
722 case ISD::VP_REDUCE_FMIN:
723 case ISD::VP_REDUCE_SEQ_FADD:
753 "Can't promote a vector with multiple results!");
754 MVT VT =
Node->getSimpleValueType(0);
755 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
759 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
763 if (
Node->getOperand(j).getValueType().isVector() && !SkipPromote)
764 if (
Node->getOperand(j)
766 .getVectorElementType()
767 .isFloatingPoint() &&
783 DAG.getIntPtrConstant(0, dl,
true));
790void VectorLegalizer::PromoteINT_TO_FP(
SDNode *
Node,
794 bool IsStrict =
Node->isStrictFPOpcode();
795 MVT VT =
Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
796 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
798 "Vectors have different number of elements!");
807 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
808 if (
Node->getOperand(j).getValueType().isVector())
809 Operands[j] = DAG.getNode(Opc, dl, NVT,
Node->getOperand(j));
831void VectorLegalizer::PromoteFP_TO_INT(
SDNode *
Node,
833 MVT VT =
Node->getSimpleValueType(0);
834 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
835 bool IsStrict =
Node->isStrictFPOpcode();
837 "Vectors have different number of elements!");
839 unsigned NewOpc =
Node->getOpcode();
853 Promoted = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
854 {
Node->getOperand(0),
Node->getOperand(1)});
857 Promoted = DAG.
getNode(NewOpc, dl, NVT,
Node->getOperand(0));
868 Promoted = DAG.
getNode(NewOpc, dl, NVT, Promoted,
876std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(
SDNode *
N) {
878 return TLI.scalarizeVectorLoad(LD, DAG);
883 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
888 switch (
Node->getOpcode()) {
890 std::pair<SDValue, SDValue> Tmp = ExpandLoad(
Node);
899 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
906 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(
Node));
909 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(
Node));
912 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(
Node));
937 if (
Node->getValueType(0).isScalableVector()) {
938 EVT CondVT = TLI.getSetCCResultType(
939 DAG.getDataLayout(), *DAG.getContext(),
Node->getValueType(0));
942 Node->getOperand(1),
Node->getOperand(4));
945 Node->getOperand(3)));
982 case ISD::VP_BITREVERSE:
983 if (
SDValue Expanded = TLI.expandVPBITREVERSE(
Node, DAG)) {
989 if (
SDValue Expanded = TLI.expandCTPOP(
Node, DAG)) {
995 if (
SDValue Expanded = TLI.expandVPCTPOP(
Node, DAG)) {
1002 if (
SDValue Expanded = TLI.expandCTLZ(
Node, DAG)) {
1008 case ISD::VP_CTLZ_ZERO_UNDEF:
1009 if (
SDValue Expanded = TLI.expandVPCTLZ(
Node, DAG)) {
1016 if (
SDValue Expanded = TLI.expandCTTZ(
Node, DAG)) {
1022 case ISD::VP_CTTZ_ZERO_UNDEF:
1023 if (
SDValue Expanded = TLI.expandVPCTTZ(
Node, DAG)) {
1032 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG)) {
1039 if (
SDValue Expanded = TLI.expandROT(
Node,
false , DAG)) {
1046 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG)) {
1055 if (
SDValue Expanded = TLI.expandIntMINMAX(
Node, DAG)) {
1076 if (
SDValue Expanded = TLI.expandAddSubSat(
Node, DAG)) {
1083 if (
SDValue Expanded = TLI.expandShlSat(
Node, DAG)) {
1091 if (
Node->getValueType(0).isScalableVector()) {
1092 if (
SDValue Expanded = TLI.expandFP_TO_INT_SAT(
Node, DAG)) {
1100 if (
SDValue Expanded = TLI.expandFixedPointMul(
Node, DAG)) {
1119#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1120 case ISD::STRICT_##DAGN:
1121#include "llvm/IR/ConstrainedOps.def"
1139 Results.push_back(TLI.expandVecReduce(
Node, DAG));
1143 Results.push_back(TLI.expandVecReduceSeq(
Node, DAG));
1153 if (tryExpandVecMathCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
1154 RTLIB::REM_F80, RTLIB::REM_F128,
1162 if (
Node->getNumValues() == 1) {
1166 "VectorLegalizer Expand returned wrong number of results!");
1176 EVT VT =
Node->getValueType(0);
1194 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1195 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1196 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand ||
1199 VT) == TargetLowering::Expand)
1200 return DAG.UnrollVectorOp(
Node);
1208 Mask = DAG.getSelect(
DL, BitTy, Mask, DAG.getAllOnesConstant(
DL, BitTy),
1209 DAG.getConstant(0,
DL, BitTy));
1212 Mask = DAG.getSplat(MaskTy,
DL, Mask);
1220 SDValue NotMask = DAG.getNOT(
DL, Mask, MaskTy);
1229 EVT VT =
Node->getValueType(0);
1232 if (TLI.getOperationAction(
ISD::SRA, VT) == TargetLowering::Expand ||
1233 TLI.getOperationAction(
ISD::SHL, VT) == TargetLowering::Expand)
1234 return DAG.UnrollVectorOp(
Node);
1237 EVT OrigTy = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1241 SDValue ShiftSz = DAG.getConstant(BW - OrigBW,
DL, VT);
1251 EVT VT =
Node->getValueType(0);
1254 EVT SrcVT = Src.getValueType();
1261 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1266 Src, DAG.getVectorIdxConstant(0,
DL));
1271 ShuffleMask.
resize(NumSrcElements, -1);
1274 int ExtLaneScale = NumSrcElements / NumElements;
1275 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1276 for (
int i = 0; i < NumElements; ++i)
1277 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1281 DAG.getVectorShuffle(SrcVT,
DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1286 EVT VT =
Node->getValueType(0);
1288 EVT SrcVT = Src.getValueType();
1299 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth,
DL, VT);
1310 EVT VT =
Node->getValueType(0);
1313 EVT SrcVT = Src.getValueType();
1320 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1325 Src, DAG.getVectorIdxConstant(0,
DL));
1333 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1335 int ExtLaneScale = NumSrcElements / NumElements;
1336 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1337 for (
int i = 0; i < NumElements; ++i)
1338 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1341 DAG.getVectorShuffle(SrcVT,
DL, Zero, Src, ShuffleMask));
1347 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
1348 ShuffleMask.
push_back((
I * ScalarSizeInBytes) + J);
1352 EVT VT =
Node->getValueType(0);
1356 return TLI.expandBSWAP(
Node, DAG);
1364 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1367 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1373 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1374 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1375 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1376 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT))
1377 return TLI.expandBSWAP(
Node, DAG);
1380 return DAG.UnrollVectorOp(
Node);
1383void VectorLegalizer::ExpandBITREVERSE(
SDNode *
Node,
1385 EVT VT =
Node->getValueType(0);
1389 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1404 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1409 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1411 (TLI.isOperationLegalOrCustom(
ISD::SHL, ByteVT) &&
1412 TLI.isOperationLegalOrCustom(
ISD::SRL, ByteVT) &&
1413 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, ByteVT) &&
1414 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, ByteVT)))) {
1417 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT),
1428 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1429 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1430 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1431 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT)) {
1432 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1456 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1457 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1458 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand)
1459 return DAG.UnrollVectorOp(
Node);
1465 auto BoolContents = TLI.getBooleanContents(Op1.
getValueType());
1466 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1467 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1469 return DAG.UnrollVectorOp(
Node);
1475 return DAG.UnrollVectorOp(
Node);
1483 SDValue NotMask = DAG.getNOT(
DL, Mask, VT);
1505 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1506 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1507 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1508 return DAG.UnrollVectorOp(
Node);
1512 return DAG.UnrollVectorOp(
Node);
1514 SDValue Ones = DAG.getAllOnesConstant(
DL, VT);
1517 Op1 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op1, Mask, Ones, EVL);
1518 Op2 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op2, NotMask, Ones, EVL);
1519 return DAG.getNode(ISD::VP_OR,
DL, VT, Op1, Op2, Ones, EVL);
1534 EVT MaskVT =
Mask.getValueType();
1546 return DAG.UnrollVectorOp(
Node);
1550 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1551 EVLVecVT) != MaskVT)
1552 return DAG.UnrollVectorOp(
Node);
1554 SDValue StepVec = DAG.getStepVector(
DL, EVLVecVT);
1555 SDValue SplatEVL = DAG.getSplat(EVLVecVT,
DL, EVL);
1557 DAG.getSetCC(
DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1560 return DAG.getSelect(
DL,
Node->getValueType(0), FullMask, Op1, Op2);
1565 EVT VT =
Node->getValueType(0);
1567 unsigned DivOpc =
Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1569 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1570 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1571 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1583 SDValue Mul = DAG.getNode(ISD::VP_MUL,
DL, VT, Divisor, Div, Mask, EVL);
1584 return DAG.getNode(ISD::VP_SUB,
DL, VT, Dividend,
Mul, Mask, EVL);
1587void VectorLegalizer::ExpandFP_TO_UINT(
SDNode *
Node,
1591 if (TLI.expandFP_TO_UINT(
Node, Result, Chain, DAG)) {
1593 if (
Node->isStrictFPOpcode())
1599 if (
Node->isStrictFPOpcode()) {
1607void VectorLegalizer::ExpandUINT_TO_FLOAT(
SDNode *
Node,
1609 bool IsStrict =
Node->isStrictFPOpcode();
1610 unsigned OpNo = IsStrict ? 1 : 0;
1612 EVT VT = Src.getValueType();
1618 if (TLI.expandUINT_TO_FP(
Node, Result, Chain, DAG)) {
1627 TargetLowering::Expand) ||
1629 TargetLowering::Expand)) ||
1630 TLI.getOperationAction(
ISD::SRL, VT) == TargetLowering::Expand) {
1641 assert((BW == 64 || BW == 32) &&
1642 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1644 SDValue HalfWord = DAG.getConstant(BW / 2,
DL, VT);
1649 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1650 SDValue HalfWordMask = DAG.getConstant(HWMask,
DL, VT);
1654 DAG.getConstantFP(1ULL << (BW / 2),
DL,
Node->getValueType(0));
1665 {
Node->getValueType(0), MVT::Other},
1666 {
Node->getOperand(0),
HI});
1670 {
Node->getValueType(0), MVT::Other},
1671 {
Node->getOperand(0),
LO});
1699 if (TLI.isOperationLegalOrCustom(
ISD::FSUB,
Node->getValueType(0))) {
1704 Node->getOperand(0));
1706 return DAG.UnrollVectorOp(
Node);
1709void VectorLegalizer::ExpandFSUB(
SDNode *
Node,
1714 EVT VT =
Node->getValueType(0);
1715 if (TLI.isOperationLegalOrCustom(
ISD::FNEG, VT) &&
1716 TLI.isOperationLegalOrCustom(
ISD::FADD, VT))
1723void VectorLegalizer::ExpandSETCC(
SDNode *
Node,
1725 bool NeedInvert =
false;
1726 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
1730 unsigned Offset = IsStrict ? 1 : 0;
1737 MVT OpVT =
LHS.getSimpleValueType();
1740 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1757 TLI.LegalizeSetCCCondCode(DAG,
Node->getValueType(0), LHS, RHS,
CC, Mask,
1758 EVL, NeedInvert, dl, Chain, IsSignaling);
1765 LHS = DAG.getNode(
Node->getOpcode(), dl,
Node->getVTList(),
1766 {Chain, LHS, RHS, CC},
Node->getFlags());
1767 Chain =
LHS.getValue(1);
1769 LHS = DAG.getNode(ISD::VP_SETCC, dl,
Node->getValueType(0),
1770 {LHS, RHS, CC, Mask, EVL},
Node->getFlags());
1781 LHS = DAG.getLogicalNOT(dl, LHS,
LHS->getValueType(0));
1783 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL,
LHS->getValueType(0));
1786 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
1790 EVT VT =
Node->getValueType(0);
1793 DAG.getBoolConstant(
true, dl, VT,
LHS.getValueType()),
1794 DAG.getBoolConstant(
false, dl, VT,
LHS.getValueType()),
CC);
1795 LHS->setFlags(
Node->getFlags());
1803void VectorLegalizer::ExpandUADDSUBO(
SDNode *
Node,
1806 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
1811void VectorLegalizer::ExpandSADDSUBO(
SDNode *
Node,
1814 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
1819void VectorLegalizer::ExpandMULO(
SDNode *
Node,
1822 if (!TLI.expandMULO(
Node, Result, Overflow, DAG))
1823 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(
Node);
1829void VectorLegalizer::ExpandFixedPointDiv(
SDNode *
Node,
1832 if (
SDValue Expanded = TLI.expandFixedPointDiv(
N->getOpcode(),
SDLoc(
N),
1833 N->getOperand(0),
N->getOperand(1),
N->getConstantOperandVal(2), DAG))
1837void VectorLegalizer::ExpandStrictFPOp(
SDNode *
Node,
1860 "Expected REM node");
1863 if (!TLI.expandREM(
Node, Result, DAG))
1878 assert(!
Node->isStrictFPOpcode() &&
"Unexpected strict fp operation!");
1880 const char *LCName = TLI.getLibcallName(LC);
1883 LLVM_DEBUG(
dbgs() <<
"Looking for vector variant of " << LCName <<
"\n");
1885 EVT VT =
Node->getValueType(0);
1903 for (
unsigned i = 0; i <
Node->getNumOperands(); ++i) {
1904 assert(
Node->getOperand(i).getValueType() == VT &&
1905 "Expected matching vector types!");
1908 FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys,
false);
1920 if (OptVFInfo->Shape.Parameters.size() !=
1929 Entry.IsSExt =
false;
1930 Entry.IsZExt =
false;
1933 for (
auto &VFParam : OptVFInfo->Shape.Parameters) {
1934 if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
1935 EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);
1936 Entry.Node = DAG.getBoolConstant(
true,
DL, MaskVT, VT);
1938 Args.push_back(Entry);
1943 if (VFParam.ParamKind != VFParamKind::Vector)
1946 Entry.Node =
Node->getOperand(OpNum++);
1948 Args.push_back(Entry);
1953 TLI.getPointerTy(DAG.getDataLayout()));
1956 .setChain(DAG.getEntryNode())
1959 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
1960 Results.push_back(CallResult.first);
1965bool VectorLegalizer::tryExpandVecMathCall(
1970 Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64,
1971 Call_F80, Call_F128, Call_PPCF128);
1973 if (LC == RTLIB::UNKNOWN_LIBCALL)
1979void VectorLegalizer::UnrollStrictFPOp(
SDNode *
Node,
1981 EVT VT =
Node->getValueType(0);
1984 unsigned NumOpers =
Node->getNumOperands();
1987 EVT TmpEltVT = EltVT;
1991 *DAG.getContext(), TmpEltVT);
1993 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1999 for (
unsigned i = 0; i < NumElems; ++i) {
2001 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
2007 for (
unsigned j = 1;
j < NumOpers; ++
j) {
2024 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
2025 DAG.getAllOnesConstant(dl, EltVT),
2026 DAG.getConstant(0, dl, EltVT));
2040 EVT VT =
Node->getValueType(0);
2046 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
2049 for (
unsigned i = 0; i < NumElems; ++i) {
2051 DAG.getVectorIdxConstant(i, dl));
2053 DAG.getVectorIdxConstant(i, dl));
2056 *DAG.getContext(), TmpEltVT),
2057 LHSElem, RHSElem,
CC);
2058 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
2059 DAG.getConstant(0, dl, EltVT));
2061 return DAG.getBuildVector(VT, dl, Ops);
2065 return VectorLegalizer(*this).Run();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
ilist< SDNode >::iterator allnodes_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Provides info so a possible vectorization of a function can be computed.
std::string getVectorFunctionABIVariantString() const
Returns a vector function ABI variant string on the form: ZGV<isa><mask><vlen><vparams><scalarname>(<...
StringRef getVectorFnName() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.