51#define DEBUG_TYPE "legalizevectorops"
55class VectorLegalizer {
67 LegalizedNodes.
insert(std::make_pair(
From, To));
70 LegalizedNodes.
insert(std::make_pair(To, To));
139 std::pair<SDValue, SDValue> ExpandLoad(
SDNode *
N);
197bool VectorLegalizer::Run() {
199 bool HasVectors =
false;
201 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I) {
221 DAG.AssignTopologicalOrder();
223 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I)
227 SDValue OldRoot = DAG.getRoot();
228 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
229 DAG.setRoot(LegalizedNodes[OldRoot]);
231 LegalizedNodes.clear();
234 DAG.RemoveDeadNodes();
241 "Unexpected number of results");
243 for (
unsigned i = 0, e =
Op->getNumValues(); i != e; ++i)
244 AddLegalizedOperand(
Op.getValue(i),
SDValue(Result, i));
249VectorLegalizer::RecursivelyLegalizeResults(
SDValue Op,
252 "Unexpected number of results");
254 for (
unsigned i = 0, e =
Results.
size(); i != e; ++i) {
256 AddLegalizedOperand(
Op.getValue(i),
Results[i]);
266 if (
I != LegalizedNodes.end())
return I->second;
270 for (
const SDValue &Oper :
Op->op_values())
273 SDNode *
Node = DAG.UpdateNodeOperands(
Op.getNode(), Ops);
275 bool HasVectorValueOrOp =
278 [](
SDValue O) { return O.getValueType().isVector(); });
279 if (!HasVectorValueOrOp)
280 return TranslateLegalizeResults(
Op,
Node);
284 switch (
Op.getOpcode()) {
286 return TranslateLegalizeResults(
Op,
Node);
290 EVT LoadedVT =
LD->getMemoryVT();
292 Action = TLI.getLoadExtAction(ExtType,
LD->getValueType(0), LoadedVT);
297 EVT StVT =
ST->getMemoryVT();
298 MVT ValVT =
ST->getValue().getSimpleValueType();
299 if (StVT.
isVector() &&
ST->isTruncatingStore())
300 Action = TLI.getTruncStoreAction(ValVT, StVT);
304 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
307 if (Action == TargetLowering::Legal)
308 Action = TargetLowering::Expand;
310#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
311 case ISD::STRICT_##DAGN:
312#include "llvm/IR/ConstrainedOps.def"
313 ValVT =
Node->getValueType(0);
316 ValVT =
Node->getOperand(1).getValueType();
319 MVT OpVT =
Node->getOperand(1).getSimpleValueType();
321 Action = TLI.getCondCodeAction(CCCode, OpVT);
322 if (Action == TargetLowering::Legal)
323 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
325 Action = TLI.getOperationAction(
Node->getOpcode(), ValVT);
332 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
333 TLI.getStrictFPOperationAction(
Node->getOpcode(), ValVT) ==
334 TargetLowering::Legal) {
336 if (TLI.getOperationAction(
Node->getOpcode(), EltVT)
337 == TargetLowering::Expand &&
338 TLI.getStrictFPOperationAction(
Node->getOpcode(), EltVT)
339 == TargetLowering::Legal)
340 Action = TargetLowering::Legal;
448 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
458 unsigned Scale =
Node->getConstantOperandVal(2);
459 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
460 Node->getValueType(0), Scale);
482 Action = TLI.getOperationAction(
Node->getOpcode(),
483 Node->getOperand(0).getValueType());
487 Action = TLI.getOperationAction(
Node->getOpcode(),
488 Node->getOperand(1).getValueType());
491 MVT OpVT =
Node->getOperand(0).getSimpleValueType();
493 Action = TLI.getCondCodeAction(CCCode, OpVT);
494 if (Action == TargetLowering::Legal)
495 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
499#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
501 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
502 : Node->getOperand(LEGALPOS).getValueType(); \
503 if (ISD::VPID == ISD::VP_SETCC) { \
504 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
505 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
506 if (Action != TargetLowering::Legal) \
510 if (!Node->getValueType(0).isVector() && \
511 Node->getValueType(0) != MVT::Other) { \
512 Action = TargetLowering::Legal; \
515 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
517#include "llvm/IR/VPIntrinsics.def"
525 case TargetLowering::Promote:
527 "This action is not supported yet!");
529 Promote(
Node, ResultVals);
530 assert(!ResultVals.
empty() &&
"No results for promotion?");
532 case TargetLowering::Legal:
535 case TargetLowering::Custom:
537 if (LowerOperationWrapper(
Node, ResultVals))
541 case TargetLowering::Expand:
543 Expand(
Node, ResultVals);
547 if (ResultVals.
empty())
548 return TranslateLegalizeResults(
Op,
Node);
551 return RecursivelyLegalizeResults(
Op, ResultVals);
556bool VectorLegalizer::LowerOperationWrapper(
SDNode *
Node,
568 if (
Node->getNumValues() == 1) {
576 "Lowering returned the wrong number of results!");
579 for (
unsigned I = 0, E =
Node->getNumValues();
I != E; ++
I)
585void VectorLegalizer::PromoteSETCC(
SDNode *
Node,
587 MVT VecVT =
Node->getOperand(0).getSimpleValueType();
588 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
595 Operands[0] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(0));
596 Operands[1] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(1));
599 if (
Node->getOpcode() == ISD::VP_SETCC) {
610void VectorLegalizer::PromoteSTRICT(
SDNode *
Node,
612 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
613 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
621 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
622 if (
Node->getOperand(j).getValueType().isVector() &&
629 {
Node->getOperand(0),
Node->getOperand(j)});
635 SDVTList VTs = DAG.getVTList(NewVecVT,
Node->getValueType(1));
645 DAG.getIntPtrConstant(0,
DL,
true)});
654 switch (
Node->getOpcode()) {
695 "Can't promote a vector with multiple results!");
696 MVT VT =
Node->getSimpleValueType(0);
697 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
701 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
705 if (
Node->getOperand(j).getValueType().isVector() && !SkipPromote)
706 if (
Node->getOperand(j)
708 .getVectorElementType()
709 .isFloatingPoint() &&
725 DAG.getIntPtrConstant(0, dl,
true));
732void VectorLegalizer::PromoteINT_TO_FP(
SDNode *
Node,
736 bool IsStrict =
Node->isStrictFPOpcode();
737 MVT VT =
Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
738 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
740 "Vectors have different number of elements!");
749 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
750 if (
Node->getOperand(j).getValueType().isVector())
751 Operands[j] = DAG.getNode(Opc, dl, NVT,
Node->getOperand(j));
773void VectorLegalizer::PromoteFP_TO_INT(
SDNode *
Node,
775 MVT VT =
Node->getSimpleValueType(0);
776 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
777 bool IsStrict =
Node->isStrictFPOpcode();
779 "Vectors have different number of elements!");
781 unsigned NewOpc =
Node->getOpcode();
795 Promoted = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
796 {
Node->getOperand(0),
Node->getOperand(1)});
799 Promoted = DAG.
getNode(NewOpc, dl, NVT,
Node->getOperand(0));
810 Promoted = DAG.
getNode(NewOpc, dl, NVT, Promoted,
818std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(
SDNode *
N) {
820 return TLI.scalarizeVectorLoad(LD, DAG);
825 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
830 switch (
Node->getOpcode()) {
832 std::pair<SDValue, SDValue> Tmp = ExpandLoad(
Node);
841 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
848 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(
Node));
851 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(
Node));
854 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(
Node));
879 if (
Node->getValueType(0).isScalableVector()) {
880 EVT CondVT = TLI.getSetCCResultType(
881 DAG.getDataLayout(), *DAG.getContext(),
Node->getValueType(0));
884 Node->getOperand(1),
Node->getOperand(4));
887 Node->getOperand(3)));
924 case ISD::VP_BITREVERSE:
925 if (
SDValue Expanded = TLI.expandVPBITREVERSE(
Node, DAG)) {
931 if (
SDValue Expanded = TLI.expandCTPOP(
Node, DAG)) {
937 if (
SDValue Expanded = TLI.expandVPCTPOP(
Node, DAG)) {
944 if (
SDValue Expanded = TLI.expandCTLZ(
Node, DAG)) {
950 case ISD::VP_CTLZ_ZERO_UNDEF:
951 if (
SDValue Expanded = TLI.expandVPCTLZ(
Node, DAG)) {
958 if (
SDValue Expanded = TLI.expandCTTZ(
Node, DAG)) {
964 case ISD::VP_CTTZ_ZERO_UNDEF:
965 if (
SDValue Expanded = TLI.expandVPCTTZ(
Node, DAG)) {
974 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG)) {
981 if (
SDValue Expanded = TLI.expandROT(
Node,
false , DAG)) {
988 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG)) {
995 Results.push_back(TLI.expandFMINIMUM_FMAXIMUM(
Node, DAG));
1001 if (
SDValue Expanded = TLI.expandIntMINMAX(
Node, DAG)) {
1022 if (
SDValue Expanded = TLI.expandAddSubSat(
Node, DAG)) {
1029 if (
SDValue Expanded = TLI.expandShlSat(
Node, DAG)) {
1037 if (
Node->getValueType(0).isScalableVector()) {
1038 if (
SDValue Expanded = TLI.expandFP_TO_INT_SAT(
Node, DAG)) {
1046 if (
SDValue Expanded = TLI.expandFixedPointMul(
Node, DAG)) {
1065#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1066 case ISD::STRICT_##DAGN:
1067#include "llvm/IR/ConstrainedOps.def"
1085 Results.push_back(TLI.expandVecReduce(
Node, DAG));
1089 Results.push_back(TLI.expandVecReduceSeq(
Node, DAG));
1099 if (tryExpandVecMathCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
1100 RTLIB::REM_F80, RTLIB::REM_F128,
1108 if (
Node->getNumValues() == 1) {
1112 "VectorLegalizer Expand returned wrong number of results!");
1122 EVT VT =
Node->getValueType(0);
1140 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1141 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1142 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand ||
1145 VT) == TargetLowering::Expand)
1146 return DAG.UnrollVectorOp(
Node);
1154 Mask = DAG.getSelect(
DL, BitTy, Mask, DAG.getAllOnesConstant(
DL, BitTy),
1155 DAG.getConstant(0,
DL, BitTy));
1158 Mask = DAG.getSplat(MaskTy,
DL, Mask);
1166 SDValue NotMask = DAG.getNOT(
DL, Mask, MaskTy);
1175 EVT VT =
Node->getValueType(0);
1178 if (TLI.getOperationAction(
ISD::SRA, VT) == TargetLowering::Expand ||
1179 TLI.getOperationAction(
ISD::SHL, VT) == TargetLowering::Expand)
1180 return DAG.UnrollVectorOp(
Node);
1183 EVT OrigTy = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1187 SDValue ShiftSz = DAG.getConstant(BW - OrigBW,
DL, VT);
1197 EVT VT =
Node->getValueType(0);
1200 EVT SrcVT = Src.getValueType();
1207 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1212 Src, DAG.getVectorIdxConstant(0,
DL));
1217 ShuffleMask.
resize(NumSrcElements, -1);
1220 int ExtLaneScale = NumSrcElements / NumElements;
1221 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1222 for (
int i = 0; i < NumElements; ++i)
1223 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1227 DAG.getVectorShuffle(SrcVT,
DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1232 EVT VT =
Node->getValueType(0);
1234 EVT SrcVT = Src.getValueType();
1245 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth,
DL, VT);
1256 EVT VT =
Node->getValueType(0);
1259 EVT SrcVT = Src.getValueType();
1266 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1271 Src, DAG.getVectorIdxConstant(0,
DL));
1279 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1281 int ExtLaneScale = NumSrcElements / NumElements;
1282 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1283 for (
int i = 0; i < NumElements; ++i)
1284 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1287 DAG.getVectorShuffle(SrcVT,
DL, Zero, Src, ShuffleMask));
1293 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
1294 ShuffleMask.
push_back((
I * ScalarSizeInBytes) + J);
1298 EVT VT =
Node->getValueType(0);
1302 return TLI.expandBSWAP(
Node, DAG);
1310 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1313 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1319 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1320 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1321 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1322 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT))
1323 return TLI.expandBSWAP(
Node, DAG);
1326 return DAG.UnrollVectorOp(
Node);
1329void VectorLegalizer::ExpandBITREVERSE(
SDNode *
Node,
1331 EVT VT =
Node->getValueType(0);
1335 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1350 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1355 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1357 (TLI.isOperationLegalOrCustom(
ISD::SHL, ByteVT) &&
1358 TLI.isOperationLegalOrCustom(
ISD::SRL, ByteVT) &&
1359 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, ByteVT) &&
1360 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, ByteVT)))) {
1363 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT),
1374 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1375 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1376 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1377 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT)) {
1378 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1402 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1403 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1404 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand)
1405 return DAG.UnrollVectorOp(
Node);
1411 auto BoolContents = TLI.getBooleanContents(Op1.
getValueType());
1412 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1413 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1415 return DAG.UnrollVectorOp(
Node);
1421 return DAG.UnrollVectorOp(
Node);
1429 SDValue NotMask = DAG.getNOT(
DL, Mask, VT);
1451 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1452 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1453 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1454 return DAG.UnrollVectorOp(
Node);
1458 return DAG.UnrollVectorOp(
Node);
1460 SDValue Ones = DAG.getAllOnesConstant(
DL, VT);
1463 Op1 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op1, Mask, Ones, EVL);
1464 Op2 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op2, NotMask, Ones, EVL);
1465 return DAG.getNode(ISD::VP_OR,
DL, VT, Op1, Op2, Ones, EVL);
1480 EVT MaskVT =
Mask.getValueType();
1492 return DAG.UnrollVectorOp(
Node);
1496 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1497 EVLVecVT) != MaskVT)
1498 return DAG.UnrollVectorOp(
Node);
1500 SDValue StepVec = DAG.getStepVector(
DL, EVLVecVT);
1501 SDValue SplatEVL = DAG.getSplat(EVLVecVT,
DL, EVL);
1503 DAG.getSetCC(
DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1506 return DAG.getSelect(
DL,
Node->getValueType(0), FullMask, Op1, Op2);
1511 EVT VT =
Node->getValueType(0);
1513 unsigned DivOpc =
Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1515 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1516 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1517 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1529 SDValue Mul = DAG.getNode(ISD::VP_MUL,
DL, VT, Divisor, Div, Mask, EVL);
1530 return DAG.getNode(ISD::VP_SUB,
DL, VT, Dividend,
Mul, Mask, EVL);
1533void VectorLegalizer::ExpandFP_TO_UINT(
SDNode *
Node,
1537 if (TLI.expandFP_TO_UINT(
Node, Result, Chain, DAG)) {
1539 if (
Node->isStrictFPOpcode())
1545 if (
Node->isStrictFPOpcode()) {
1553void VectorLegalizer::ExpandUINT_TO_FLOAT(
SDNode *
Node,
1555 bool IsStrict =
Node->isStrictFPOpcode();
1556 unsigned OpNo = IsStrict ? 1 : 0;
1558 EVT VT = Src.getValueType();
1564 if (TLI.expandUINT_TO_FP(
Node, Result, Chain, DAG)) {
1573 TargetLowering::Expand) ||
1575 TargetLowering::Expand)) ||
1576 TLI.getOperationAction(
ISD::SRL, VT) == TargetLowering::Expand) {
1587 assert((BW == 64 || BW == 32) &&
1588 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1590 SDValue HalfWord = DAG.getConstant(BW / 2,
DL, VT);
1595 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1596 SDValue HalfWordMask = DAG.getConstant(HWMask,
DL, VT);
1600 DAG.getConstantFP(1ULL << (BW / 2),
DL,
Node->getValueType(0));
1611 {
Node->getValueType(0), MVT::Other},
1612 {
Node->getOperand(0),
HI});
1616 {
Node->getValueType(0), MVT::Other},
1617 {
Node->getOperand(0),
LO});
1645 if (TLI.isOperationLegalOrCustom(
ISD::FSUB,
Node->getValueType(0))) {
1650 Node->getOperand(0));
1652 return DAG.UnrollVectorOp(
Node);
1655void VectorLegalizer::ExpandFSUB(
SDNode *
Node,
1660 EVT VT =
Node->getValueType(0);
1661 if (TLI.isOperationLegalOrCustom(
ISD::FNEG, VT) &&
1662 TLI.isOperationLegalOrCustom(
ISD::FADD, VT))
1669void VectorLegalizer::ExpandSETCC(
SDNode *
Node,
1671 bool NeedInvert =
false;
1672 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
1676 unsigned Offset = IsStrict ? 1 : 0;
1683 MVT OpVT =
LHS.getSimpleValueType();
1686 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1703 TLI.LegalizeSetCCCondCode(DAG,
Node->getValueType(0), LHS, RHS,
CC, Mask,
1704 EVL, NeedInvert, dl, Chain, IsSignaling);
1711 LHS = DAG.getNode(
Node->getOpcode(), dl,
Node->getVTList(),
1712 {Chain, LHS, RHS, CC},
Node->getFlags());
1713 Chain =
LHS.getValue(1);
1715 LHS = DAG.getNode(ISD::VP_SETCC, dl,
Node->getValueType(0),
1716 {LHS, RHS, CC, Mask, EVL},
Node->getFlags());
1727 LHS = DAG.getLogicalNOT(dl, LHS,
LHS->getValueType(0));
1729 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL,
LHS->getValueType(0));
1732 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
1736 EVT VT =
Node->getValueType(0);
1739 DAG.getBoolConstant(
true, dl, VT,
LHS.getValueType()),
1740 DAG.getBoolConstant(
false, dl, VT,
LHS.getValueType()),
CC);
1741 LHS->setFlags(
Node->getFlags());
1749void VectorLegalizer::ExpandUADDSUBO(
SDNode *
Node,
1752 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
1757void VectorLegalizer::ExpandSADDSUBO(
SDNode *
Node,
1760 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
1765void VectorLegalizer::ExpandMULO(
SDNode *
Node,
1768 if (!TLI.expandMULO(
Node, Result, Overflow, DAG))
1769 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(
Node);
1775void VectorLegalizer::ExpandFixedPointDiv(
SDNode *
Node,
1778 if (
SDValue Expanded = TLI.expandFixedPointDiv(
N->getOpcode(),
SDLoc(
N),
1779 N->getOperand(0),
N->getOperand(1),
N->getConstantOperandVal(2), DAG))
1783void VectorLegalizer::ExpandStrictFPOp(
SDNode *
Node,
1806 "Expected REM node");
1809 if (!TLI.expandREM(
Node, Result, DAG))
1824 assert(!
Node->isStrictFPOpcode() &&
"Unexpected strict fp operation!");
1826 const char *LCName = TLI.getLibcallName(LC);
1829 LLVM_DEBUG(
dbgs() <<
"Looking for vector variant of " << LCName <<
"\n");
1831 EVT VT =
Node->getValueType(0);
1849 for (
unsigned i = 0; i <
Node->getNumOperands(); ++i) {
1850 assert(
Node->getOperand(i).getValueType() == VT &&
1851 "Expected matching vector types!");
1854 FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys,
false);
1866 if (OptVFInfo->Shape.Parameters.size() !=
1875 Entry.IsSExt =
false;
1876 Entry.IsZExt =
false;
1879 for (
auto &VFParam : OptVFInfo->Shape.Parameters) {
1880 if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
1881 EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);
1882 Entry.Node = DAG.getBoolConstant(
true,
DL, MaskVT, VT);
1884 Args.push_back(Entry);
1889 if (VFParam.ParamKind != VFParamKind::Vector)
1892 Entry.Node =
Node->getOperand(OpNum++);
1894 Args.push_back(Entry);
1899 TLI.getPointerTy(DAG.getDataLayout()));
1902 .setChain(DAG.getEntryNode())
1905 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
1906 Results.push_back(CallResult.first);
1911bool VectorLegalizer::tryExpandVecMathCall(
1916 Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64,
1917 Call_F80, Call_F128, Call_PPCF128);
1919 if (LC == RTLIB::UNKNOWN_LIBCALL)
1925void VectorLegalizer::UnrollStrictFPOp(
SDNode *
Node,
1927 EVT VT =
Node->getValueType(0);
1930 unsigned NumOpers =
Node->getNumOperands();
1933 EVT TmpEltVT = EltVT;
1937 *DAG.getContext(), TmpEltVT);
1939 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1945 for (
unsigned i = 0; i < NumElems; ++i) {
1947 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1953 for (
unsigned j = 1;
j < NumOpers; ++
j) {
1970 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1971 DAG.getAllOnesConstant(dl, EltVT),
1972 DAG.getConstant(0, dl, EltVT));
1986 EVT VT =
Node->getValueType(0);
1992 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
1995 for (
unsigned i = 0; i < NumElems; ++i) {
1997 DAG.getVectorIdxConstant(i, dl));
1999 DAG.getVectorIdxConstant(i, dl));
2002 *DAG.getContext(), TmpEltVT),
2003 LHSElem, RHSElem,
CC);
2004 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
2005 DAG.getConstant(0, dl, EltVT));
2007 return DAG.getBuildVector(VT, dl, Ops);
2011 return VectorLegalizer(*this).Run();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
ilist< SDNode >::iterator allnodes_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Provides info so a possible vectorization of a function can be computed.
std::string getVectorFunctionABIVariantString() const
Returns a vector function ABI variant string on the form: ZGV<isa><mask><vlen><vparams><scalarname>(<...
StringRef getVectorFnName() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.