49#define DEBUG_TYPE "legalizevectorops"
53class VectorLegalizer {
65 LegalizedNodes.
insert(std::make_pair(
From, To));
68 LegalizedNodes.
insert(std::make_pair(To, To));
137 std::pair<SDValue, SDValue> ExpandLoad(
SDNode *
N);
179bool VectorLegalizer::Run() {
181 bool HasVectors =
false;
183 E = std::prev(DAG.allnodes_end());
I != std::next(
E); ++
I) {
203 DAG.AssignTopologicalOrder();
205 E = std::prev(DAG.allnodes_end());
I != std::next(
E); ++
I)
209 SDValue OldRoot = DAG.getRoot();
210 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
211 DAG.setRoot(LegalizedNodes[OldRoot]);
213 LegalizedNodes.clear();
216 DAG.RemoveDeadNodes();
223 "Unexpected number of results");
225 for (
unsigned i = 0, e =
Op->getNumValues(); i != e; ++i)
226 AddLegalizedOperand(
Op.getValue(i),
SDValue(Result, i));
231VectorLegalizer::RecursivelyLegalizeResults(
SDValue Op,
234 "Unexpected number of results");
236 for (
unsigned i = 0, e =
Results.
size(); i != e; ++i) {
238 AddLegalizedOperand(
Op.getValue(i),
Results[i]);
248 if (
I != LegalizedNodes.end())
return I->second;
252 for (
const SDValue &Oper :
Op->op_values())
255 SDNode *
Node = DAG.UpdateNodeOperands(
Op.getNode(), Ops);
257 bool HasVectorValueOrOp =
260 [](
SDValue O) { return O.getValueType().isVector(); });
261 if (!HasVectorValueOrOp)
262 return TranslateLegalizeResults(Op,
Node);
266 switch (
Op.getOpcode()) {
268 return TranslateLegalizeResults(Op,
Node);
272 EVT LoadedVT =
LD->getMemoryVT();
274 Action = TLI.getLoadExtAction(ExtType,
LD->getValueType(0), LoadedVT);
279 EVT StVT =
ST->getMemoryVT();
280 MVT ValVT =
ST->getValue().getSimpleValueType();
281 if (StVT.
isVector() &&
ST->isTruncatingStore())
282 Action = TLI.getTruncStoreAction(ValVT, StVT);
286 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
289 if (Action == TargetLowering::Legal)
290 Action = TargetLowering::Expand;
292#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
293 case ISD::STRICT_##DAGN:
294#include "llvm/IR/ConstrainedOps.def"
295 ValVT =
Node->getValueType(0);
298 ValVT =
Node->getOperand(1).getValueType();
299 Action = TLI.getOperationAction(
Node->getOpcode(), ValVT);
305 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
306 TLI.getStrictFPOperationAction(
Node->getOpcode(), ValVT) ==
307 TargetLowering::Legal) {
309 if (TLI.getOperationAction(
Node->getOpcode(), EltVT)
310 == TargetLowering::Expand &&
311 TLI.getStrictFPOperationAction(
Node->getOpcode(), EltVT)
312 == TargetLowering::Legal)
313 Action = TargetLowering::Legal;
414 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
424 unsigned Scale =
Node->getConstantOperandVal(2);
425 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
426 Node->getValueType(0), Scale);
444 Action = TLI.getOperationAction(
Node->getOpcode(),
445 Node->getOperand(0).getValueType());
449 Action = TLI.getOperationAction(
Node->getOpcode(),
450 Node->getOperand(1).getValueType());
453 MVT OpVT =
Node->getOperand(0).getSimpleValueType();
455 Action = TLI.getCondCodeAction(CCCode, OpVT);
456 if (Action == TargetLowering::Legal)
457 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
461#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
463 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
464 : Node->getOperand(LEGALPOS).getValueType(); \
465 if (ISD::VPID == ISD::VP_SETCC) { \
466 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
467 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
468 if (Action != TargetLowering::Legal) \
471 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
473#include "llvm/IR/VPIntrinsics.def"
481 case TargetLowering::Promote:
483 "This action is not supported yet!");
485 Promote(
Node, ResultVals);
486 assert(!ResultVals.
empty() &&
"No results for promotion?");
488 case TargetLowering::Legal:
491 case TargetLowering::Custom:
493 if (LowerOperationWrapper(
Node, ResultVals))
497 case TargetLowering::Expand:
499 Expand(
Node, ResultVals);
503 if (ResultVals.
empty())
504 return TranslateLegalizeResults(Op,
Node);
507 return RecursivelyLegalizeResults(Op, ResultVals);
512bool VectorLegalizer::LowerOperationWrapper(
SDNode *
Node,
524 if (
Node->getNumValues() == 1) {
532 "Lowering returned the wrong number of results!");
535 for (
unsigned I = 0,
E =
Node->getNumValues();
I !=
E; ++
I)
544 switch (
Node->getOpcode()) {
572 "Can't promote a vector with multiple results!");
573 MVT VT =
Node->getSimpleValueType(0);
574 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
578 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
579 if (
Node->getOperand(j).getValueType().isVector())
580 if (
Node->getOperand(j)
582 .getVectorElementType()
583 .isFloatingPoint() &&
599 DAG.getIntPtrConstant(0, dl,
true));
606void VectorLegalizer::PromoteINT_TO_FP(
SDNode *
Node,
610 bool IsStrict =
Node->isStrictFPOpcode();
611 MVT VT =
Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
612 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
614 "Vectors have different number of elements!");
623 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
624 if (
Node->getOperand(j).getValueType().isVector())
625 Operands[j] = DAG.getNode(Opc, dl, NVT,
Node->getOperand(j));
647void VectorLegalizer::PromoteFP_TO_INT(
SDNode *
Node,
649 MVT VT =
Node->getSimpleValueType(0);
650 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
651 bool IsStrict =
Node->isStrictFPOpcode();
653 "Vectors have different number of elements!");
655 unsigned NewOpc =
Node->getOpcode();
670 {
Node->getOperand(0),
Node->getOperand(1)});
673 Promoted = DAG.
getNode(NewOpc, dl, NVT,
Node->getOperand(0));
684 Promoted = DAG.
getNode(NewOpc, dl, NVT, Promoted,
692std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(
SDNode *
N) {
694 return TLI.scalarizeVectorLoad(LD, DAG);
699 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
704 switch (
Node->getOpcode()) {
706 std::pair<SDValue, SDValue> Tmp = ExpandLoad(
Node);
715 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
722 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(
Node));
725 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(
Node));
728 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(
Node));
753 if (
Node->getValueType(0).isScalableVector()) {
754 EVT CondVT = TLI.getSetCCResultType(
755 DAG.getDataLayout(), *DAG.getContext(),
Node->getValueType(0));
758 Node->getOperand(1),
Node->getOperand(4));
761 Node->getOperand(3)));
791 case ISD::VP_BITREVERSE:
792 if (
SDValue Expanded = TLI.expandVPBITREVERSE(
Node, DAG)) {
798 if (
SDValue Expanded = TLI.expandCTPOP(
Node, DAG)) {
804 if (
SDValue Expanded = TLI.expandVPCTPOP(
Node, DAG)) {
811 if (
SDValue Expanded = TLI.expandCTLZ(
Node, DAG)) {
817 case ISD::VP_CTLZ_ZERO_UNDEF:
818 if (
SDValue Expanded = TLI.expandVPCTLZ(
Node, DAG)) {
825 if (
SDValue Expanded = TLI.expandCTTZ(
Node, DAG)) {
831 case ISD::VP_CTTZ_ZERO_UNDEF:
832 if (
SDValue Expanded = TLI.expandVPCTTZ(
Node, DAG)) {
841 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG)) {
848 if (
SDValue Expanded = TLI.expandROT(
Node,
false , DAG)) {
855 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG)) {
864 if (
SDValue Expanded = TLI.expandIntMINMAX(
Node, DAG)) {
885 if (
SDValue Expanded = TLI.expandAddSubSat(
Node, DAG)) {
892 if (
SDValue Expanded = TLI.expandShlSat(
Node, DAG)) {
900 if (
Node->getValueType(0).isScalableVector()) {
901 if (
SDValue Expanded = TLI.expandFP_TO_INT_SAT(
Node, DAG)) {
909 if (
SDValue Expanded = TLI.expandFixedPointMul(
Node, DAG)) {
928#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
929 case ISD::STRICT_##DAGN:
930#include "llvm/IR/ConstrainedOps.def"
950 Results.push_back(TLI.expandVecReduceSeq(
Node, DAG));
968 EVT VT =
Node->getValueType(0);
986 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
987 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
988 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand ||
991 VT) == TargetLowering::Expand)
992 return DAG.UnrollVectorOp(
Node);
1000 Mask = DAG.getSelect(
DL, BitTy, Mask, DAG.getAllOnesConstant(
DL, BitTy),
1001 DAG.getConstant(0,
DL, BitTy));
1004 Mask = DAG.getSplat(MaskTy,
DL, Mask);
1012 SDValue NotMask = DAG.getNOT(
DL, Mask, MaskTy);
1021 EVT VT =
Node->getValueType(0);
1024 if (TLI.getOperationAction(
ISD::SRA, VT) == TargetLowering::Expand ||
1025 TLI.getOperationAction(
ISD::SHL, VT) == TargetLowering::Expand)
1026 return DAG.UnrollVectorOp(
Node);
1029 EVT OrigTy = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1033 SDValue ShiftSz = DAG.getConstant(BW - OrigBW,
DL, VT);
1036 return DAG.getNode(
ISD::SRA,
DL, VT, Op, ShiftSz);
1043 EVT VT =
Node->getValueType(0);
1046 EVT SrcVT = Src.getValueType();
1053 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1058 Src, DAG.getVectorIdxConstant(0,
DL));
1063 ShuffleMask.
resize(NumSrcElements, -1);
1066 int ExtLaneScale = NumSrcElements / NumElements;
1067 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1068 for (
int i = 0; i < NumElements; ++i)
1069 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1073 DAG.getVectorShuffle(SrcVT,
DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1078 EVT VT =
Node->getValueType(0);
1080 EVT SrcVT = Src.getValueType();
1091 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth,
DL, VT);
1093 DAG.getNode(
ISD::SHL,
DL, VT, Op, ShiftAmount),
1102 EVT VT =
Node->getValueType(0);
1105 EVT SrcVT = Src.getValueType();
1112 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1117 Src, DAG.getVectorIdxConstant(0,
DL));
1125 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1127 int ExtLaneScale = NumSrcElements / NumElements;
1128 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1129 for (
int i = 0; i < NumElements; ++i)
1130 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1133 DAG.getVectorShuffle(SrcVT,
DL, Zero, Src, ShuffleMask));
1139 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
1140 ShuffleMask.
push_back((
I * ScalarSizeInBytes) + J);
1144 EVT VT =
Node->getValueType(0);
1148 return TLI.expandBSWAP(
Node, DAG);
1156 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1159 Op = DAG.getVectorShuffle(ByteVT,
DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1165 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1166 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1167 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1168 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT))
1169 return TLI.expandBSWAP(
Node, DAG);
1172 return DAG.UnrollVectorOp(
Node);
1175void VectorLegalizer::ExpandBITREVERSE(
SDNode *
Node,
1177 EVT VT =
Node->getValueType(0);
1181 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1196 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1201 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1203 (TLI.isOperationLegalOrCustom(
ISD::SHL, ByteVT) &&
1204 TLI.isOperationLegalOrCustom(
ISD::SRL, ByteVT) &&
1205 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, ByteVT) &&
1206 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, ByteVT)))) {
1209 Op = DAG.getVectorShuffle(ByteVT,
DL, Op, DAG.getUNDEF(ByteVT),
1220 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1221 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1222 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1223 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT)) {
1224 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1248 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1249 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1250 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand)
1251 return DAG.UnrollVectorOp(
Node);
1257 auto BoolContents = TLI.getBooleanContents(Op1.
getValueType());
1258 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1259 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1261 return DAG.UnrollVectorOp(
Node);
1267 return DAG.UnrollVectorOp(
Node);
1275 SDValue NotMask = DAG.getNOT(
DL, Mask, VT);
1297 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1298 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1299 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1300 return DAG.UnrollVectorOp(
Node);
1304 return DAG.UnrollVectorOp(
Node);
1306 SDValue Ones = DAG.getAllOnesConstant(
DL, VT);
1309 Op1 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op1, Mask, Mask, EVL);
1310 Op2 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op2, NotMask, Mask, EVL);
1311 return DAG.getNode(ISD::VP_OR,
DL, VT, Op1, Op2, Mask, EVL);
1326 EVT MaskVT =
Mask.getValueType();
1338 return DAG.UnrollVectorOp(
Node);
1342 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1343 EVLVecVT) != MaskVT)
1344 return DAG.UnrollVectorOp(
Node);
1346 SDValue StepVec = DAG.getStepVector(
DL, EVLVecVT);
1347 SDValue SplatEVL = DAG.getSplat(EVLVecVT,
DL, EVL);
1349 DAG.getSetCC(
DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1352 return DAG.getSelect(
DL,
Node->getValueType(0), FullMask, Op1, Op2);
1357 EVT VT =
Node->getValueType(0);
1359 unsigned DivOpc =
Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1361 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1362 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1363 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1375 SDValue Mul = DAG.getNode(ISD::VP_MUL,
DL, VT, Divisor, Div, Mask, EVL);
1376 return DAG.getNode(ISD::VP_SUB,
DL, VT, Dividend,
Mul, Mask, EVL);
1379void VectorLegalizer::ExpandFP_TO_UINT(
SDNode *
Node,
1383 if (TLI.expandFP_TO_UINT(
Node, Result, Chain, DAG)) {
1385 if (
Node->isStrictFPOpcode())
1391 if (
Node->isStrictFPOpcode()) {
1399void VectorLegalizer::ExpandUINT_TO_FLOAT(
SDNode *
Node,
1401 bool IsStrict =
Node->isStrictFPOpcode();
1402 unsigned OpNo = IsStrict ? 1 : 0;
1404 EVT VT = Src.getValueType();
1410 if (TLI.expandUINT_TO_FP(
Node, Result, Chain, DAG)) {
1419 TargetLowering::Expand) ||
1421 TargetLowering::Expand)) ||
1422 TLI.getOperationAction(
ISD::SRL, VT) == TargetLowering::Expand) {
1433 assert((BW == 64 || BW == 32) &&
1434 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1436 SDValue HalfWord = DAG.getConstant(BW / 2,
DL, VT);
1441 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1442 SDValue HalfWordMask = DAG.getConstant(HWMask,
DL, VT);
1446 DAG.getConstantFP(1ULL << (BW / 2),
DL,
Node->getValueType(0));
1458 {
Node->getOperand(0),
HI});
1463 {
Node->getOperand(0),
LO});
1491 if (TLI.isOperationLegalOrCustom(
ISD::FSUB,
Node->getValueType(0))) {
1496 Node->getOperand(0));
1498 return DAG.UnrollVectorOp(
Node);
1501void VectorLegalizer::ExpandFSUB(
SDNode *
Node,
1506 EVT VT =
Node->getValueType(0);
1507 if (TLI.isOperationLegalOrCustom(
ISD::FNEG, VT) &&
1508 TLI.isOperationLegalOrCustom(
ISD::FADD, VT))
1515void VectorLegalizer::ExpandSETCC(
SDNode *
Node,
1517 bool NeedInvert =
false;
1518 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
1520 MVT OpVT =
Node->getOperand(0).getSimpleValueType();
1523 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1535 EVL =
Node->getOperand(4);
1539 TLI.LegalizeSetCCCondCode(DAG,
Node->getValueType(0), LHS, RHS,
CC, Mask,
1540 EVL, NeedInvert, dl, Chain);
1550 LHS = DAG.getNode(ISD::VP_SETCC, dl,
Node->getValueType(0),
1551 {LHS, RHS, CC, Mask, EVL},
Node->getFlags());
1558 LHS = DAG.getLogicalNOT(dl, LHS,
LHS->getValueType(0));
1560 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL,
LHS->getValueType(0));
1565 EVT VT =
Node->getValueType(0);
1568 DAG.getBoolConstant(
true, dl, VT,
LHS.getValueType()),
1569 DAG.getBoolConstant(
false, dl, VT,
LHS.getValueType()),
CC);
1570 LHS->setFlags(
Node->getFlags());
1576void VectorLegalizer::ExpandUADDSUBO(
SDNode *
Node,
1579 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
1584void VectorLegalizer::ExpandSADDSUBO(
SDNode *
Node,
1587 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
1592void VectorLegalizer::ExpandMULO(
SDNode *
Node,
1595 if (!TLI.expandMULO(
Node, Result, Overflow, DAG))
1596 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(
Node);
1602void VectorLegalizer::ExpandFixedPointDiv(
SDNode *
Node,
1605 if (
SDValue Expanded = TLI.expandFixedPointDiv(
N->getOpcode(),
SDLoc(
N),
1606 N->getOperand(0),
N->getOperand(1),
N->getConstantOperandVal(2), DAG))
1610void VectorLegalizer::ExpandStrictFPOp(
SDNode *
Node,
1627 "Expected REM node");
1630 if (!TLI.expandREM(
Node, Result, DAG))
1635void VectorLegalizer::UnrollStrictFPOp(
SDNode *
Node,
1637 EVT VT =
Node->getValueType(0);
1640 unsigned NumOpers =
Node->getNumOperands();
1643 EVT TmpEltVT = EltVT;
1647 *DAG.getContext(), TmpEltVT);
1655 for (
unsigned i = 0; i < NumElems; ++i) {
1657 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1663 for (
unsigned j = 1;
j < NumOpers; ++
j) {
1680 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1681 DAG.getAllOnesConstant(dl, EltVT),
1682 DAG.getConstant(0, dl, EltVT));
1696 EVT VT =
Node->getValueType(0);
1702 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
1705 for (
unsigned i = 0; i < NumElems; ++i) {
1707 DAG.getVectorIdxConstant(i, dl));
1709 DAG.getVectorIdxConstant(i, dl));
1712 *DAG.getContext(), TmpEltVT),
1713 LHSElem, RHSElem,
CC);
1714 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
1715 DAG.getConstant(0, dl, EltVT));
1717 return DAG.getBuildVector(VT, dl, Ops);
1721 return VectorLegalizer(*this).Run();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Iterator for intrusive lists based on ilist_node.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.