47#define LV_NAME "loop-vectorize"
48#define DEBUG_TYPE LV_NAME
56 case VPInterleaveEVLSC:
59 case VPWidenStoreEVLSC:
67 ->getCalledScalarFunction()
69 case VPWidenIntrinsicSC:
71 case VPCanonicalIVPHISC:
72 case VPBranchOnMaskSC:
73 case VPFirstOrderRecurrencePHISC:
74 case VPReductionPHISC:
75 case VPScalarIVStepsSC:
79 case VPReductionEVLSC:
81 case VPVectorPointerSC:
82 case VPWidenCanonicalIVSC:
85 case VPWidenIntOrFpInductionSC:
86 case VPWidenLoadEVLSC:
90 case VPWidenSelectSC: {
94 assert((!
I || !
I->mayWriteToMemory()) &&
95 "underlying instruction may write to memory");
107 case VPInstructionSC:
109 case VPWidenLoadEVLSC:
114 ->mayReadFromMemory();
117 ->getCalledScalarFunction()
118 ->onlyWritesMemory();
119 case VPWidenIntrinsicSC:
121 case VPBranchOnMaskSC:
122 case VPFirstOrderRecurrencePHISC:
123 case VPPredInstPHISC:
124 case VPScalarIVStepsSC:
125 case VPWidenStoreEVLSC:
129 case VPReductionEVLSC:
131 case VPVectorPointerSC:
132 case VPWidenCanonicalIVSC:
135 case VPWidenIntOrFpInductionSC:
138 case VPWidenSelectSC: {
142 assert((!
I || !
I->mayReadFromMemory()) &&
143 "underlying instruction may read from memory");
157 case VPFirstOrderRecurrencePHISC:
158 case VPPredInstPHISC:
159 case VPVectorEndPointerSC:
161 case VPInstructionSC:
163 case VPWidenCallSC: {
167 case VPWidenIntrinsicSC:
170 case VPReductionEVLSC:
172 case VPScalarIVStepsSC:
173 case VPVectorPointerSC:
174 case VPWidenCanonicalIVSC:
177 case VPWidenIntOrFpInductionSC:
179 case VPWidenPointerInductionSC:
181 case VPWidenSelectSC: {
185 assert((!
I || !
I->mayHaveSideEffects()) &&
186 "underlying instruction has side-effects");
189 case VPInterleaveEVLSC:
192 case VPWidenLoadEVLSC:
194 case VPWidenStoreEVLSC:
199 "mayHaveSideffects result for ingredient differs from this "
202 case VPReplicateSC: {
204 return R->getUnderlyingInstr()->mayHaveSideEffects();
212 assert(!Parent &&
"Recipe already in some VPBasicBlock");
214 "Insertion position not in any VPBasicBlock");
220 assert(!Parent &&
"Recipe already in some VPBasicBlock");
226 assert(!Parent &&
"Recipe already in some VPBasicBlock");
228 "Insertion position not in any VPBasicBlock");
263 UI = IG->getInsertPos();
265 UI = &WidenMem->getIngredient();
268 if (UI && Ctx.skipCostComputation(UI, VF.
isVector())) {
278 dbgs() <<
"Cost of " << RecipeCost <<
" for VF " << VF <<
": ";
302 std::optional<unsigned> Opcode;
308 OpR =
Op->getDefiningRecipe();
311 Type *InputTypeA =
nullptr, *InputTypeB =
nullptr;
321 if (WidenCastR->getOpcode() == Instruction::CastOps::ZExt)
323 if (WidenCastR->getOpcode() == Instruction::CastOps::SExt)
334 Opcode =
Widen->getOpcode();
337 InputTypeA = Ctx.Types.inferScalarType(ExtAR ? ExtAR->
getOperand(0)
338 :
Widen->getOperand(0));
339 InputTypeB = Ctx.Types.inferScalarType(ExtBR ? ExtBR->
getOperand(0)
340 :
Widen->getOperand(1));
341 ExtAType = GetExtendKind(ExtAR);
342 ExtBType = GetExtendKind(ExtBR);
348 InputTypeB = InputTypeA;
354 InputTypeA = Ctx.Types.inferScalarType(OpR->
getOperand(0));
355 ExtAType = GetExtendKind(OpR);
359 InputTypeA = Ctx.Types.inferScalarType(RedPhiOp1R->getOperand(0));
360 ExtAType = GetExtendKind(RedPhiOp1R);
366 return Reduction->computeCost(VF, Ctx);
368 auto *PhiType = Ctx.Types.inferScalarType(
getOperand(1));
369 return Ctx.TTI.getPartialReductionCost(
getOpcode(), InputTypeA, InputTypeB,
370 PhiType, VF, ExtAType, ExtBType,
371 Opcode, Ctx.CostKind);
375 auto &Builder = State.Builder;
378 "Unhandled partial reduction opcode");
382 assert(PhiVal && BinOpVal &&
"Phi and Mul must be set");
387 Builder.CreateIntrinsic(RetTy, Intrinsic::vector_partial_reduce_add,
388 {PhiVal, BinOpVal},
nullptr,
"partial.reduce");
393#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
396 O << Indent <<
"PARTIAL-REDUCE ";
404 assert(OpType == Other.OpType &&
"OpType must match");
406 case OperationType::OverflowingBinOp:
407 WrapFlags.HasNUW &= Other.WrapFlags.HasNUW;
408 WrapFlags.HasNSW &= Other.WrapFlags.HasNSW;
410 case OperationType::Trunc:
414 case OperationType::DisjointOp:
417 case OperationType::PossiblyExactOp:
418 ExactFlags.IsExact &= Other.ExactFlags.IsExact;
420 case OperationType::GEPOp:
423 case OperationType::FPMathOp:
424 FMFs.NoNaNs &= Other.FMFs.NoNaNs;
425 FMFs.NoInfs &= Other.FMFs.NoInfs;
427 case OperationType::NonNegOp:
430 case OperationType::Cmp:
433 case OperationType::Other:
440 assert(OpType == OperationType::FPMathOp &&
441 "recipe doesn't have fast math flags");
453#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
457template <
unsigned PartOpIdx>
460 if (U.getNumOperands() == PartOpIdx + 1)
461 return U.getOperand(PartOpIdx);
465template <
unsigned PartOpIdx>
484 "Set flags not supported for the provided opcode");
485 assert((getNumOperandsForOpcode(Opcode) == -1u ||
487 "number of operands does not match opcode");
491unsigned VPInstruction::getNumOperandsForOpcode(
unsigned Opcode) {
502 case Instruction::Alloca:
503 case Instruction::ExtractValue:
504 case Instruction::Freeze:
505 case Instruction::Load:
520 case Instruction::ICmp:
521 case Instruction::FCmp:
522 case Instruction::Store:
531 case Instruction::Select:
538 case Instruction::Call:
539 case Instruction::GetElementPtr:
540 case Instruction::PHI:
541 case Instruction::Switch:
553bool VPInstruction::canGenerateScalarForFirstLane()
const {
559 case Instruction::Freeze:
560 case Instruction::ICmp:
561 case Instruction::PHI:
562 case Instruction::Select:
588 BasicBlock *SecondIRSucc = State.CFG.VPBB2IRBB.lookup(SecondVPSucc);
590 BranchInst *CondBr = State.Builder.CreateCondBr(
Cond, IRBB, SecondIRSucc);
598 IRBuilderBase &Builder = State.
Builder;
617 case Instruction::ExtractElement: {
620 unsigned IdxToExtract =
628 case Instruction::Freeze: {
632 case Instruction::FCmp:
633 case Instruction::ICmp: {
639 case Instruction::PHI: {
642 case Instruction::Select: {
667 {VIVElem0, ScalarTC},
nullptr, Name);
683 if (!V1->getType()->isVectorTy())
703 "Requested vector length should be an integer.");
710 {AVL, VFArg, State.Builder.getTrue()});
716 assert(Part != 0 &&
"Must have a positive part");
747 for (
unsigned FieldIndex = 0; FieldIndex != StructTy->getNumElements();
771 IRBuilderBase::FastMathFlagGuard FMFG(Builder);
801 RecurKind RK = PhiR->getRecurrenceKind();
803 "Unexpected reduction kind");
804 assert(!PhiR->isInLoop() &&
805 "In-loop FindLastIV reduction is not supported yet");
817 for (
unsigned Part = 1; Part <
UF; ++Part)
818 ReducedPartRdx =
createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx,
832 RecurKind RK = PhiR->getRecurrenceKind();
834 "should be handled by ComputeFindIVResult");
840 for (
unsigned Part = 0; Part <
UF; ++Part)
841 RdxParts[Part] = State.
get(
getOperand(1 + Part), PhiR->isInLoop());
843 IRBuilderBase::FastMathFlagGuard FMFG(Builder);
848 Value *ReducedPartRdx = RdxParts[0];
849 if (PhiR->isOrdered()) {
850 ReducedPartRdx = RdxParts[
UF - 1];
853 for (
unsigned Part = 1; Part <
UF; ++Part) {
854 Value *RdxPart = RdxParts[Part];
856 ReducedPartRdx =
createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
862 Opcode = Instruction::Add;
867 Builder.
CreateBinOp(Opcode, RdxPart, ReducedPartRdx,
"bin.rdx");
874 if (State.
VF.
isVector() && !PhiR->isInLoop()) {
881 return ReducedPartRdx;
891 "invalid offset to extract from");
895 assert(
Offset <= 1 &&
"invalid offset to extract from");
909 "can only generate first lane for PtrAdd");
929 Value *Res =
nullptr;
934 Builder.CreateMul(RuntimeVF, ConstantInt::get(IdxTy, Idx - 1));
935 Value *VectorIdx = Idx == 1
937 : Builder.CreateSub(LaneToExtract, VectorStart);
938 Value *Ext = State.VF.isScalar()
940 : Builder.CreateExtractElement(
943 Value *Cmp = Builder.CreateICmpUGE(LaneToExtract, VectorStart);
944 Res = Builder.CreateSelect(Cmp, Ext, Res);
963 Value *Res =
nullptr;
964 for (
int Idx = LastOpIdx; Idx >= 0; --Idx) {
965 Value *TrailingZeros =
995 Type *ScalarTy = Ctx.Types.inferScalarType(
this);
998 case Instruction::FNeg:
999 return Ctx.TTI.getArithmeticInstrCost(Opcode, ResultTy, Ctx.CostKind);
1000 case Instruction::UDiv:
1001 case Instruction::SDiv:
1002 case Instruction::SRem:
1003 case Instruction::URem:
1004 case Instruction::Add:
1005 case Instruction::FAdd:
1006 case Instruction::Sub:
1007 case Instruction::FSub:
1008 case Instruction::Mul:
1009 case Instruction::FMul:
1010 case Instruction::FDiv:
1011 case Instruction::FRem:
1012 case Instruction::Shl:
1013 case Instruction::LShr:
1014 case Instruction::AShr:
1015 case Instruction::And:
1016 case Instruction::Or:
1017 case Instruction::Xor: {
1025 RHSInfo = Ctx.getOperandInfo(RHS);
1036 return Ctx.TTI.getArithmeticInstrCost(
1037 Opcode, ResultTy, Ctx.CostKind,
1038 {TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
1039 RHSInfo, Operands, CtxI, &Ctx.TLI);
1041 case Instruction::Freeze:
1043 return Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, ResultTy,
1045 case Instruction::ExtractValue:
1046 return Ctx.TTI.getInsertExtractValueCost(Instruction::ExtractValue,
1048 case Instruction::ICmp:
1049 case Instruction::FCmp: {
1053 return Ctx.TTI.getCmpSelInstrCost(
1055 Ctx.CostKind, {TTI::OK_AnyValue, TTI::OP_None},
1056 {TTI::OK_AnyValue, TTI::OP_None}, CtxI);
1072 "Should only generate a vector value or single scalar, not scalars "
1080 case Instruction::Select: {
1084 auto *CondTy = Ctx.Types.inferScalarType(
getOperand(0));
1085 auto *VecTy = Ctx.Types.inferScalarType(
getOperand(1));
1090 return Ctx.TTI.getCmpSelInstrCost(Instruction::Select, VecTy, CondTy, Pred,
1093 case Instruction::ExtractElement:
1103 return Ctx.TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy,
1107 auto *VecTy =
toVectorTy(Ctx.Types.inferScalarType(
this), VF);
1108 return Ctx.TTI.getArithmeticReductionCost(
1114 return Ctx.TTI.getCmpSelInstrCost(Instruction::ICmp, ScalarTy,
1121 {PredTy, Type::getInt1Ty(Ctx.LLVMCtx)});
1122 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
1128 Type *VectorTy =
toVectorTy(Ctx.Types.inferScalarType(
this), VF);
1137 unsigned Multiplier =
1142 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
1149 I32Ty, {Arg0Ty, I32Ty, I1Ty});
1150 return Ctx.TTI.getIntrinsicInstrCost(Attrs, Ctx.CostKind);
1155 return Ctx.TTI.getIndexedVectorInstrCostFromEnd(Instruction::ExtractElement,
1156 VecTy, Ctx.CostKind, 0);
1166 "unexpected VPInstruction witht underlying value");
1175 getOpcode() == Instruction::ExtractElement ||
1186 case Instruction::PHI:
1197 assert(!State.Lane &&
"VPInstruction executing an Lane");
1200 "Set flags not supported for the provided opcode");
1203 Value *GeneratedValue = generate(State);
1206 assert(GeneratedValue &&
"generate must produce a value");
1207 bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&
1212 !GeneratesPerFirstLaneOnly) ||
1213 State.VF.isScalar()) &&
1214 "scalar value but not only first lane defined");
1215 State.set(
this, GeneratedValue,
1216 GeneratesPerFirstLaneOnly);
1223 case Instruction::ExtractElement:
1224 case Instruction::Freeze:
1225 case Instruction::FCmp:
1226 case Instruction::ICmp:
1227 case Instruction::Select:
1228 case Instruction::PHI:
1265 case Instruction::ExtractElement:
1267 case Instruction::PHI:
1269 case Instruction::FCmp:
1270 case Instruction::ICmp:
1271 case Instruction::Select:
1272 case Instruction::Or:
1273 case Instruction::Freeze:
1314 case Instruction::FCmp:
1315 case Instruction::ICmp:
1316 case Instruction::Select:
1326#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1334 O << Indent <<
"EMIT" << (
isSingleScalar() ?
"-SCALAR" :
"") <<
" ";
1346 O <<
"combined load";
1349 O <<
"combined store";
1352 O <<
"active lane mask";
1355 O <<
"EXPLICIT-VECTOR-LENGTH";
1358 O <<
"first-order splice";
1361 O <<
"branch-on-cond";
1364 O <<
"TC > VF ? TC - VF : 0";
1370 O <<
"branch-on-count";
1376 O <<
"buildstructvector";
1382 O <<
"extract-lane";
1385 O <<
"extract-last-element";
1388 O <<
"extract-last-lane-per-part";
1391 O <<
"extract-penultimate-element";
1394 O <<
"compute-anyof-result";
1397 O <<
"compute-find-iv-result";
1400 O <<
"compute-reduction-result";
1415 O <<
"first-active-lane";
1418 O <<
"reduction-start-vector";
1421 O <<
"resume-for-epilogue";
1446 State.set(
this, Cast,
VPLane(0));
1457 Value *
VScale = State.Builder.CreateVScale(ResultTy);
1458 State.set(
this,
VScale,
true);
1467#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1470 O << Indent <<
"EMIT" << (
isSingleScalar() ?
"-SCALAR" :
"") <<
" ";
1476 O <<
"wide-iv-step ";
1480 O <<
"step-vector " << *ResultTy;
1483 O <<
"vscale " << *ResultTy;
1489 O <<
" to " << *ResultTy;
1496 PHINode *NewPhi = State.Builder.CreatePHI(
1497 State.TypeAnalysis.inferScalarType(
this), 2,
getName());
1504 for (
unsigned Idx = 0; Idx != NumIncoming; ++Idx) {
1509 State.set(
this, NewPhi,
VPLane(0));
1512#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1515 O << Indent <<
"EMIT" << (
isSingleScalar() ?
"-SCALAR" :
"") <<
" ";
1530 "PHINodes must be handled by VPIRPhi");
1533 State.Builder.SetInsertPoint(I.getParent(), std::next(I.getIterator()));
1545 "can only update exiting operands to phi nodes");
1555#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1558 O << Indent <<
"IR " << I;
1570 auto *PredVPBB = Pred->getExitingBasicBlock();
1571 BasicBlock *PredBB = State.CFG.VPBB2IRBB[PredVPBB];
1578 if (Phi->getBasicBlockIndex(PredBB) == -1)
1579 Phi->addIncoming(V, PredBB);
1581 Phi->setIncomingValueForBlock(PredBB, V);
1586 State.Builder.SetInsertPoint(Phi->getParent(), std::next(Phi->getIterator()));
1591 assert(R->getNumOperands() == R->getParent()->getNumPredecessors() &&
1592 "Number of phi operands must match number of predecessors");
1593 unsigned Position = R->getParent()->getIndexForPredecessor(IncomingBlock);
1594 R->removeOperand(Position);
1597#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1611#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1617 O <<
" (extra operand" << (
getNumOperands() > 1 ?
"s" :
"") <<
": ";
1622 std::get<1>(
Op)->printAsOperand(O);
1635 Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD);
1637 Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD);
1641 for (
const auto &[Kind,
Node] : Metadata)
1642 I.setMetadata(Kind,
Node);
1647 for (
const auto &[KindA, MDA] : Metadata) {
1648 for (
const auto &[KindB, MDB] :
Other.Metadata) {
1649 if (KindA == KindB && MDA == MDB) {
1655 Metadata = std::move(MetadataIntersection);
1659 assert(State.VF.isVector() &&
"not widening");
1660 assert(Variant !=
nullptr &&
"Can't create vector function.");
1671 Arg = State.get(
I.value(),
VPLane(0));
1674 Args.push_back(Arg);
1680 CI->getOperandBundlesAsDefs(OpBundles);
1682 CallInst *V = State.Builder.CreateCall(Variant, Args, OpBundles);
1685 V->setCallingConv(Variant->getCallingConv());
1687 if (!V->getType()->isVoidTy())
1693 return Ctx.TTI.getCallInstrCost(
nullptr, Variant->getReturnType(),
1694 Variant->getFunctionType()->params(),
1698#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1701 O << Indent <<
"WIDEN-CALL ";
1713 O <<
" @" << CalledFn->
getName() <<
"(";
1719 O <<
" (using library function";
1720 if (Variant->hasName())
1721 O <<
": " << Variant->getName();
1727 assert(State.VF.isVector() &&
"not widening");
1740 Arg = State.get(
I.value(),
VPLane(0));
1746 Args.push_back(Arg);
1750 Module *M = State.Builder.GetInsertBlock()->getModule();
1754 "Can't retrieve vector intrinsic or vector-predication intrinsics.");
1759 CI->getOperandBundlesAsDefs(OpBundles);
1761 CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
1766 if (!V->getType()->isVoidTy())
1782 for (
const auto &[Idx,
Op] :
enumerate(Operands)) {
1783 auto *V =
Op->getUnderlyingValue();
1786 Arguments.push_back(UI->getArgOperand(Idx));
1795 Type *ScalarRetTy = Ctx.Types.inferScalarType(&R);
1801 : Ctx.Types.inferScalarType(
Op));
1806 R.hasFastMathFlags() ? R.getFastMathFlags() :
FastMathFlags();
1811 return Ctx.TTI.getIntrinsicInstrCost(CostAttrs, Ctx.CostKind);
1833#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1836 O << Indent <<
"WIDEN-INTRINSIC ";
1837 if (ResultTy->isVoidTy()) {
1865 Value *Mask =
nullptr;
1867 Mask = State.get(VPMask);
1870 Builder.CreateVectorSplat(VTy->
getElementCount(), Builder.getInt1(1));
1874 if (Opcode == Instruction::Sub)
1875 IncAmt = Builder.CreateNeg(IncAmt);
1877 assert(Opcode == Instruction::Add &&
"only add or sub supported for now");
1879 State.Builder.CreateIntrinsic(Intrinsic::experimental_vector_histogram_add,
1894 Type *IncTy = Ctx.Types.inferScalarType(IncAmt);
1900 Ctx.TTI.getArithmeticInstrCost(Instruction::Mul, VTy, Ctx.CostKind);
1913 {PtrTy, IncTy, MaskTy});
1916 return Ctx.TTI.getIntrinsicInstrCost(ICA, Ctx.CostKind) + MulCost +
1917 Ctx.TTI.getArithmeticInstrCost(Opcode, VTy, Ctx.CostKind);
1920#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1923 O << Indent <<
"WIDEN-HISTOGRAM buckets: ";
1926 if (Opcode == Instruction::Sub)
1929 assert(Opcode == Instruction::Add);
1942 O << Indent <<
"WIDEN-SELECT ";
1964 Value *Sel = State.Builder.CreateSelect(
Cond, Op0, Op1);
1965 State.set(
this, Sel);
1977 Type *ScalarTy = Ctx.Types.inferScalarType(
this);
1978 Type *VectorTy =
toVectorTy(Ctx.Types.inferScalarType(
this), VF);
1986 const auto [Op1VK, Op1VP] = Ctx.getOperandInfo(Op0);
1987 const auto [Op2VK, Op2VP] = Ctx.getOperandInfo(Op1);
1991 [](
VPValue *
Op) {
return Op->getUnderlyingValue(); }))
1992 Operands.
append(
SI->op_begin(),
SI->op_end());
1994 return Ctx.TTI.getArithmeticInstrCost(
1995 IsLogicalOr ? Instruction::Or : Instruction::And, VectorTy,
1996 Ctx.CostKind, {Op1VK, Op1VP}, {Op2VK, Op2VP}, Operands,
SI);
2005 Pred = Cmp->getPredicate();
2006 return Ctx.TTI.getCmpSelInstrCost(
2007 Instruction::Select, VectorTy, CondTy, Pred, Ctx.CostKind,
2008 {TTI::OK_AnyValue, TTI::OP_None}, {TTI::OK_AnyValue, TTI::OP_None},
SI);
2011VPIRFlags::FastMathFlagsTy::FastMathFlagsTy(
const FastMathFlags &FMF) {
2024 case OperationType::OverflowingBinOp:
2025 return Opcode == Instruction::Add || Opcode == Instruction::Sub ||
2026 Opcode == Instruction::Mul ||
2027 Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart;
2028 case OperationType::Trunc:
2029 return Opcode == Instruction::Trunc;
2030 case OperationType::DisjointOp:
2031 return Opcode == Instruction::Or;
2032 case OperationType::PossiblyExactOp:
2033 return Opcode == Instruction::AShr;
2034 case OperationType::GEPOp:
2035 return Opcode == Instruction::GetElementPtr ||
2038 case OperationType::FPMathOp:
2039 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
2040 Opcode == Instruction::FSub || Opcode == Instruction::FNeg ||
2041 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
2042 Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc ||
2043 Opcode == Instruction::FCmp || Opcode == Instruction::Select ||
2047 case OperationType::NonNegOp:
2048 return Opcode == Instruction::ZExt || Opcode == Instruction::UIToFP;
2049 case OperationType::Cmp:
2050 return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
2051 case OperationType::Other:
2058#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2061 case OperationType::Cmp:
2064 case OperationType::DisjointOp:
2068 case OperationType::PossiblyExactOp:
2072 case OperationType::OverflowingBinOp:
2078 case OperationType::Trunc:
2084 case OperationType::FPMathOp:
2087 case OperationType::GEPOp:
2090 else if (
GEPFlags.hasNoUnsignedSignedWrap())
2095 case OperationType::NonNegOp:
2099 case OperationType::Other:
2107 auto &Builder = State.Builder;
2109 case Instruction::Call:
2110 case Instruction::Br:
2111 case Instruction::PHI:
2112 case Instruction::GetElementPtr:
2113 case Instruction::Select:
2115 case Instruction::UDiv:
2116 case Instruction::SDiv:
2117 case Instruction::SRem:
2118 case Instruction::URem:
2119 case Instruction::Add:
2120 case Instruction::FAdd:
2121 case Instruction::Sub:
2122 case Instruction::FSub:
2123 case Instruction::FNeg:
2124 case Instruction::Mul:
2125 case Instruction::FMul:
2126 case Instruction::FDiv:
2127 case Instruction::FRem:
2128 case Instruction::Shl:
2129 case Instruction::LShr:
2130 case Instruction::AShr:
2131 case Instruction::And:
2132 case Instruction::Or:
2133 case Instruction::Xor: {
2137 Ops.push_back(State.get(VPOp));
2139 Value *V = Builder.CreateNAryOp(Opcode,
Ops);
2150 case Instruction::ExtractValue: {
2154 Value *Extract = Builder.CreateExtractValue(
Op, CI->getZExtValue());
2155 State.set(
this, Extract);
2158 case Instruction::Freeze: {
2160 Value *Freeze = Builder.CreateFreeze(
Op);
2161 State.set(
this, Freeze);
2164 case Instruction::ICmp:
2165 case Instruction::FCmp: {
2167 bool FCmp = Opcode == Instruction::FCmp;
2173 C = Builder.CreateFCmpFMF(
2195 State.get(
this)->getType() &&
2196 "inferred type and type from generated instructions do not match");
2203 case Instruction::UDiv:
2204 case Instruction::SDiv:
2205 case Instruction::SRem:
2206 case Instruction::URem:
2211 case Instruction::FNeg:
2212 case Instruction::Add:
2213 case Instruction::FAdd:
2214 case Instruction::Sub:
2215 case Instruction::FSub:
2216 case Instruction::Mul:
2217 case Instruction::FMul:
2218 case Instruction::FDiv:
2219 case Instruction::FRem:
2220 case Instruction::Shl:
2221 case Instruction::LShr:
2222 case Instruction::AShr:
2223 case Instruction::And:
2224 case Instruction::Or:
2225 case Instruction::Xor:
2226 case Instruction::Freeze:
2227 case Instruction::ExtractValue:
2228 case Instruction::ICmp:
2229 case Instruction::FCmp:
2236#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2239 O << Indent <<
"WIDEN ";
2248 auto &Builder = State.Builder;
2250 assert(State.VF.isVector() &&
"Not vectorizing?");
2255 State.set(
this, Cast);
2279 if (WidenMemoryRecipe ==
nullptr)
2281 if (!WidenMemoryRecipe->isConsecutive())
2283 if (WidenMemoryRecipe->isReverse())
2285 if (WidenMemoryRecipe->isMasked())
2293 if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) &&
2296 CCH = ComputeCCH(StoreRecipe);
2299 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
2300 Opcode == Instruction::FPExt) {
2311 return Ctx.TTI.getCastInstrCost(
2312 Opcode, DestTy, SrcTy, CCH, Ctx.CostKind,
2316#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2319 O << Indent <<
"WIDEN-CAST ";
2330 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
2337 : ConstantFP::get(Ty,
C);
2340#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2345 O <<
" = WIDEN-INDUCTION ";
2349 O <<
" (truncated to " << *TI->getType() <<
")";
2361 auto *CanIV =
getRegion()->getCanonicalIV();
2362 return StartC && StartC->isZero() && StepC && StepC->isOne() &&
2366#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2371 O <<
" = DERIVED-IV ";
2395 assert(BaseIVTy == Step->
getType() &&
"Types of BaseIV and Step must match!");
2402 AddOp = Instruction::Add;
2403 MulOp = Instruction::Mul;
2405 AddOp = InductionOpcode;
2406 MulOp = Instruction::FMul;
2415 Type *VecIVTy =
nullptr;
2416 Value *UnitStepVec =
nullptr, *SplatStep =
nullptr, *SplatIV =
nullptr;
2417 if (!FirstLaneOnly && State.VF.isScalable()) {
2421 SplatStep = Builder.CreateVectorSplat(State.VF, Step);
2422 SplatIV = Builder.CreateVectorSplat(State.VF, BaseIV);
2425 unsigned StartLane = 0;
2426 unsigned EndLane = FirstLaneOnly ? 1 : State.VF.getKnownMinValue();
2428 StartLane = State.Lane->getKnownLane();
2429 EndLane = StartLane + 1;
2433 StartIdx0 = ConstantInt::get(IntStepTy, 0);
2438 Builder.CreateMul(StartIdx0, ConstantInt::get(StartIdx0->
getType(),
2441 StartIdx0 = Builder.CreateSExtOrTrunc(StartIdx0, IntStepTy);
2444 if (!FirstLaneOnly && State.VF.isScalable()) {
2445 auto *SplatStartIdx = Builder.CreateVectorSplat(State.VF, StartIdx0);
2446 auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec);
2448 InitVec = Builder.CreateSIToFP(InitVec, VecIVTy);
2449 auto *
Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep);
2450 auto *
Add = Builder.CreateBinOp(AddOp, SplatIV,
Mul);
2451 State.set(
this,
Add);
2458 StartIdx0 = Builder.CreateSIToFP(StartIdx0, BaseIVTy);
2460 for (
unsigned Lane = StartLane; Lane < EndLane; ++Lane) {
2461 Value *StartIdx = Builder.CreateBinOp(
2466 "Expected StartIdx to be folded to a constant when VF is not "
2468 auto *
Mul = Builder.CreateBinOp(MulOp, StartIdx, Step);
2469 auto *
Add = Builder.CreateBinOp(AddOp, BaseIV,
Mul);
2474#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2479 O <<
" = SCALAR-STEPS ";
2485 assert(State.VF.isVector() &&
"not widening");
2492 if (areAllOperandsInvariant()) {
2512 Value *
Splat = State.Builder.CreateVectorSplat(State.VF, NewGEP);
2513 State.set(
this,
Splat);
2519 auto *
Ptr = State.get(
getOperand(0), isPointerLoopInvariant());
2526 Indices.
push_back(State.get(Operand, isIndexLoopInvariant(
I - 1)));
2533 assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
2534 "NewGEP is not a pointer vector");
2535 State.set(
this, NewGEP);
2539#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2542 O << Indent <<
"WIDEN-GEP ";
2543 O << (isPointerLoopInvariant() ?
"Inv" :
"Var");
2545 O <<
"[" << (isIndexLoopInvariant(
I) ?
"Inv" :
"Var") <<
"]";
2549 O <<
" = getelementptr";
2559 const DataLayout &
DL = Builder.GetInsertBlock()->getDataLayout();
2560 return !IsUnitStride || (IsScalable && (IsReverse || CurrentPart > 0))
2561 ?
DL.getIndexType(Builder.getPtrTy(0))
2562 : Builder.getInt32Ty();
2566 auto &Builder = State.Builder;
2568 bool IsUnitStride = Stride == 1 || Stride == -1;
2570 IsUnitStride, CurrentPart, Builder);
2574 if (IndexTy != RunTimeVF->
getType())
2575 RunTimeVF = Builder.CreateZExtOrTrunc(RunTimeVF, IndexTy);
2577 Value *NumElt = Builder.CreateMul(
2578 ConstantInt::get(IndexTy, Stride * (int64_t)CurrentPart), RunTimeVF);
2580 Value *LastLane = Builder.CreateSub(RunTimeVF, ConstantInt::get(IndexTy, 1));
2582 LastLane = Builder.CreateMul(ConstantInt::get(IndexTy, Stride), LastLane);
2586 ResultPtr = Builder.CreateGEP(IndexedTy, ResultPtr, LastLane,
"",
2589 State.set(
this, ResultPtr,
true);
2592#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2597 O <<
" = vector-end-pointer";
2604 auto &Builder = State.Builder;
2607 true, CurrentPart, Builder);
2614 State.set(
this, ResultPtr,
true);
2617#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2622 O <<
" = vector-pointer ";
2633 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
2635 Type *ResultTy =
toVectorTy(Ctx.Types.inferScalarType(
this), VF);
2638 Ctx.TTI.getCmpSelInstrCost(Instruction::Select, ResultTy, CmpTy,
2642#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2645 O << Indent <<
"BLEND ";
2667 assert(!State.Lane &&
"Reduction being replicated.");
2671 "In-loop AnyOf reductions aren't currently supported");
2677 Value *NewCond = State.get(
Cond, State.VF.isScalar());
2682 if (State.VF.isVector())
2683 Start = State.Builder.CreateVectorSplat(VecTy->
getElementCount(), Start);
2685 Value *
Select = State.Builder.CreateSelect(NewCond, NewVecOp, Start);
2691 if (State.VF.isVector())
2695 NewRed = State.Builder.CreateBinOp(
2697 PrevInChain, NewVecOp);
2698 PrevInChain = NewRed;
2699 NextInChain = NewRed;
2704 NextInChain =
createMinMaxOp(State.Builder, Kind, NewRed, PrevInChain);
2706 NextInChain = State.Builder.CreateBinOp(
2708 PrevInChain, NewRed);
2710 State.set(
this, NextInChain,
true);
2714 assert(!State.Lane &&
"Reduction being replicated.");
2716 auto &Builder = State.Builder;
2728 Mask = State.get(CondOp);
2730 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
2740 NewRed = Builder.CreateBinOp(
2744 State.set(
this, NewRed,
true);
2750 Type *ElementTy = Ctx.Types.inferScalarType(
this);
2754 std::optional<FastMathFlags> OptionalFMF =
2761 "Any-of reduction not implemented in VPlan-based cost model currently.");
2767 return Ctx.TTI.getMinMaxReductionCost(Id, VectorTy,
FMFs, Ctx.CostKind);
2772 return Ctx.TTI.getArithmeticReductionCost(Opcode, VectorTy, OptionalFMF,
2777 ExpressionTypes ExpressionType,
2780 ExpressionRecipes(ExpressionRecipes),
ExpressionType(ExpressionType) {
2781 assert(!ExpressionRecipes.empty() &&
"Nothing to combine?");
2785 "expression cannot contain recipes with side-effects");
2789 for (
auto *R : ExpressionRecipes)
2790 ExpressionRecipesAsSetOfUsers.
insert(R);
2796 if (R != ExpressionRecipes.back() &&
2797 any_of(
R->users(), [&ExpressionRecipesAsSetOfUsers](
VPUser *U) {
2798 return !ExpressionRecipesAsSetOfUsers.contains(U);
2803 R->replaceUsesWithIf(CopyForExtUsers, [&ExpressionRecipesAsSetOfUsers](
2805 return !ExpressionRecipesAsSetOfUsers.contains(&U);
2810 R->removeFromParent();
2817 for (
auto *R : ExpressionRecipes) {
2818 for (
const auto &[Idx,
Op] :
enumerate(
R->operands())) {
2819 auto *
Def =
Op->getDefiningRecipe();
2820 if (Def && ExpressionRecipesAsSetOfUsers.contains(Def))
2823 LiveInPlaceholders.push_back(
new VPValue());
2829 for (
auto *R : ExpressionRecipes)
2830 for (
auto const &[LiveIn, Tmp] :
zip(operands(), LiveInPlaceholders))
2831 R->replaceUsesOfWith(LiveIn, Tmp);
2835 for (
auto *R : ExpressionRecipes)
2838 if (!R->getParent())
2839 R->insertBefore(
this);
2842 LiveInPlaceholders[Idx]->replaceAllUsesWith(
Op);
2845 ExpressionRecipes.clear();
2850 Type *RedTy = Ctx.Types.inferScalarType(
this);
2854 "VPExpressionRecipe only supports integer types currently.");
2857 switch (ExpressionType) {
2858 case ExpressionTypes::ExtendedReduction: {
2859 return Ctx.TTI.getExtendedReductionCost(
2863 RedTy, SrcVecTy, std::nullopt, Ctx.CostKind);
2865 case ExpressionTypes::MulAccReduction:
2866 return Ctx.TTI.getMulAccReductionCost(
false, Opcode, RedTy, SrcVecTy,
2869 case ExpressionTypes::ExtNegatedMulAccReduction:
2870 assert(Opcode == Instruction::Add &&
"Unexpected opcode");
2871 Opcode = Instruction::Sub;
2873 case ExpressionTypes::ExtMulAccReduction: {
2874 return Ctx.TTI.getMulAccReductionCost(
2877 Opcode, RedTy, SrcVecTy, Ctx.CostKind);
2885 return R->mayReadFromMemory() || R->mayWriteToMemory();
2893 "expression cannot contain recipes with side-effects");
2904#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2908 O << Indent <<
"EXPRESSION ";
2914 switch (ExpressionType) {
2915 case ExpressionTypes::ExtendedReduction: {
2924 << *Ext0->getResultType();
2925 if (Red->isConditional()) {
2932 case ExpressionTypes::ExtNegatedMulAccReduction: {
2944 << *Ext0->getResultType() <<
"), (";
2948 << *Ext1->getResultType() <<
")";
2949 if (Red->isConditional()) {
2956 case ExpressionTypes::MulAccReduction:
2957 case ExpressionTypes::ExtMulAccReduction: {
2965 bool IsExtended = ExpressionType == ExpressionTypes::ExtMulAccReduction;
2967 : ExpressionRecipes[0]);
2975 << *Ext0->getResultType() <<
"), (";
2983 << *Ext1->getResultType() <<
")";
2985 if (Red->isConditional()) {
2997 O << Indent <<
"REDUCE ";
3017 O << Indent <<
"REDUCE ";
3045 assert((!Instr->getType()->isAggregateType() ||
3047 "Expected vectorizable or non-aggregate type.");
3050 bool IsVoidRetTy = Instr->getType()->isVoidTy();
3054 Cloned->
setName(Instr->getName() +
".cloned");
3055 Type *ResultTy = State.TypeAnalysis.inferScalarType(RepRecipe);
3059 if (ResultTy != Cloned->
getType())
3070 State.setDebugLocFrom(
DL);
3075 auto InputLane = Lane;
3079 Cloned->
setOperand(
I.index(), State.get(Operand, InputLane));
3083 State.Builder.Insert(Cloned);
3085 State.set(RepRecipe, Cloned, Lane);
3089 State.AC->registerAssumption(
II);
3095 [](
VPValue *
Op) { return Op->isDefinedOutsideLoopRegions(); })) &&
3096 "Expected a recipe is either within a region or all of its operands "
3097 "are defined outside the vectorized region.");
3104 assert(IsSingleScalar &&
"VPReplicateRecipes outside replicate regions "
3105 "must have already been unrolled");
3111 "uniform recipe shouldn't be predicated");
3112 assert(!State.VF.isScalable() &&
"Can't scalarize a scalable vector");
3117 State.Lane->isFirstLane()
3120 State.set(
this, State.packScalarIntoVectorizedValue(
this, WideValue,
3138 auto *PtrR =
Ptr->getDefiningRecipe();
3141 Instruction::GetElementPtr) ||
3149 if (!Opd->isDefinedOutsideLoopRegions() &&
3163 while (!WorkList.
empty()) {
3165 if (!Cur || !Seen.
insert(Cur).second)
3173 return Seen.contains(
3174 Blend->getIncomingValue(I)->getDefiningRecipe());
3178 for (
VPUser *U : Cur->users()) {
3180 if (InterleaveR->getAddr() == Cur)
3183 if (RepR->getOpcode() == Instruction::Load &&
3184 RepR->getOperand(0) == Cur)
3186 if (RepR->getOpcode() == Instruction::Store &&
3187 RepR->getOperand(1) == Cur)
3191 if (MemR->getAddr() == Cur && MemR->isConsecutive())
3212 Ctx.SkipCostComputation.insert(UI);
3218 case Instruction::GetElementPtr:
3224 case Instruction::Call: {
3230 for (
const VPValue *ArgOp : ArgOps)
3231 Tys.
push_back(Ctx.Types.inferScalarType(ArgOp));
3233 if (CalledFn->isIntrinsic())
3236 switch (CalledFn->getIntrinsicID()) {
3237 case Intrinsic::assume:
3238 case Intrinsic::lifetime_end:
3239 case Intrinsic::lifetime_start:
3240 case Intrinsic::sideeffect:
3241 case Intrinsic::pseudoprobe:
3242 case Intrinsic::experimental_noalias_scope_decl: {
3245 "scalarizing intrinsic should be free");
3252 Type *ResultTy = Ctx.Types.inferScalarType(
this);
3254 Ctx.TTI.getCallInstrCost(CalledFn, ResultTy, Tys, Ctx.CostKind);
3256 if (CalledFn->isIntrinsic())
3257 ScalarCallCost = std::min(
3261 return ScalarCallCost;
3265 Ctx.getScalarizationOverhead(ResultTy, ArgOps, VF);
3267 case Instruction::Add:
3268 case Instruction::Sub:
3269 case Instruction::FAdd:
3270 case Instruction::FSub:
3271 case Instruction::Mul:
3272 case Instruction::FMul:
3273 case Instruction::FDiv:
3274 case Instruction::FRem:
3275 case Instruction::Shl:
3276 case Instruction::LShr:
3277 case Instruction::AShr:
3278 case Instruction::And:
3279 case Instruction::Or:
3280 case Instruction::Xor:
3281 case Instruction::ICmp:
3282 case Instruction::FCmp:
3286 case Instruction::SDiv:
3287 case Instruction::UDiv:
3288 case Instruction::SRem:
3289 case Instruction::URem: {
3296 Ctx.getScalarizationOverhead(Ctx.Types.inferScalarType(
this),
3305 Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
3312 case Instruction::Load:
3313 case Instruction::Store: {
3320 bool IsLoad = UI->
getOpcode() == Instruction::Load;
3327 Type *ValTy = Ctx.Types.inferScalarType(IsLoad ?
this :
getOperand(0));
3328 Type *ScalarPtrTy = Ctx.Types.inferScalarType(PtrOp);
3333 UI->
getOpcode(), ValTy, Alignment, AS, Ctx.CostKind, OpInfo);
3336 bool PreferVectorizedAddressing = Ctx.TTI.prefersVectorizedAddressing();
3337 bool UsedByLoadStoreAddress =
3340 ScalarMemOpCost + Ctx.TTI.getAddressComputationCost(
3341 PtrTy, UsedByLoadStoreAddress ?
nullptr : &Ctx.SE,
3342 nullptr, Ctx.CostKind);
3352 if (!UsedByLoadStoreAddress) {
3353 bool EfficientVectorLoadStore =
3354 Ctx.TTI.supportsEfficientVectorElementLoadStore();
3355 if (!(IsLoad && !PreferVectorizedAddressing) &&
3356 !(!IsLoad && EfficientVectorLoadStore))
3359 if (!EfficientVectorLoadStore)
3360 ResultTy = Ctx.Types.inferScalarType(
this);
3364 Ctx.getScalarizationOverhead(ResultTy, OpsToScalarize, VF,
true);
3368 return Ctx.getLegacyCost(UI, VF);
3371#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3374 O << Indent << (IsSingleScalar ?
"CLONE " :
"REPLICATE ");
3383 O <<
"@" << CB->getCalledFunction()->getName() <<
"(";
3401 assert(State.Lane &&
"Branch on Mask works only on single instance.");
3404 Value *ConditionBit = State.get(BlockInMask, *State.Lane);
3408 auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();
3410 "Expected to replace unreachable terminator with conditional branch.");
3412 State.Builder.CreateCondBr(ConditionBit, State.CFG.PrevBB,
nullptr);
3413 CondBr->setSuccessor(0,
nullptr);
3414 CurrentTerminator->eraseFromParent();
3426 assert(State.Lane &&
"Predicated instruction PHI works per instance.");
3431 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
3433 "operand must be VPReplicateRecipe");
3444 "Packed operands must generate an insertelement or insertvalue");
3452 for (
unsigned I = 0;
I < StructTy->getNumContainedTypes() - 1;
I++)
3455 PHINode *VPhi = State.Builder.CreatePHI(VecI->getType(), 2);
3456 VPhi->
addIncoming(VecI->getOperand(0), PredicatingBB);
3458 if (State.hasVectorValue(
this))
3459 State.reset(
this, VPhi);
3461 State.set(
this, VPhi);
3469 Type *PredInstType = State.TypeAnalysis.inferScalarType(
getOperand(0));
3470 PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
3473 Phi->addIncoming(ScalarPredInst, PredicatedBB);
3474 if (State.hasScalarValue(
this, *State.Lane))
3475 State.reset(
this, Phi, *State.Lane);
3477 State.set(
this, Phi, *State.Lane);
3480 State.reset(
getOperand(0), Phi, *State.Lane);
3484#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3487 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
3499 ->getAddressSpace();
3502 : Instruction::Store;
3509 "Inconsecutive memory access should not have the order.");
3519 return Ctx.TTI.getAddressComputationCost(PtrTy,
nullptr,
nullptr,
3521 Ctx.TTI.getGatherScatterOpCost(Opcode, Ty,
Ptr,
IsMasked, Alignment,
3528 Ctx.TTI.getMaskedMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind);
3533 Cost += Ctx.TTI.getMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind,
3539 return Cost += Ctx.TTI.getShuffleCost(
3550 auto &Builder = State.Builder;
3551 Value *Mask =
nullptr;
3552 if (
auto *VPMask =
getMask()) {
3555 Mask = State.get(VPMask);
3557 Mask = Builder.CreateVectorReverse(Mask,
"reverse");
3563 NewLI = Builder.CreateMaskedGather(DataTy, Addr, Alignment, Mask,
nullptr,
3564 "wide.masked.gather");
3567 Builder.CreateMaskedLoad(DataTy, Addr, Alignment, Mask,
3570 NewLI = Builder.CreateAlignedLoad(DataTy, Addr, Alignment,
"wide.load");
3574 NewLI = Builder.CreateVectorReverse(NewLI,
"reverse");
3575 State.set(
this, NewLI);
3578#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3581 O << Indent <<
"WIDEN ";
3593 Value *AllTrueMask =
3594 Builder.CreateVectorSplat(ValTy->getElementCount(), Builder.getTrue());
3595 return Builder.CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,
3596 {Operand, AllTrueMask, EVL},
nullptr, Name);
3605 auto &Builder = State.Builder;
3609 Value *Mask =
nullptr;
3611 Mask = State.get(VPMask);
3615 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
3620 Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},
3621 nullptr,
"wide.masked.gather");
3623 NewLI = Builder.CreateIntrinsic(DataTy, Intrinsic::vp_load,
3624 {Addr, Mask, EVL},
nullptr,
"vp.op.load");
3632 State.set(
this, Res);
3649 Instruction::Load, Ty, Alignment, AS, Ctx.CostKind);
3653 return Cost + Ctx.TTI.getShuffleCost(
3658#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3661 O << Indent <<
"WIDEN ";
3673 auto &Builder = State.Builder;
3675 Value *Mask =
nullptr;
3676 if (
auto *VPMask =
getMask()) {
3679 Mask = State.get(VPMask);
3681 Mask = Builder.CreateVectorReverse(Mask,
"reverse");
3684 Value *StoredVal = State.get(StoredVPValue);
3688 StoredVal = Builder.CreateVectorReverse(StoredVal,
"reverse");
3695 NewSI = Builder.CreateMaskedScatter(StoredVal, Addr, Alignment, Mask);
3697 NewSI = Builder.CreateMaskedStore(StoredVal, Addr, Alignment, Mask);
3699 NewSI = Builder.CreateAlignedStore(StoredVal, Addr, Alignment);
3703#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3706 O << Indent <<
"WIDEN store ";
3716 auto &Builder = State.Builder;
3719 Value *StoredVal = State.get(StoredValue);
3723 Value *Mask =
nullptr;
3725 Mask = State.get(VPMask);
3729 Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
3732 if (CreateScatter) {
3734 Intrinsic::vp_scatter,
3735 {StoredVal, Addr, Mask, EVL});
3738 Intrinsic::vp_store,
3739 {StoredVal, Addr, Mask, EVL});
3760 Instruction::Store, Ty, Alignment, AS, Ctx.CostKind);
3764 return Cost + Ctx.TTI.getShuffleCost(
3769#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3772 O << Indent <<
"WIDEN vp.store ";
3780 auto VF = DstVTy->getElementCount();
3782 assert(VF == SrcVecTy->getElementCount() &&
"Vector dimensions do not match");
3783 Type *SrcElemTy = SrcVecTy->getElementType();
3784 Type *DstElemTy = DstVTy->getElementType();
3785 assert((
DL.getTypeSizeInBits(SrcElemTy) ==
DL.getTypeSizeInBits(DstElemTy)) &&
3786 "Vector elements must have same size");
3790 return Builder.CreateBitOrPointerCast(V, DstVTy);
3797 "Only one type should be a pointer type");
3799 "Only one type should be a floating point type");
3803 Value *CastVal = Builder.CreateBitOrPointerCast(V, VecIntTy);
3804 return Builder.CreateBitOrPointerCast(CastVal, DstVTy);
3810 const Twine &Name) {
3811 unsigned Factor = Vals.
size();
3812 assert(Factor > 1 &&
"Tried to interleave invalid number of vectors");
3816 for (
Value *Val : Vals)
3817 assert(Val->getType() == VecTy &&
"Tried to interleave mismatched types");
3822 if (VecTy->isScalableTy()) {
3823 assert(Factor <= 8 &&
"Unsupported interleave factor for scalable vectors");
3824 return Builder.CreateVectorInterleave(Vals, Name);
3831 const unsigned NumElts = VecTy->getElementCount().getFixedValue();
3832 return Builder.CreateShuffleVector(
3865 assert(!State.Lane &&
"Interleave group being replicated.");
3867 "Masking gaps for scalable vectors is not yet supported.");
3873 unsigned InterleaveFactor = Group->
getFactor();
3880 auto CreateGroupMask = [&BlockInMask, &State,
3881 &InterleaveFactor](
Value *MaskForGaps) ->
Value * {
3882 if (State.VF.isScalable()) {
3883 assert(!MaskForGaps &&
"Interleaved groups with gaps are not supported.");
3884 assert(InterleaveFactor <= 8 &&
3885 "Unsupported deinterleave factor for scalable vectors");
3886 auto *ResBlockInMask = State.get(BlockInMask);
3894 Value *ResBlockInMask = State.get(BlockInMask);
3895 Value *ShuffledMask = State.Builder.CreateShuffleVector(
3898 "interleaved.mask");
3899 return MaskForGaps ? State.Builder.CreateBinOp(Instruction::And,
3900 ShuffledMask, MaskForGaps)
3904 const DataLayout &DL = Instr->getDataLayout();
3907 Value *MaskForGaps =
nullptr;
3911 assert(MaskForGaps &&
"Mask for Gaps is required but it is null");
3915 if (BlockInMask || MaskForGaps) {
3916 Value *GroupMask = CreateGroupMask(MaskForGaps);
3918 NewLoad = State.Builder.CreateMaskedLoad(VecTy, ResAddr,
3920 PoisonVec,
"wide.masked.vec");
3922 NewLoad = State.Builder.CreateAlignedLoad(VecTy, ResAddr,
3929 if (VecTy->isScalableTy()) {
3932 assert(InterleaveFactor <= 8 &&
3933 "Unsupported deinterleave factor for scalable vectors");
3934 NewLoad = State.Builder.CreateIntrinsic(
3937 nullptr,
"strided.vec");
3940 auto CreateStridedVector = [&InterleaveFactor, &State,
3941 &NewLoad](
unsigned Index) ->
Value * {
3942 assert(Index < InterleaveFactor &&
"Illegal group index");
3943 if (State.VF.isScalable())
3944 return State.Builder.CreateExtractValue(NewLoad, Index);
3950 return State.Builder.CreateShuffleVector(NewLoad, StrideMask,
3954 for (
unsigned I = 0, J = 0;
I < InterleaveFactor; ++
I) {
3961 Value *StridedVec = CreateStridedVector(
I);
3964 if (Member->getType() != ScalarTy) {
3971 StridedVec = State.Builder.CreateVectorReverse(StridedVec,
"reverse");
3973 State.set(VPDefs[J], StridedVec);
3983 Value *MaskForGaps =
3986 "Mismatch between NeedsMaskForGaps and MaskForGaps");
3990 unsigned StoredIdx = 0;
3991 for (
unsigned i = 0; i < InterleaveFactor; i++) {
3993 "Fail to get a member from an interleaved store group");
4003 Value *StoredVec = State.get(StoredValues[StoredIdx]);
4007 StoredVec = State.Builder.CreateVectorReverse(StoredVec,
"reverse");
4011 if (StoredVec->
getType() != SubVT)
4020 if (BlockInMask || MaskForGaps) {
4021 Value *GroupMask = CreateGroupMask(MaskForGaps);
4022 NewStoreInstr = State.Builder.CreateMaskedStore(
4023 IVec, ResAddr, Group->
getAlign(), GroupMask);
4026 State.Builder.CreateAlignedStore(IVec, ResAddr, Group->
getAlign());
4033#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4037 O << Indent <<
"INTERLEAVE-GROUP with factor " << IG->getFactor() <<
" at ";
4038 IG->getInsertPos()->printAsOperand(O,
false);
4048 for (
unsigned i = 0; i < IG->getFactor(); ++i) {
4049 if (!IG->getMember(i))
4052 O <<
"\n" << Indent <<
" store ";
4054 O <<
" to index " << i;
4056 O <<
"\n" << Indent <<
" ";
4058 O <<
" = load from index " << i;
4066 assert(!State.Lane &&
"Interleave group being replicated.");
4067 assert(State.VF.isScalable() &&
4068 "Only support scalable VF for EVL tail-folding.");
4070 "Masking gaps for scalable vectors is not yet supported.");
4076 unsigned InterleaveFactor = Group->
getFactor();
4077 assert(InterleaveFactor <= 8 &&
4078 "Unsupported deinterleave/interleave factor for scalable vectors");
4085 Value *InterleaveEVL = State.Builder.CreateMul(
4086 EVL, ConstantInt::get(EVL->
getType(), InterleaveFactor),
"interleave.evl",
4090 Value *GroupMask =
nullptr;
4096 State.Builder.CreateVectorSplat(WideVF, State.Builder.getTrue());
4101 CallInst *NewLoad = State.Builder.CreateIntrinsic(
4102 VecTy, Intrinsic::vp_load, {ResAddr, GroupMask, InterleaveEVL},
nullptr,
4113 NewLoad = State.Builder.CreateIntrinsic(
4116 nullptr,
"strided.vec");
4118 const DataLayout &DL = Instr->getDataLayout();
4119 for (
unsigned I = 0, J = 0;
I < InterleaveFactor; ++
I) {
4125 Value *StridedVec = State.Builder.CreateExtractValue(NewLoad,
I);
4127 if (Member->getType() != ScalarTy) {
4145 const DataLayout &DL = Instr->getDataLayout();
4146 for (
unsigned I = 0, StoredIdx = 0;
I < InterleaveFactor;
I++) {
4154 Value *StoredVec = State.get(StoredValues[StoredIdx]);
4156 if (StoredVec->
getType() != SubVT)
4166 State.Builder.CreateIntrinsic(
Type::getVoidTy(Ctx), Intrinsic::vp_store,
4167 {IVec, ResAddr, GroupMask, InterleaveEVL});
4176#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4180 O << Indent <<
"INTERLEAVE-GROUP with factor " << IG->getFactor() <<
" at ";
4181 IG->getInsertPos()->printAsOperand(O,
false);
4192 for (
unsigned i = 0; i < IG->getFactor(); ++i) {
4193 if (!IG->getMember(i))
4196 O <<
"\n" << Indent <<
" vp.store ";
4198 O <<
" to index " << i;
4200 O <<
"\n" << Indent <<
" ";
4202 O <<
" = vp.load from index " << i;
4213 unsigned InsertPosIdx = 0;
4214 for (
unsigned Idx = 0; IG->getFactor(); ++Idx)
4215 if (
auto *Member = IG->getMember(Idx)) {
4216 if (Member == InsertPos)
4220 Type *ValTy = Ctx.Types.inferScalarType(
4226 unsigned InterleaveFactor = IG->getFactor();
4231 for (
unsigned IF = 0; IF < InterleaveFactor; IF++)
4232 if (IG->getMember(IF))
4237 InsertPos->
getOpcode(), WideVecTy, IG->getFactor(), Indices,
4238 IG->getAlign(), AS, Ctx.CostKind,
getMask(), NeedsMaskForGaps);
4240 if (!IG->isReverse())
4243 return Cost + IG->getNumMembers() *
4245 VectorTy, VectorTy, {}, Ctx.CostKind,
4249#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4252 O << Indent <<
"EMIT ";
4254 O <<
" = CANONICAL-INDUCTION ";
4260 return IsScalarAfterVectorization &&
4264#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4268 "unexpected number of operands");
4269 O << Indent <<
"EMIT ";
4271 O <<
" = WIDEN-POINTER-INDUCTION ";
4287 O << Indent <<
"EMIT ";
4289 O <<
" = EXPAND SCEV " << *Expr;
4296 IRBuilder<> Builder(State.CFG.PrevBB->getTerminator());
4300 : Builder.CreateVectorSplat(VF, CanonicalIV,
"broadcast");
4303 VStep = Builder.CreateVectorSplat(VF, VStep);
4305 Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->
getType()));
4307 Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep,
"vec.iv");
4308 State.set(
this, CanonicalVectorIV);
4311#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4314 O << Indent <<
"EMIT ";
4316 O <<
" = WIDEN-CANONICAL-INDUCTION ";
4322 auto &Builder = State.Builder;
4326 Type *VecTy = State.VF.isScalar()
4327 ? VectorInit->getType()
4331 State.CFG.VPBB2IRBB.at(
getParent()->getCFGPredecessor(0));
4332 if (State.VF.isVector()) {
4334 auto *One = ConstantInt::get(IdxTy, 1);
4337 auto *RuntimeVF =
getRuntimeVF(Builder, IdxTy, State.VF);
4338 auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
4339 VectorInit = Builder.CreateInsertElement(
4345 Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
4346 Phi->addIncoming(VectorInit, VectorPH);
4347 State.set(
this, Phi);
4354 return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
4359#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4362 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
4379 State.CFG.VPBB2IRBB.at(
getParent()->getCFGPredecessor(0));
4380 bool ScalarPHI = State.VF.isScalar() || IsInLoop;
4381 Value *StartV = State.get(StartVPV, ScalarPHI);
4385 assert(State.CurrentParentLoop->getHeader() == HeaderBB &&
4386 "recipe must be in the vector loop header");
4389 State.set(
this, Phi, IsInLoop);
4391 Phi->addIncoming(StartV, VectorPH);
4394#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4397 O << Indent <<
"WIDEN-REDUCTION-PHI ";
4402 if (VFScaleFactor != 1)
4403 O <<
" (VF scaled by 1/" << VFScaleFactor <<
")";
4410 Instruction *VecPhi = State.Builder.CreatePHI(VecTy, 2, Name);
4411 State.set(
this, VecPhi);
4414#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4417 O << Indent <<
"WIDEN-PHI ";
4429 State.CFG.VPBB2IRBB.at(
getParent()->getCFGPredecessor(0));
4432 State.Builder.CreatePHI(StartMask->
getType(), 2,
"active.lane.mask");
4433 Phi->addIncoming(StartMask, VectorPH);
4434 State.set(
this, Phi);
4437#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4440 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
4448#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4451 O << Indent <<
"EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::pair< Value *, APInt > getMask(Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
This file provides a LoopVectorizationPlanner class.
static bool isOrdered(const Instruction *I)
MachineInstr unsigned OpIdx
uint64_t IntrinsicInst * II
const SmallVectorImpl< MachineOperand > & Cond
This file defines the SmallVector class.
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static SymbolRef::Type getType(const Symbol *Sym)
This file contains the declarations of different VPlan-related auxiliary helpers.
static Instruction * createReverseEVL(IRBuilderBase &Builder, Value *Operand, Value *EVL, const Twine &Name)
Use all-true mask for reverse rather than actual mask, as it avoids a dependence w/o affecting the re...
static Value * interleaveVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name)
Return a vector containing interleaved elements from multiple smaller input vectors.
static InstructionCost getCostForIntrinsics(Intrinsic::ID ID, ArrayRef< const VPValue * > Operands, const VPRecipeWithIRFlags &R, ElementCount VF, VPCostContext &Ctx)
Compute the cost for the intrinsic ID with Operands, produced by R.
static Value * createBitOrPointerCast(IRBuilderBase &Builder, Value *V, VectorType *DstVTy, const DataLayout &DL)
static Type * getGEPIndexTy(bool IsScalable, bool IsReverse, bool IsUnitStride, unsigned CurrentPart, IRBuilderBase &Builder)
SmallVector< Value *, 2 > VectorParts
static bool isUsedByLoadStoreAddress(const VPUser *V)
Returns true if V is used as part of the address of another load or store.
static void scalarizeInstruction(const Instruction *Instr, VPReplicateRecipe *RepRecipe, const VPLane &Lane, VPTransformState &State)
A helper function to scalarize a single Instruction in the innermost loop.
static bool shouldUseAddressAccessSCEV(const VPValue *Ptr)
Returns true if Ptr is a pointer computation for which the legacy cost model computes a SCEV expressi...
static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)
A helper function that returns an integer or floating-point constant with value C.
static BranchInst * createCondBranch(Value *Cond, VPBasicBlock *VPBB, VPTransformState &State)
Create a conditional branch using Cond branching to the successors of VPBB.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
Class for arbitrary precision integers.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
static LLVM_ABI Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
LLVM Basic Block Representation.
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const
Returns an iterator to the first instruction in this block that is not a PHINode instruction.
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Conditional or Unconditional Branch instruction.
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static LLVM_ABI bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
static LLVM_ABI StringRef getPredicateName(Predicate P)
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
constexpr bool isVector() const
One or more elements.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
LLVM_ABI void print(raw_ostream &O) const
Print fast-math flags to O.
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
bool willReturn() const
Determine if the function will return.
bool doesNotThrow() const
Determine if the function cannot unwind.
Type * getReturnType() const
Returns the type of the ret val.
Common base class shared among various IRBuilders.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LLVM_ABI Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateFreeze(Value *V, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
LLVM_ABI CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="", Instruction *MDFrom=nullptr)
LLVM_ABI CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateCountTrailingZeroElems(Type *ResTy, Value *Mask, bool ZeroIsPoison=true, const Twine &Name="")
Create a call to llvm.experimental_cttz_elts.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
LLVMContext & getContext() const
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getFalse()
Get the constant value for i1 false.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="", bool IsDisjoint=false)
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
void addMetadata(InstTy *NewInst) const
Add metadata (e.g.
This is an important class for using LLVM in a threaded context.
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
std::pair< MDNode *, MDNode * > getNoAliasMetadataFor(const Instruction *OrigInst) const
Returns a pair containing the alias_scope and noalias metadata nodes for OrigInst,...
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
static bool isSignedRecurrenceKind(RecurKind Kind)
Returns true if recurrece kind is a signed redux kind.
static LLVM_ABI unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
unsigned getOpcode() const
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isFindLastIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isFindIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class represents the LLVM 'select' instruction.
This class provides computation of slot numbers for LLVM Assembly writing.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
reference emplace_back(ArgTypes &&... Args)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI IntegerType * getInt64Ty(LLVMContext &C)
bool isVectorTy() const
True if this is an instance of VectorType.
static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)
bool isPointerTy() const
True if this is an instance of PointerType.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
LLVM_ABI unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static LLVM_ABI IntegerType * getInt1Ty(LLVMContext &C)
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isVoidTy() const
Return true if this is 'void'.
value_op_iterator value_op_end()
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
value_op_iterator value_op_begin()
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenMemoryRecipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account when normalized the first incoming value wi...
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
const VPBlocksTy & getPredecessors() const
void printAsOperand(raw_ostream &OS, bool PrintType=false) const
const VPBlocksTy & getSuccessors() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPBranchOnMaskRecipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
VPlan-based builder utility analogous to IRBuilder.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
void dump() const
Dump the VPDef to stderr (for debugging).
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
unsigned getVPDefID() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPValue * getStartValue() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void decompose()
Insert the recipes of the expression back into the VPlan, directly before the current recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isSingleScalar() const
Returns true if the result of this VPExpressionRecipe is a single-scalar.
bool mayHaveSideEffects() const
Returns true if this expression contains recipes that may have side effects.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...
bool mayReadOrWriteMemory() const
Returns true if this expression contains recipes that may read from or write to memory.
void execute(VPTransformState &State) override
Produce a vectorized histogram operation.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPHistogramRecipe.
VPValue * getMask() const
Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Class to record and manage LLVM IR flags.
bool flagsValidForOpcode(unsigned Opcode) const
Returns true if the set flags are valid for Opcode.
CmpInst::Predicate CmpPredicate
void printFlags(raw_ostream &O) const
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
LLVM_ABI_FOR_TEST FastMathFlags getFastMathFlags() const
CmpInst::Predicate getPredicate() const
bool hasNoSignedWrap() const
void intersectFlags(const VPIRFlags &Other)
Only keep flags also present in Other.
GEPNoWrapFlags getGEPNoWrapFlags() const
bool hasPredicate() const
Returns true if the recipe has a comparison predicate.
DisjointFlagsTy DisjointFlags
bool hasNoUnsignedWrap() const
NonNegFlagsTy NonNegFlags
void applyFlags(Instruction &I) const
Apply the IR flags to I.
Instruction & getInstruction() const
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void extractLastLaneOfFirstOperand(VPBuilder &Builder)
Update the recipes first operand to the last lane of the operand using Builder.
LLVM_ABI_FOR_TEST InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPIRInstruction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIRInstruction(Instruction &I)
VPIRInstruction::create() should be used to create VPIRInstructions, as subclasses may need to be cre...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the instruction.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPInstruction.
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown(), const Twine &Name="")
bool doesGeneratePerAllLanes() const
Returns true if this VPInstruction generates scalar values for all lanes.
@ ExtractLane
Extracts a single lane (first operand) from a set of vector operands.
@ ComputeAnyOfResult
Compute the final result of a AnyOf reduction with select(cmp(),x,y), where one of (x,...
@ WideIVStep
Scale the first operand (vector step) by the second operand (scalar-step).
@ ExtractPenultimateElement
@ ResumeForEpilogue
Explicit user for the resume phi of the canonical induction in the main VPlan, used by the epilogue v...
@ Unpack
Extracts all lanes from its (non-scalable) vector operand.
@ FirstOrderRecurrenceSplice
@ ReductionStartVector
Start vector for reductions with 3 operands: the original start value, the identity value for the red...
@ BuildVector
Creates a fixed-width vector containing all operands.
@ BuildStructVector
Given operands of (the same) struct type, creates a struct of fixed- width vectors each containing a ...
@ VScale
Returns the value for vscale.
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
bool opcodeMayReadOrWriteFromMemory() const
Returns true if the underlying opcode may read from or write to memory.
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
StringRef getName() const
Returns the symbolic name assigned to the VPInstruction.
unsigned getOpcode() const
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
bool isVectorToScalar() const
Returns true if this VPInstruction produces a scalar value from a vector, e.g.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
bool isSingleScalar() const
Returns true if this VPInstruction's operands are single scalars and the result is also a single scal...
void execute(VPTransformState &State) override
Generate the instruction.
bool needsMaskForGaps() const
Return true if the access needs a mask because of the gaps.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this recipe.
Instruction * getInsertPos() const
const InterleaveGroup< Instruction > * getInterleaveGroup() const
VPValue * getMask() const
Return the mask used by this recipe.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
VPValue * getAddr() const
Return the address accessed by this recipe.
VPValue * getEVL() const
The VPValue of the explicit vector length.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getNumStoreOperands() const override
Returns the number of stored operands of this interleave group.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
unsigned getNumStoreOperands() const override
Returns the number of stored operands of this interleave group.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)
static VPLane getFirstLane()
void execute(VPTransformState &State) override
Generate the reduction in the loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPPartialReductionRecipe.
unsigned getOpcode() const
Get the binary op's opcode.
virtual const VPRecipeBase * getAsRecipe() const =0
Return a VPRecipeBase* to the current object.
virtual unsigned getNumIncoming() const
Returns the number of incoming values, also number of incoming blocks.
void removeIncomingValueFor(VPBlockBase *IncomingBlock) const
Removes the incoming value for IncomingBlock, which must be a predecessor.
const VPBasicBlock * getIncomingBlock(unsigned Idx) const
Returns the incoming block with index Idx.
detail::zippy< llvm::detail::zip_first, VPUser::const_operand_range, const_incoming_blocks_range > incoming_values_and_blocks() const
Returns an iterator range over pairs of incoming values and corresponding incoming blocks.
VPValue * getIncomingValue(unsigned Idx) const
Returns the incoming VPValue with index Idx.
void printPhiOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the recipe.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs (from a replicate region) as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
VPRegionBlock * getRegion()
bool isPhi() const
Returns true for PHI-like recipes.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
virtual InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const
Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this recipe, taking into account if the cost computation should be skipped and the...
bool isScalarCast() const
Return true if the recipe is a scalar cast.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPValue * getEVL() const
The VPValue of the explicit vector length.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool isConditional() const
Return true if the in-loop reduction is conditional.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of VPReductionRecipe.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
RecurKind getRecurrenceKind() const
Return the recurrence kind for the in-loop reduction.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
bool isSingleScalar() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPReplicateRecipe.
unsigned getOpcode() const
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
LLVM_DUMP_METHOD void dump() const
Print this VPSingleDefRecipe to dbgs() (for debugging).
VPSingleDefRecipe(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
This class can be used to assign names to VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
Helper to access the operand that contains the unroll part for this recipe after unrolling.
VPValue * getUnrollPartOperand(const VPUser &U) const
Return the VPValue operand containing the unroll part or null if there is no such operand.
unsigned getUnrollPart(const VPUser &U) const
Return the unroll part.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
void setOperand(unsigned I, VPValue *New)
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
virtual bool onlyFirstLaneUsed(const VPValue *Op) const
Returns true if the VPUser only uses the first lane of operand Op.
bool isDefinedOutsideLoopRegions() const
Returns true if the VPValue is defined outside any loop.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
friend class VPExpressionRecipe
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
bool hasMoreThanOneUniqueUser() const
Returns true if the value has more than one unique user.
Value * getLiveInIRValue() const
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
VPValue(const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr)
void replaceAllUsesWith(VPValue *New)
user_iterator user_begin()
unsigned getNumUsers() const
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getSourceElementType() const
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Function * getCalledScalarFunction() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCallRecipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCastRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
Type * getSourceElementType() const
VPValue * getStepValue()
Returns the step value of the induction.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Type * getScalarType() const
Returns the scalar type of the induction.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
Intrinsic::ID getVectorIntrinsicID() const
Return the ID of the intrinsic.
StringRef getIntrinsicName() const
Return to name of the intrinsic as string.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Return the scalar return type of the intrinsic.
void execute(VPTransformState &State) override
Produce a widened version of the vector intrinsic.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this vector intrinsic.
bool IsMasked
Whether the memory access is masked.
bool Reverse
Whether the consecutive accessed addresses are in reverse order.
bool isConsecutive() const
Return whether the loaded-from / stored-to addresses are consecutive.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenMemoryRecipe.
bool Consecutive
Whether the accessed addresses are consecutive.
VPValue * getMask() const
Return the mask used by this recipe.
VPValue * getAddr() const
Return the address accessed by this recipe.
bool isReverse() const
Return whether the consecutive loaded/stored addresses are in reverse order.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenRecipe.
void execute(VPTransformState &State) override
Produce a widened instruction using the opcode and operands of the recipe, processing State....
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
LLVM_ABI_FOR_TEST VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
LLVM_ABI LLVMContext & getContext() const
All values hold a context through their type.
void mutateType(Type *Ty)
Mutate the type of this Value to be of the specified type.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
const ParentTy * getParent() const
self_iterator getIterator()
base_list_type::iterator iterator
iterator erase(iterator where)
pointer remove(iterator &IT)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor)
Returns the corresponding llvm.vector.deinterleaveN intrinsic for factor N.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
SpecificConstantMatch m_ZeroInt()
Convenience matchers for specific integer values.
ap_match< APInt > m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
bool match(Val *V, const Pattern &P)
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
GEPLikeRecipe_match< Op0_t, Op1_t > m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1)
class_match< VPValue > m_VPValue()
Match an arbitrary VPValue and ignore it.
NodeAddr< DefNode * > Def
bool isSingleScalar(const VPValue *VPV)
Returns true if VPV is a single scalar, either because it produces the same value for all lanes or on...
bool onlyFirstPartUsed(const VPValue *Def)
Returns true if only the first part of Def is used.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
bool onlyScalarValuesUsed(const VPValue *Def)
Returns true if only scalar values of Def are used by all users.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
LLVM_ABI Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)
Create a reduction of the given vector.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
FunctionAddr VTableAddr Value
LLVM_ABI Value * createFindLastIVReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind, Value *Start, Value *Sentinel)
Create a reduction of the given vector Src for a reduction of the kind RecurKind::FindLastIV.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
LLVM_ABI Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
auto cast_or_null(const Y &Val)
LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
LLVM_ABI Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)
Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)
Create a mask that filters the members of an interleave group where there are gaps.
LLVM_ABI llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
auto reverse(ContainerTy &&C)
LLVM_ABI llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)
Create a mask with replicated elements.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Type * toVectorizedTy(Type *Ty, ElementCount EC)
A helper for converting to vectorized types.
bool canConstantBeExtended(const APInt *C, Type *NarrowType, TTI::PartialReductionExtendKind ExtKind)
Check if a constant CI can be safely treated as having been extended from a narrower type with the gi...
cl::opt< unsigned > ForceTargetInstructionCost
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
bool canVectorizeTy(Type *Ty)
Returns true if Ty is a valid vector element type, void, or an unpacked literal struct where all elem...
LLVM_ABI llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
RecurKind
These are the kinds of recurrences that we support.
@ UMin
Unsigned integer min implemented in terms of select(cmp()).
@ Mul
Product of integers.
@ AnyOf
AnyOf reduction with select(cmp(),x,y) where one of (x,y) is loop invariant, and both x and y are int...
@ SMax
Signed integer max implemented in terms of select(cmp()).
@ SMin
Signed integer min implemented in terms of select(cmp()).
@ Sub
Subtraction of integers.
@ UMax
Unsigned integer max implemented in terms of select(cmp()).
LLVM_ABI bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic has a scalar operand.
LLVM_ABI Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
DWARFExpression::Operation Op
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
LLVM_ABI Value * createOrderedReduction(IRBuilderBase &B, RecurKind RdxKind, Value *Src, Value *Start)
Create an ordered reduction intrinsic using the given recurrence kind RdxKind.
auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
unsigned getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind)
A helper function that returns how much we should divide the cost of a predicated block by.
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
LLVM_ABI Value * createAnyOfReduction(IRBuilderBase &B, Value *Src, Value *InitVal, PHINode *OrigPhi)
Create a reduction of the given vector Src for a reduction of kind RecurKind::AnyOf.
LLVM_ABI bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
This struct is a compact representation of a valid (non-zero power of two) alignment.
Struct to hold various analysis needed for cost computations.
void execute(VPTransformState &State) override
Generate the phi nodes.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this first-order recurrence phi recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
An overlay for VPIRInstructions wrapping PHI nodes enabling convenient use cast/dyn_cast/isa and exec...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the instruction.
A pure-virtual common base class for recipes defining a single VPValue and using IR flags.
InstructionCost getCostForRecipeWithOpcode(unsigned Opcode, ElementCount VF, VPCostContext &Ctx) const
Compute the cost for this recipe for VF, using Opcode and Ctx.
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL=DebugLoc::getUnknown())
void execute(VPTransformState &State) override
Generate the wide load or gather.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenLoadEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate a wide load or gather.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isInvariantCond() const
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenSelectRecipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
VPValue * getStoredValue() const
Return the address accessed by this recipe.
void execute(VPTransformState &State) override
Generate the wide store or scatter.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenStoreEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
void execute(VPTransformState &State) override
Generate a wide store or scatter.
VPValue * getStoredValue() const
Return the value stored by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.