48#define LV_NAME "loop-vectorize"
49#define DEBUG_TYPE LV_NAME
54 return cast<VPInstruction>(
this)->opcodeMayReadOrWriteFromMemory();
56 return cast<VPInterleaveRecipe>(
this)->getNumStoreOperands() > 0;
57 case VPWidenStoreEVLSC:
64 return !cast<VPWidenCallRecipe>(
this)
65 ->getCalledScalarFunction()
67 case VPWidenIntrinsicSC:
68 return cast<VPWidenIntrinsicRecipe>(
this)->mayWriteToMemory();
69 case VPBranchOnMaskSC:
70 case VPScalarIVStepsSC:
74 case VPReductionEVLSC:
76 case VPVectorPointerSC:
77 case VPWidenCanonicalIVSC:
80 case VPWidenIntOrFpInductionSC:
81 case VPWidenLoadEVLSC:
86 case VPWidenSelectSC: {
90 assert((!
I || !
I->mayWriteToMemory()) &&
91 "underlying instruction may write to memory");
101 case VPInstructionSC:
102 return cast<VPInstruction>(
this)->opcodeMayReadOrWriteFromMemory();
103 case VPWidenLoadEVLSC:
108 ->mayReadFromMemory();
110 return !cast<VPWidenCallRecipe>(
this)
111 ->getCalledScalarFunction()
112 ->onlyWritesMemory();
113 case VPWidenIntrinsicSC:
114 return cast<VPWidenIntrinsicRecipe>(
this)->mayReadFromMemory();
115 case VPBranchOnMaskSC:
116 case VPPredInstPHISC:
117 case VPScalarIVStepsSC:
118 case VPWidenStoreEVLSC:
122 case VPReductionEVLSC:
124 case VPVectorPointerSC:
125 case VPWidenCanonicalIVSC:
128 case VPWidenIntOrFpInductionSC:
132 case VPWidenSelectSC: {
136 assert((!
I || !
I->mayReadFromMemory()) &&
137 "underlying instruction may read from memory");
148 case VPPredInstPHISC:
150 case VPReverseVectorPointerSC:
152 case VPInstructionSC:
154 case VPWidenCallSC: {
155 Function *Fn = cast<VPWidenCallRecipe>(
this)->getCalledScalarFunction();
158 case VPWidenIntrinsicSC:
159 return cast<VPWidenIntrinsicRecipe>(
this)->mayHaveSideEffects();
161 case VPReductionEVLSC:
163 case VPScalarIVStepsSC:
164 case VPVectorPointerSC:
165 case VPWidenCanonicalIVSC:
168 case VPWidenIntOrFpInductionSC:
170 case VPWidenPointerInductionSC:
173 case VPWidenSelectSC: {
177 assert((!
I || !
I->mayHaveSideEffects()) &&
178 "underlying instruction has side-effects");
183 case VPWidenLoadEVLSC:
185 case VPWidenStoreEVLSC:
190 "mayHaveSideffects result for ingredient differs from this "
193 case VPReplicateSC: {
194 auto *R = cast<VPReplicateRecipe>(
this);
195 return R->getUnderlyingInstr()->mayHaveSideEffects();
203 assert(!Parent &&
"Recipe already in some VPBasicBlock");
205 "Insertion position not in any VPBasicBlock");
211 assert(!Parent &&
"Recipe already in some VPBasicBlock");
217 assert(!Parent &&
"Recipe already in some VPBasicBlock");
219 "Insertion position not in any VPBasicBlock");
251 if (
auto *S = dyn_cast<VPSingleDefRecipe>(
this))
252 UI = dyn_cast_or_null<Instruction>(S->getUnderlyingValue());
253 else if (
auto *IG = dyn_cast<VPInterleaveRecipe>(
this))
254 UI = IG->getInsertPos();
255 else if (
auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(
this))
256 UI = &WidenMem->getIngredient();
269 dbgs() <<
"Cost of " << RecipeCost <<
" for VF " << VF <<
": ";
283 std::optional<unsigned> Opcode = std::nullopt;
285 if (
auto *WidenR = dyn_cast<VPWidenRecipe>(BinOpR))
286 Opcode = std::make_optional(WidenR->getOpcode());
301 auto *WidenCastR = dyn_cast<VPWidenCastRecipe>(R);
304 if (WidenCastR->getOpcode() == Instruction::CastOps::ZExt)
306 if (WidenCastR->getOpcode() == Instruction::CastOps::SExt)
312 PhiType, VF, GetExtendKind(ExtAR),
313 GetExtendKind(ExtBR), Opcode);
321 "Unhandled partial reduction opcode");
325 assert(PhiVal && BinOpVal &&
"Phi and Mul must be set");
329 CallInst *V = Builder.CreateIntrinsic(
330 RetTy, Intrinsic::experimental_vector_partial_reduce_add,
331 {PhiVal, BinOpVal},
nullptr,
"partial.reduce");
336#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
339 O << Indent <<
"PARTIAL-REDUCE ";
347 assert(OpType == OperationType::FPMathOp &&
348 "recipe doesn't have fast math flags");
360#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
364template <
unsigned PartOpIdx>
367 if (U.getNumOperands() == PartOpIdx + 1)
368 return U.getOperand(PartOpIdx);
372template <
unsigned PartOpIdx>
374 if (
auto *UnrollPartOp = getUnrollPartOperand(U))
375 return cast<ConstantInt>(UnrollPartOp->getLiveInIRValue())->getZExtValue();
385 assert(Opcode == Instruction::ICmp &&
386 "only ICmp predicates supported at the moment");
390 std::initializer_list<VPValue *>
Operands,
395 assert(isFPMathOp() &&
"this op can't take fast-math flags");
398bool VPInstruction::doesGeneratePerAllLanes()
const {
402bool VPInstruction::canGenerateScalarForFirstLane()
const {
408 case Instruction::ICmp:
409 case Instruction::Select:
428 "only PtrAdd opcodes are supported for now");
442 if (
auto *
I = dyn_cast<Instruction>(Res))
452 case Instruction::ICmp: {
458 case Instruction::Select: {
481 {VIVElem0, ScalarTC},
nullptr, Name);
497 if (!V1->getType()->isVectorTy())
517 "Requested vector length should be an integer.");
524 {AVL, VFArg, State.Builder.getTrue()});
530 assert(Part != 0 &&
"Must have a positive part");
580 auto *PhiR = cast<VPReductionPHIRecipe>(
getOperand(0));
581 auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
587 Type *PhiTy = OrigPhi->getType();
592 for (
unsigned Part = 0; Part < UF; ++Part)
593 RdxParts[Part] = State.
get(
getOperand(1 + Part), PhiR->isInLoop());
601 for (
unsigned Part = 0; Part < UF; ++Part)
602 RdxParts[Part] = Builder.
CreateTrunc(RdxParts[Part], RdxVecTy);
605 Value *ReducedPartRdx = RdxParts[0];
608 Op = Instruction::Or;
610 if (PhiR->isOrdered()) {
611 ReducedPartRdx = RdxParts[UF - 1];
616 for (
unsigned Part = 1; Part < UF; ++Part) {
617 Value *RdxPart = RdxParts[Part];
618 if (
Op != Instruction::ICmp &&
Op != Instruction::FCmp)
625 ReducedPartRdx =
createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
645 return ReducedPartRdx;
649 unsigned Offset = CI->getZExtValue();
650 assert(
Offset > 0 &&
"Offset from end must be positive");
654 "invalid offset to extract from");
658 assert(
Offset <= 1 &&
"invalid offset to extract from");
661 if (isa<ExtractElementInst>(Res))
672 "can only generate first lane for PtrAdd");
678 Value *IncomingFromVPlanPred =
680 Value *IncomingFromOtherPreds =
687 NewPhi->addIncoming(IncomingFromVPlanPred, VPlanPred);
689 if (OtherPred == VPlanPred)
691 NewPhi->addIncoming(IncomingFromOtherPreds, OtherPred);
716bool VPInstruction::isFPMathOp()
const {
719 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
720 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
721 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
722 Opcode == Instruction::FCmp || Opcode == Instruction::Select;
727 assert(!State.
Lane &&
"VPInstruction executing an Lane");
731 "Recipe not a FPMathOp but has fast-math flags?");
735 bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&
738 bool GeneratesPerAllLanes = doesGeneratePerAllLanes();
739 if (GeneratesPerAllLanes) {
741 Lane != NumLanes; ++Lane) {
742 Value *GeneratedValue = generatePerLane(State,
VPLane(Lane));
743 assert(GeneratedValue &&
"generatePerLane must produce a value");
744 State.
set(
this, GeneratedValue,
VPLane(Lane));
749 Value *GeneratedValue = generate(State);
752 assert(GeneratedValue &&
"generate must produce a value");
756 "scalar value but not only first lane defined");
757 State.
set(
this, GeneratedValue,
758 GeneratesPerFirstLaneOnly);
765 case Instruction::ICmp:
766 case Instruction::Select:
789 case Instruction::ICmp:
790 case Instruction::Select:
791 case Instruction::Or:
815 case Instruction::ICmp:
816 case Instruction::Select:
826#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
834 O << Indent <<
"EMIT ";
846 O <<
"combined load";
849 O <<
"combined store";
852 O <<
"active lane mask";
858 O <<
"EXPLICIT-VECTOR-LENGTH";
861 O <<
"first-order splice";
864 O <<
"branch-on-cond";
867 O <<
"TC > VF ? TC - VF : 0";
873 O <<
"branch-on-count";
876 O <<
"extract-from-end";
879 O <<
"compute-reduction-result";
906 "Only PHINodes can have extra operands");
919 auto *Phi = cast<PHINode>(&I);
922 if (Phi->getBasicBlockIndex(PredBB) == -1)
923 Phi->addIncoming(V, PredBB);
925 Phi->setIncomingValueForBlock(PredBB, V);
940#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
943 O << Indent <<
"IR " << I;
946 O <<
" (extra operand" << (
getNumOperands() > 1 ?
"s" :
"") <<
": ";
977 assert(Variant !=
nullptr &&
"Can't create vector function.");
982 CI->getOperandBundlesAsDefs(OpBundles);
987 if (!V->getType()->isVoidTy())
1000#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1003 O << Indent <<
"WIDEN-CALL ";
1015 O <<
" @" << CalledFn->
getName() <<
"(";
1021 O <<
" (using library function";
1023 O <<
": " << Variant->
getName();
1049 Args.push_back(Arg);
1057 "Can't retrieve vector intrinsic or vector-predication intrinsics.");
1062 CI->getOperandBundlesAsDefs(OpBundles);
1068 if (!V->getType()->isVoidTy())
1084 auto *V =
Op->getUnderlyingValue();
1131#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1134 O << Indent <<
"WIDEN-INTRINSIC ";
1164 Value *Mask =
nullptr;
1166 Mask = State.
get(VPMask);
1173 if (Opcode == Instruction::Sub)
1176 assert(Opcode == Instruction::Add &&
"only add or sub supported for now");
1212 {PtrTy, IncTy, MaskTy});
1220#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1223 O << Indent <<
"WIDEN-HISTOGRAM buckets: ";
1226 if (Opcode == Instruction::Sub)
1229 assert(Opcode == Instruction::Add);
1242 O << Indent <<
"WIDEN-SELECT ";
1268 State.
set(
this, Sel);
1292 [](
VPValue *
Op) {
return Op->getUnderlyingValue(); }))
1293 Operands.append(SI->op_begin(), SI->op_end());
1294 bool IsLogicalOr =
match(
this,
m_LogicalOr(m_VPValue(Op0), m_VPValue(Op1)));
1296 IsLogicalOr ? Instruction::Or : Instruction::And, VectorTy,
CostKind,
1297 {Op1VK, Op1VP}, {Op2VK, Op2VP},
Operands, SI);
1305 if (
auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
1306 Pred = Cmp->getPredicate();
1312VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
1323#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1326 case OperationType::Cmp:
1329 case OperationType::DisjointOp:
1333 case OperationType::PossiblyExactOp:
1337 case OperationType::OverflowingBinOp:
1343 case OperationType::FPMathOp:
1346 case OperationType::GEPOp:
1354 case OperationType::NonNegOp:
1358 case OperationType::Other:
1368 auto &Builder = State.
Builder;
1370 case Instruction::Call:
1371 case Instruction::Br:
1372 case Instruction::PHI:
1373 case Instruction::GetElementPtr:
1374 case Instruction::Select:
1376 case Instruction::UDiv:
1377 case Instruction::SDiv:
1378 case Instruction::SRem:
1379 case Instruction::URem:
1380 case Instruction::Add:
1381 case Instruction::FAdd:
1382 case Instruction::Sub:
1383 case Instruction::FSub:
1384 case Instruction::FNeg:
1385 case Instruction::Mul:
1386 case Instruction::FMul:
1387 case Instruction::FDiv:
1388 case Instruction::FRem:
1389 case Instruction::Shl:
1390 case Instruction::LShr:
1391 case Instruction::AShr:
1392 case Instruction::And:
1393 case Instruction::Or:
1394 case Instruction::Xor: {
1402 if (
auto *VecOp = dyn_cast<Instruction>(V))
1410 case Instruction::Freeze: {
1414 State.
set(
this, Freeze);
1417 case Instruction::ICmp:
1418 case Instruction::FCmp: {
1420 bool FCmp = Opcode == Instruction::FCmp;
1448 "inferred type and type from generated instructions do not match");
1456 case Instruction::FNeg: {
1464 case Instruction::UDiv:
1465 case Instruction::SDiv:
1466 case Instruction::SRem:
1467 case Instruction::URem:
1470 case Instruction::Add:
1471 case Instruction::FAdd:
1472 case Instruction::Sub:
1473 case Instruction::FSub:
1474 case Instruction::Mul:
1475 case Instruction::FMul:
1476 case Instruction::FDiv:
1477 case Instruction::FRem:
1478 case Instruction::Shl:
1479 case Instruction::LShr:
1480 case Instruction::AShr:
1481 case Instruction::And:
1482 case Instruction::Or:
1483 case Instruction::Xor: {
1489 if (
RHS->isLiveIn())
1506 case Instruction::Freeze: {
1511 case Instruction::ICmp:
1512 case Instruction::FCmp: {
1534 "VPWidenEVLRecipe should not be used for scalars");
1537 Value *EVLArg = State.
get(EVL,
true);
1553 if (isa<FPMathOperator>(VPInst))
1554 setFlags(cast<Instruction>(VPInst));
1556 State.
set(
this, VPInst);
1561#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1564 O << Indent <<
"WIDEN ";
1573 O << Indent <<
"WIDEN ";
1583 auto &Builder = State.
Builder;
1590 State.
set(
this, Cast);
1592 if (
auto *CastOp = dyn_cast<Instruction>(Cast))
1607 if (isa<VPInterleaveRecipe>(R))
1609 if (
const auto *ReplicateRecipe = dyn_cast<VPReplicateRecipe>(R))
1612 const auto *WidenMemoryRecipe = dyn_cast<VPWidenMemoryRecipe>(R);
1613 if (WidenMemoryRecipe ==
nullptr)
1615 if (!WidenMemoryRecipe->isConsecutive())
1617 if (WidenMemoryRecipe->isReverse())
1619 if (WidenMemoryRecipe->isMasked())
1627 if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) &&
1629 if (
auto *StoreRecipe = dyn_cast<VPRecipeBase>(*
user_begin()))
1630 CCH = ComputeCCH(StoreRecipe);
1633 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
1634 Opcode == Instruction::FPExt) {
1650#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1653 O << Indent <<
"WIDEN-CAST ";
1677 auto *ValVTy = cast<VectorType>(Val->
getType());
1682 "Induction Step must be an integer or FP");
1690 Type *InitVecValSTy =
1701 Step = Builder.
CreateMul(InitVec, Step);
1702 return Builder.
CreateAdd(Val, Step,
"induction");
1706 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
1707 "Binary Opcode should be specified for FP induction");
1712 return Builder.
CreateBinOp(BinOp, Val, MulOp,
"induction");
1719 : ConstantFP::get(Ty,
C);
1723 assert(!State.
Lane &&
"Int or FP induction being replicated.");
1730 "Types must match");
1739 if (
ID.getInductionBinOp() && isa<FPMathOperator>(
ID.getInductionBinOp()))
1745 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
1746 "Expected either an induction phi-node or a truncate of it!");
1749 auto CurrIP = Builder.
saveIP();
1752 if (isa<TruncInst>(EntryVal)) {
1753 assert(Start->getType()->isIntegerTy() &&
1754 "Truncation requires an integer type");
1755 auto *TruncType = cast<IntegerType>(EntryVal->
getType());
1757 Start = Builder.
CreateCast(Instruction::Trunc, Start, TruncType);
1769 AddOp = Instruction::Add;
1770 MulOp = Instruction::Mul;
1772 AddOp =
ID.getInductionOpcode();
1773 MulOp = Instruction::FMul;
1780 SplatVF = State.
get(SplatVFOperand);
1803 State.
set(
this, VecInd);
1806 Builder.
CreateBinOp(AddOp, VecInd, SplatVF,
"vec.ind.next"));
1807 if (isa<TruncInst>(EntryVal))
1820#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1825 O <<
" = WIDEN-INDUCTION ";
1829 O <<
" (truncated to " << *TI->getType() <<
")";
1841 auto *CanIV = cast<VPCanonicalIVPHIRecipe>(&*
getParent()->begin());
1842 return StartC && StartC->isZero() && StepC && StepC->isOne() &&
1846#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1851 O <<
" = DERIVED-IV ";
1875 assert(BaseIVTy == Step->
getType() &&
"Types of BaseIV and Step must match!");
1882 AddOp = Instruction::Add;
1883 MulOp = Instruction::Mul;
1885 AddOp = InductionOpcode;
1886 MulOp = Instruction::FMul;
1895 Type *VecIVTy =
nullptr;
1896 Value *UnitStepVec =
nullptr, *SplatStep =
nullptr, *SplatIV =
nullptr;
1905 unsigned StartLane = 0;
1908 StartLane = State.
Lane->getKnownLane();
1909 EndLane = StartLane + 1;
1916 auto *InitVec = Builder.
CreateAdd(SplatStartIdx, UnitStepVec);
1930 for (
unsigned Lane = StartLane; Lane < EndLane; ++Lane) {
1936 "Expected StartIdx to be folded to a constant when VF is not "
1944#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1949 O <<
" = SCALAR-STEPS ";
1963 if (areAllOperandsInvariant()) {
1999 if (isIndexLoopInvariant(
I - 1))
2010 "NewGEP is not a pointer vector");
2011 State.
set(
this, NewGEP);
2016#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2019 O << Indent <<
"WIDEN-GEP ";
2020 O << (isPointerLoopInvariant() ?
"Inv" :
"Var");
2022 O <<
"[" << (isIndexLoopInvariant(
I) ?
"Inv" :
"Var") <<
"]";
2026 O <<
" = getelementptr";
2037 return IsScalable && (IsReverse || CurrentPart > 0)
2043 auto &Builder = State.
Builder;
2047 CurrentPart, Builder);
2051 if (IndexTy != RunTimeVF->
getType())
2055 ConstantInt::get(IndexTy, -(int64_t)CurrentPart), RunTimeVF);
2057 Value *LastLane = Builder.
CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
2061 ResultPtr = Builder.
CreateGEP(IndexedTy, ResultPtr, LastLane,
"",
2064 State.
set(
this, ResultPtr,
true);
2067#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2072 O <<
" = reverse-vector-pointer";
2079 auto &Builder = State.
Builder;
2083 CurrentPart, Builder);
2090 State.
set(
this, ResultPtr,
true);
2093#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2098 O <<
" = vector-pointer ";
2124 Value *Result =
nullptr;
2125 for (
unsigned In = 0; In < NumIncoming; ++In) {
2138 State.
set(
this, Result, OnlyFirstLaneUsed);
2157#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2160 O << Indent <<
"BLEND ";
2182 assert(!State.
Lane &&
"Reduction being replicated.");
2216 PrevInChain = NewRed;
2217 NextInChain = NewRed;
2223 NewRed, PrevInChain);
2228 State.
set(
this, NextInChain,
true);
2232 assert(!State.
Lane &&
"Reduction being replicated.");
2234 auto &Builder = State.
Builder;
2250 Mask = State.
get(CondOp);
2266 State.
set(
this, NewRed,
true);
2273 auto *VectorTy = cast<VectorType>(
toVectorTy(ElementTy, VF));
2281 "Any-of reduction not implemented in VPlan-based cost model currently.");
2283 (!cast<VPReductionPHIRecipe>(
getOperand(0))->isInLoop() ||
2285 "In-loop reduction not implemented in VPlan-based cost model currently.");
2288 "Inferred type and recurrence type mismatch.");
2303#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2306 O << Indent <<
"REDUCE ";
2321 O <<
" (with final reduction value stored in invariant address sank "
2328 O << Indent <<
"REDUCE ";
2345 O <<
" (with final reduction value stored in invariant address sank "
2354 if (
auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
2355 return any_of(PredR->users(), [PredR](
const VPUser *U) {
2356 return !U->usesScalars(PredR);
2371#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2374 O << Indent << (IsUniform ?
"CLONE " :
"REPLICATE ");
2383 O <<
"@" << CB->getCalledFunction()->getName() <<
"(";
2403 "Codegen only implemented for first lane.");
2405 case Instruction::SExt:
2406 case Instruction::ZExt:
2407 case Instruction::Trunc: {
2418 State.
set(
this, generate(State),
VPLane(0));
2421#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2424 O << Indent <<
"SCALAR-CAST ";
2428 O <<
" to " << *ResultTy;
2433 assert(State.
Lane &&
"Branch on Mask works only on single instance.");
2435 unsigned Lane = State.
Lane->getKnownLane();
2437 Value *ConditionBit =
nullptr;
2440 ConditionBit = State.
get(BlockInMask);
2450 assert(isa<UnreachableInst>(CurrentTerminator) &&
2451 "Expected to replace unreachable terminator with conditional branch.");
2467 assert(State.
Lane &&
"Predicated instruction PHI works per instance.");
2472 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
2474 "operand must be VPReplicateRecipe");
2489 State.
reset(
this, VPhi);
2491 State.
set(
this, VPhi);
2503 Phi->addIncoming(ScalarPredInst, PredicatedBB);
2507 State.
set(
this, Phi, *State.
Lane);
2514#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2517 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
2527 const Align Alignment =
2539 "Inconsecutive memory access should not have the order.");
2560 cast<VectorType>(Ty), {},
CostKind, 0);
2571 auto &Builder = State.
Builder;
2573 Value *Mask =
nullptr;
2574 if (
auto *VPMask =
getMask()) {
2577 Mask = State.
get(VPMask);
2586 "wide.masked.gather");
2598 State.
set(
this, NewLI);
2601#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2604 O << Indent <<
"WIDEN ";
2616 Value *AllTrueMask =
2618 return Builder.
CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,
2619 {Operand, AllTrueMask, EVL},
nullptr,
Name);
2630 auto &Builder = State.
Builder;
2635 Value *Mask =
nullptr;
2637 Mask = State.
get(VPMask);
2647 nullptr,
"wide.masked.gather");
2652 Instruction::Load, DataTy,
Addr,
"vp.op.load"));
2660 State.
set(
this, Res);
2674 const Align Alignment =
2685 cast<VectorType>(Ty), {},
CostKind, 0);
2688#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2691 O << Indent <<
"WIDEN ";
2705 auto &Builder = State.
Builder;
2708 Value *Mask =
nullptr;
2709 if (
auto *VPMask =
getMask()) {
2712 Mask = State.
get(VPMask);
2717 Value *StoredVal = State.
get(StoredVPValue);
2736#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2739 O << Indent <<
"WIDEN store ";
2751 auto &Builder = State.
Builder;
2755 Value *StoredVal = State.
get(StoredValue);
2759 Value *Mask =
nullptr;
2761 Mask = State.
get(VPMask);
2768 if (CreateScatter) {
2770 Intrinsic::vp_scatter,
2771 {StoredVal, Addr, Mask, EVL});
2777 {StoredVal, Addr}));
2795 const Align Alignment =
2806 cast<VectorType>(Ty), {},
CostKind, 0);
2809#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2812 O << Indent <<
"WIDEN vp.store ";
2820 auto VF = DstVTy->getElementCount();
2821 auto *SrcVecTy = cast<VectorType>(V->getType());
2822 assert(VF == SrcVecTy->getElementCount() &&
"Vector dimensions do not match");
2823 Type *SrcElemTy = SrcVecTy->getElementType();
2824 Type *DstElemTy = DstVTy->getElementType();
2825 assert((
DL.getTypeSizeInBits(SrcElemTy) ==
DL.getTypeSizeInBits(DstElemTy)) &&
2826 "Vector elements must have same size");
2837 "Only one type should be a pointer type");
2839 "Only one type should be a floating point type");
2851 unsigned Factor = Vals.
size();
2852 assert(Factor > 1 &&
"Tried to interleave invalid number of vectors");
2856 for (
Value *Val : Vals)
2857 assert(Val->getType() == VecTy &&
"Tried to interleave mismatched types");
2862 if (VecTy->isScalableTy()) {
2864 return Builder.
CreateIntrinsic(WideVecTy, Intrinsic::vector_interleave2,
2873 const unsigned NumElts = VecTy->getElementCount().getFixedValue();
2907 assert(!State.
Lane &&
"Interleave group being replicated.");
2913 unsigned InterleaveFactor = Group->
getFactor();
2919 "Reversed masked interleave-group not supported.");
2923 if (
auto *
I = dyn_cast<Instruction>(ResAddr))
2939 bool InBounds =
false;
2941 InBounds = Gep->isInBounds();
2948 auto CreateGroupMask = [&BlockInMask, &State,
2949 &InterleaveFactor](
Value *MaskForGaps) ->
Value * {
2951 assert(!MaskForGaps &&
"Interleaved groups with gaps are not supported.");
2952 assert(InterleaveFactor == 2 &&
2953 "Unsupported deinterleave factor for scalable vectors");
2954 auto *ResBlockInMask = State.
get(BlockInMask);
2959 MaskTy, Intrinsic::vector_interleave2, Ops,
2960 nullptr,
"interleaved.mask");
2966 Value *ResBlockInMask = State.
get(BlockInMask);
2970 "interleaved.mask");
2972 ShuffledMask, MaskForGaps)
2978 if (isa<LoadInst>(Instr)) {
2979 Value *MaskForGaps =
nullptr;
2980 if (NeedsMaskForGaps) {
2983 assert(MaskForGaps &&
"Mask for Gaps is required but it is null");
2987 if (BlockInMask || MaskForGaps) {
2988 Value *GroupMask = CreateGroupMask(MaskForGaps);
2991 PoisonVec,
"wide.masked.vec");
2999 if (VecTy->isScalableTy()) {
3000 assert(InterleaveFactor == 2 &&
3001 "Unsupported deinterleave factor for scalable vectors");
3006 Intrinsic::vector_deinterleave2, VecTy, NewLoad,
3007 nullptr,
"strided.vec");
3009 for (
unsigned I = 0;
I < InterleaveFactor; ++
I) {
3017 if (Member->getType() != ScalarTy) {
3026 State.
set(VPDefs[J], StridedVec);
3036 for (
unsigned I = 0;
I < InterleaveFactor; ++
I) {
3049 if (Member->getType() != ScalarTy) {
3059 State.
set(VPDefs[J], StridedVec);
3069 Value *MaskForGaps =
3072 "masking gaps for scalable vectors is not yet supported.");
3076 unsigned StoredIdx = 0;
3077 for (
unsigned i = 0; i < InterleaveFactor; i++) {
3079 "Fail to get a member from an interleaved store group");
3089 Value *StoredVec = State.
get(StoredValues[StoredIdx]);
3097 if (StoredVec->
getType() != SubVT)
3106 if (BlockInMask || MaskForGaps) {
3107 Value *GroupMask = CreateGroupMask(MaskForGaps);
3109 IVec, ResAddr, Group->
getAlign(), GroupMask);
3117#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3120 O << Indent <<
"INTERLEAVE-GROUP with factor " << IG->getFactor() <<
" at ";
3121 IG->getInsertPos()->printAsOperand(O,
false);
3131 for (
unsigned i = 0; i < IG->getFactor(); ++i) {
3132 if (!IG->getMember(i))
3135 O <<
"\n" << Indent <<
" store ";
3137 O <<
" to index " << i;
3139 O <<
"\n" << Indent <<
" ";
3141 O <<
" = load from index " << i;
3152 unsigned InsertPosIdx = 0;
3153 for (
unsigned Idx = 0; IG->getFactor(); ++
Idx)
3154 if (
auto *Member = IG->getMember(
Idx)) {
3155 if (Member == InsertPos)
3162 auto *VectorTy = cast<VectorType>(
toVectorTy(ValTy, VF));
3166 unsigned InterleaveFactor = IG->getFactor();
3171 for (
unsigned IF = 0; IF < InterleaveFactor; IF++)
3172 if (IG->getMember(IF))
3177 InsertPos->
getOpcode(), WideVecTy, IG->getFactor(), Indices,
3180 if (!IG->isReverse())
3183 return Cost + IG->getNumMembers() *
3185 VectorTy, std::nullopt,
CostKind, 0);
3188#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3191 O << Indent <<
"EMIT ";
3193 O <<
" = CANONICAL-INDUCTION ";
3199 return IsScalarAfterVectorization &&
3206 "Not a pointer induction according to InductionDescriptor!");
3208 "Unexpected type.");
3210 "Recipe should have been replaced");
3216 Type *ScStValueType = ScalarStartValue->
getType();
3219 PHINode *NewPointerPhi =
nullptr;
3220 if (CurrentPart == 0) {
3221 auto *IVR = cast<VPHeaderPHIRecipe>(&
getParent()
3223 ->getVectorLoopRegion()
3224 ->getEntryBasicBlock()
3226 PHINode *CanonicalIV = cast<PHINode>(State.
get(IVR,
true));
3229 NewPointerPhi->
addIncoming(ScalarStartValue, VectorPH);
3236 NewPointerPhi = cast<PHINode>(
GEP->getPointerOperand());
3249 if (CurrentPart == 0) {
3253 Value *NumUnrolledElems =
3261 NewPointerPhi->
addIncoming(InductionGEP, VectorPH);
3268 RuntimeVF, ConstantInt::get(PhiType, CurrentPart));
3269 Value *StartOffset =
3276 "scalar step must be the same across all parts");
3280 State.
VF, ScalarStepValue)),
3285#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3289 "unexpected number of operands");
3290 O << Indent <<
"EMIT ";
3292 O <<
" = WIDEN-POINTER-INDUCTION ";
3306 assert(!State.
Lane &&
"cannot be used in per-lane");
3314 "Results must match");
3327#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3330 O << Indent <<
"EMIT ";
3332 O <<
" = EXPAND SCEV " << *Expr;
3350 Value *CanonicalVectorIV = Builder.
CreateAdd(VStart, VStep,
"vec.iv");
3351 State.
set(
this, CanonicalVectorIV);
3354#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3357 O << Indent <<
"EMIT ";
3359 O <<
" = WIDEN-CANONICAL-INDUCTION ";
3365 auto &Builder = State.
Builder;
3370 ? VectorInit->getType()
3376 auto *One = ConstantInt::get(IdxTy, 1);
3380 auto *LastIdx = Builder.
CreateSub(RuntimeVF, One);
3388 Phi->addIncoming(VectorInit, VectorPH);
3389 State.
set(
this, Phi);
3408 cast<VectorType>(VectorTy), Mask,
CostKind,
3412#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3415 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
3423 auto &Builder = State.
Builder;
3438 bool ScalarPHI = State.
VF.
isScalar() || IsInLoop;
3444 "recipe must be in the vector loop header");
3447 State.
set(
this, Phi, IsInLoop);
3451 Value *Iden =
nullptr;
3463 StartV = Iden = State.
get(StartVPV);
3486 if (CurrentPart == 0) {
3501 Phi = cast<PHINode>(State.
get(
this, IsInLoop));
3502 Value *StartVal = (CurrentPart == 0) ? StartV : Iden;
3503 Phi->addIncoming(StartVal, VectorPH);
3506#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3509 O << Indent <<
"WIDEN-REDUCTION-PHI ";
3514 if (VFScaleFactor != 1)
3515 O <<
" (VF scaled by 1/" << VFScaleFactor <<
")";
3521 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
3526 State.
set(
this, VecPhi);
3529#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3532 O << Indent <<
"WIDEN-PHI ";
3557 Phi->addIncoming(StartMask, VectorPH);
3559 State.
set(
this, Phi);
3562#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3565 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
3573#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3576 O << Indent <<
"EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
3588 Phi->addIncoming(Start, VectorPH);
3590 State.
set(
this, Phi,
true);
3593#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3596 O << Indent <<
"SCALAR-PHI ";
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static cl::opt< TargetTransformInfo::TargetCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(TargetTransformInfo::TCK_RecipThroughput), cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(TargetTransformInfo::TCK_Latency, "latency", "Instruction latency"), clEnumValN(TargetTransformInfo::TCK_CodeSize, "code-size", "Code size"), clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency")))
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
cl::opt< unsigned > ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))
mir Rename Register Operands
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
static Instruction * createReverseEVL(IRBuilderBase &Builder, Value *Operand, Value *EVL, const Twine &Name)
Use all-true mask for reverse rather than actual mask, as it avoids a dependence w/o affecting the re...
static Value * interleaveVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name)
Return a vector containing interleaved elements from multiple smaller input vectors.
static Value * createBitOrPointerCast(IRBuilderBase &Builder, Value *V, VectorType *DstVTy, const DataLayout &DL)
cl::opt< unsigned > ForceTargetInstructionCost
static Value * getStepVector(Value *Val, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)
This function adds (0 * Step, 1 * Step, 2 * Step, ...) to each vector element of Val.
static Type * getGEPIndexTy(bool IsScalable, bool IsReverse, unsigned CurrentPart, IRBuilderBase &Builder)
static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)
A helper function that returns an integer or floating-point constant with value C.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
InstListType::iterator iterator
Instruction iterators...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
static StringRef getPredicateName(Predicate P)
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
constexpr bool isVector() const
One or more elements.
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
void print(raw_ostream &O) const
Print fast-math flags to O.
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
ArrayRef< Type * > params() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
bool willReturn() const
Determine if the function will return.
bool doesNotThrow() const
Determine if the function cannot unwind.
Type * getReturnType() const
Returns the type of the ret val.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Common base class shared among various IRBuilders.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
ConstantInt * getTrue()
Get the constant value for i1 true.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateFreeze(Value *V, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
BasicBlock * GetInsertBlock() const
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Value * CreateVectorReverse(Value *V, const Twine &Name="")
Return a vector value that contains the vector V reversed.
Value * CreateFCmpFMF(CmpInst::Predicate P, Value *LHS, Value *RHS, FMFSource FMFSource, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
InsertPoint saveIP() const
Returns the current insert point.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="")
void restoreIP(InsertPoint IP)
Sets the current insert point to a previously-saved location.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
@ IK_PtrInduction
Pointer induction var. Step = C.
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
static InstructionCost getInvalid(CostType Val=0)
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
void addMetadata(InstTy *NewInst) const
Add metadata (e.g.
BlockT * getHeader() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
FastMathFlags getFastMathFlags() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Type * getRecurrenceType() const
Returns the type of the recurrence.
TrackingVH< Value > getRecurrenceStartValue() const
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isFindLastIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class uses information about analyze scalars to rewrite expressions in canonical form.
Type * getType() const
Return the LLVM type of this SCEV expression.
This class represents the LLVM 'select' instruction.
This class provides computation of slot numbers for LLVM Assembly writing.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
static IntegerType * getInt1Ty(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeID getTypeID() const
Return the type id for the type.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
value_op_iterator value_op_end()
Value * getOperand(unsigned i) const
value_op_iterator value_op_begin()
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenMemoryRecipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account when normalized the first incoming value wi...
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool isNormalized() const
A normalized blend is one that has an odd number of operands, whereby the first operand does not have...
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
const VPBlocksTy & getPredecessors() const
const VPBasicBlock * getEntryBasicBlock() const
VPValue * getMask() const
Return the mask used by this recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPBranchOnMaskRecipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
void dump() const
Dump the VPDef to stderr (for debugging).
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
unsigned getVPDefID() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPValue * getStartValue() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a vectorized histogram operation.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPHistogramRecipe.
VPValue * getMask() const
Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPIRInstruction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
@ ResumePhi
Creates a scalar phi in a leaf VPBB with a single predecessor in VPlan.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
bool opcodeMayReadOrWriteFromMemory() const
Returns true if the underlying opcode may read from or write to memory.
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
bool isVectorToScalar() const
Returns true if this VPInstruction produces a scalar value from a vector, e.g.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
bool isSingleScalar() const
Returns true if this VPInstruction's operands are single scalars and the result is also a single scal...
void execute(VPTransformState &State) override
Generate the instruction.
VPValue * getAddr() const
Return the address accessed by this recipe.
VPValue * getMask() const
Return the mask used by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
Instruction * getInsertPos() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPInterleaveRecipe.
unsigned getNumStoreOperands() const
Returns the number of stored operands of this interleave group.
static bool isVPIntrinsic(Intrinsic::ID)
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)
static VPLane getFirstLane()
void execute(VPTransformState &State) override
Generate the reduction in the loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPPartialReductionRecipe.
unsigned getOpcode() const
Get the binary op's opcode.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs (from a replicate region) as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
virtual InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const
Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this recipe, taking into account if the cost computation should be skipped and the...
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
NonNegFlagsTy NonNegFlags
GEPNoWrapFlags getGEPNoWrapFlags() const
void setFlags(Instruction *I) const
Set the IR flags for I.
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
DisjointFlagsTy DisjointFlags
bool hasNoUnsignedWrap() const
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
bool hasNoSignedWrap() const
FastMathFlags getFastMathFlags() const
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPValue * getEVL() const
The VPValue of the explicit vector length.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool isConditional() const
Return true if the in-loop reduction is conditional.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of VPReductionRecipe.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
const RecurrenceDescriptor & getRecurrenceDescriptor() const
Return the recurrence decriptor for the in-loop reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
bool isOrdered() const
Return true if the in-loop reduction is ordered.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPReplicateRecipe.
unsigned getOpcode() const
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
LLVM_DUMP_METHOD void dump() const
Print this VPSingleDefRecipe to dbgs() (for debugging).
This class can be used to assign names to VPValues.
LLVMContext & getContext()
Return the LLVMContext used by the analysis.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
VPValue * getUnrollPartOperand(VPUser &U) const
Return the VPValue operand containing the unroll part or null if there is no such operand.
unsigned getUnrollPart(VPUser &U) const
Return the unroll part.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
virtual bool onlyFirstLaneUsed(const VPValue *Op) const
Returns true if the VPUser only uses the first lane of operand Op.
bool isDefinedOutsideLoopRegions() const
Returns true if the VPValue is defined outside any loop region.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
friend class VPInstruction
bool hasMoreThanOneUniqueUser() const
Returns true if the value has more than one unique user.
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
user_iterator user_begin()
unsigned getNumUsers() const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Function * getCalledScalarFunction() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCallRecipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
operand_range arg_operands()
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCastRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override final
Print the recipe.
void execute(VPTransformState &State) override final
Produce a vp-intrinsic using the opcode and operands of the recipe, processing EVL elements.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
PHINode * getPHINode() const
VPValue * getStepValue()
Returns the step value of the induction.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
Type * getScalarType() const
Returns the scalar type of the induction.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getSplatVFValue()
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
StringRef getIntrinsicName() const
Return to name of the intrinsic as string.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Return the scalar return type of the intrinsic.
void execute(VPTransformState &State) override
Produce a widened version of the vector intrinsic.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this vector intrinsic.
bool IsMasked
Whether the memory access is masked.
bool Reverse
Whether the consecutive accessed addresses are in reverse order.
bool isConsecutive() const
Return whether the loaded-from / stored-to addresses are consecutive.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenMemoryRecipe.
bool Consecutive
Whether the accessed addresses are consecutive.
VPValue * getMask() const
Return the mask used by this recipe.
VPValue * getAddr() const
Return the address accessed by this recipe.
bool isReverse() const
Return whether the consecutive loaded/stored addresses are in reverse order.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
VPValue * getFirstUnrolledPartOperand()
Returns the VPValue representing the value of this induction at the first unrolled part,...
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenRecipe.
void execute(VPTransformState &State) override
Produce a widened instruction using the opcode and operands of the recipe, processing State....
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getOpcode() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
VectorBuilder & setEVL(Value *NewExplicitVectorLength)
VectorBuilder & setMask(Value *NewMask)
Value * createVectorInstruction(unsigned Opcode, Type *ReturnTy, ArrayRef< Value * > VecOpArray, const Twine &Name=Twine())
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
static VectorType * getDoubleElementsVectorType(VectorType *VTy)
This static method returns a VectorType with twice as many elements as the input type and the same el...
Type * getElementType() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
const ParentTy * getParent() const
self_iterator getIterator()
base_list_type::iterator iterator
iterator erase(iterator where)
pointer remove(iterator &IT)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
bool match(Val *V, const Pattern &P)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
bool isUniformAfterVectorization(const VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstPartUsed(const VPValue *Def)
Returns true if only the first part of Def is used.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)
Create a reduction of the given vector.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)
Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)
Create a mask that filters the members of an interleave group where there are gaps.
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)
Create a mask with replicated elements.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
Value * createOrderedReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, Value *Start)
Create an ordered reduction intrinsic using the given recurrence descriptor Desc.
Value * createReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, PHINode *OrigPhi=nullptr)
Create a generic reduction using a recurrence descriptor Desc Fast-math-flags are propagated using th...
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic has a scalar operand.
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
DWARFExpression::Operation Op
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
This struct is a compact representation of a valid (non-zero power of two) alignment.
Struct to hold various analysis needed for cost computations.
TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const
Returns the OperandInfo for V, if it is a live-in.
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
const TargetLibraryInfo & TLI
const TargetTransformInfo & TTI
SmallPtrSet< Instruction *, 8 > SkipCostComputation
void execute(VPTransformState &State) override
Generate the phi nodes.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this first-order recurrence phi recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the wide load or gather.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenLoadEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate a wide load or gather.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isInvariantCond() const
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenSelectRecipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
VPValue * getStoredValue() const
Return the address accessed by this recipe.
void execute(VPTransformState &State) override
Generate the wide store or scatter.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenStoreEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
void execute(VPTransformState &State) override
Generate a wide store or scatter.
VPValue * getStoredValue() const
Return the value stored by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.