48#define LV_NAME "loop-vectorize"
49#define DEBUG_TYPE LV_NAME
54 return cast<VPInstruction>(
this)->opcodeMayReadOrWriteFromMemory();
56 return cast<VPInterleaveRecipe>(
this)->getNumStoreOperands() > 0;
57 case VPWidenStoreEVLSC:
64 return !cast<VPWidenCallRecipe>(
this)
65 ->getCalledScalarFunction()
67 case VPWidenIntrinsicSC:
68 return cast<VPWidenIntrinsicRecipe>(
this)->mayWriteToMemory();
69 case VPBranchOnMaskSC:
70 case VPScalarIVStepsSC:
74 case VPReductionEVLSC:
76 case VPVectorPointerSC:
77 case VPWidenCanonicalIVSC:
80 case VPWidenIntOrFpInductionSC:
81 case VPWidenLoadEVLSC:
86 case VPWidenSelectSC: {
90 assert((!
I || !
I->mayWriteToMemory()) &&
91 "underlying instruction may write to memory");
101 case VPInstructionSC:
102 return cast<VPInstruction>(
this)->opcodeMayReadOrWriteFromMemory();
103 case VPWidenLoadEVLSC:
108 ->mayReadFromMemory();
110 return !cast<VPWidenCallRecipe>(
this)
111 ->getCalledScalarFunction()
112 ->onlyWritesMemory();
113 case VPWidenIntrinsicSC:
114 return cast<VPWidenIntrinsicRecipe>(
this)->mayReadFromMemory();
115 case VPBranchOnMaskSC:
116 case VPPredInstPHISC:
117 case VPScalarIVStepsSC:
118 case VPWidenStoreEVLSC:
122 case VPReductionEVLSC:
124 case VPVectorPointerSC:
125 case VPWidenCanonicalIVSC:
128 case VPWidenIntOrFpInductionSC:
132 case VPWidenSelectSC: {
136 assert((!
I || !
I->mayReadFromMemory()) &&
137 "underlying instruction may read from memory");
148 case VPPredInstPHISC:
150 case VPReverseVectorPointerSC:
152 case VPInstructionSC:
154 case VPWidenCallSC: {
155 Function *Fn = cast<VPWidenCallRecipe>(
this)->getCalledScalarFunction();
158 case VPWidenIntrinsicSC:
159 return cast<VPWidenIntrinsicRecipe>(
this)->mayHaveSideEffects();
161 case VPReductionEVLSC:
163 case VPScalarIVStepsSC:
164 case VPVectorPointerSC:
165 case VPWidenCanonicalIVSC:
168 case VPWidenIntOrFpInductionSC:
170 case VPWidenPointerInductionSC:
173 case VPWidenSelectSC: {
177 assert((!
I || !
I->mayHaveSideEffects()) &&
178 "underlying instruction has side-effects");
183 case VPWidenLoadEVLSC:
185 case VPWidenStoreEVLSC:
190 "mayHaveSideffects result for ingredient differs from this "
193 case VPReplicateSC: {
194 auto *R = cast<VPReplicateRecipe>(
this);
195 return R->getUnderlyingInstr()->mayHaveSideEffects();
203 assert(!Parent &&
"Recipe already in some VPBasicBlock");
205 "Insertion position not in any VPBasicBlock");
211 assert(!Parent &&
"Recipe already in some VPBasicBlock");
217 assert(!Parent &&
"Recipe already in some VPBasicBlock");
219 "Insertion position not in any VPBasicBlock");
251 if (
auto *S = dyn_cast<VPSingleDefRecipe>(
this))
252 UI = dyn_cast_or_null<Instruction>(S->getUnderlyingValue());
253 else if (
auto *IG = dyn_cast<VPInterleaveRecipe>(
this))
254 UI = IG->getInsertPos();
255 else if (
auto *WidenMem = dyn_cast<VPWidenMemoryRecipe>(
this))
256 UI = &WidenMem->getIngredient();
269 dbgs() <<
"Cost of " << RecipeCost <<
" for VF " << VF <<
": ";
281 assert(OpType == OperationType::FPMathOp &&
282 "recipe doesn't have fast math flags");
294#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
298template <
unsigned PartOpIdx>
301 if (U.getNumOperands() == PartOpIdx + 1)
302 return U.getOperand(PartOpIdx);
306template <
unsigned PartOpIdx>
308 if (
auto *UnrollPartOp = getUnrollPartOperand(U))
309 return cast<ConstantInt>(UnrollPartOp->getLiveInIRValue())->getZExtValue();
319 assert(Opcode == Instruction::ICmp &&
320 "only ICmp predicates supported at the moment");
324 std::initializer_list<VPValue *>
Operands,
329 assert(isFPMathOp() &&
"this op can't take fast-math flags");
332bool VPInstruction::doesGeneratePerAllLanes()
const {
336bool VPInstruction::canGenerateScalarForFirstLane()
const {
342 case Instruction::ICmp:
343 case Instruction::Select:
362 "only PtrAdd opcodes are supported for now");
376 if (
auto *
I = dyn_cast<Instruction>(Res))
386 case Instruction::ICmp: {
392 case Instruction::Select: {
415 {VIVElem0, ScalarTC},
nullptr, Name);
431 if (!V1->getType()->isVectorTy())
451 "Requested vector length should be an integer.");
458 {AVL, VFArg, State.Builder.getTrue()});
464 assert(Part != 0 &&
"Must have a positive part");
514 auto *PhiR = cast<VPReductionPHIRecipe>(
getOperand(0));
515 auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
521 Type *PhiTy = OrigPhi->getType();
526 for (
unsigned Part = 0; Part < UF; ++Part)
527 RdxParts[Part] = State.
get(
getOperand(1 + Part), PhiR->isInLoop());
535 for (
unsigned Part = 0; Part < UF; ++Part)
536 RdxParts[Part] = Builder.
CreateTrunc(RdxParts[Part], RdxVecTy);
539 Value *ReducedPartRdx = RdxParts[0];
542 Op = Instruction::Or;
544 if (PhiR->isOrdered()) {
545 ReducedPartRdx = RdxParts[UF - 1];
550 for (
unsigned Part = 1; Part < UF; ++Part) {
551 Value *RdxPart = RdxParts[Part];
552 if (
Op != Instruction::ICmp &&
Op != Instruction::FCmp)
559 ReducedPartRdx =
createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
579 return ReducedPartRdx;
583 unsigned Offset = CI->getZExtValue();
584 assert(
Offset > 0 &&
"Offset from end must be positive");
588 "invalid offset to extract from");
592 assert(
Offset <= 1 &&
"invalid offset to extract from");
595 if (isa<ExtractElementInst>(Res))
606 "can only generate first lane for PtrAdd");
612 Value *IncomingFromVPlanPred =
614 Value *IncomingFromOtherPreds =
621 NewPhi->addIncoming(IncomingFromVPlanPred, VPlanPred);
623 if (OtherPred == VPlanPred)
625 NewPhi->addIncoming(IncomingFromOtherPreds, OtherPred);
650bool VPInstruction::isFPMathOp()
const {
653 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
654 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
655 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
656 Opcode == Instruction::FCmp || Opcode == Instruction::Select;
661 assert(!State.
Lane &&
"VPInstruction executing an Lane");
665 "Recipe not a FPMathOp but has fast-math flags?");
669 bool GeneratesPerFirstLaneOnly = canGenerateScalarForFirstLane() &&
672 bool GeneratesPerAllLanes = doesGeneratePerAllLanes();
673 if (GeneratesPerAllLanes) {
675 Lane != NumLanes; ++Lane) {
676 Value *GeneratedValue = generatePerLane(State,
VPLane(Lane));
677 assert(GeneratedValue &&
"generatePerLane must produce a value");
678 State.
set(
this, GeneratedValue,
VPLane(Lane));
683 Value *GeneratedValue = generate(State);
686 assert(GeneratedValue &&
"generate must produce a value");
690 "scalar value but not only first lane defined");
691 State.
set(
this, GeneratedValue,
692 GeneratesPerFirstLaneOnly);
699 case Instruction::ICmp:
700 case Instruction::Select:
723 case Instruction::ICmp:
724 case Instruction::Select:
725 case Instruction::Or:
749 case Instruction::ICmp:
750 case Instruction::Select:
760#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
768 O << Indent <<
"EMIT ";
780 O <<
"combined load";
783 O <<
"combined store";
786 O <<
"active lane mask";
792 O <<
"EXPLICIT-VECTOR-LENGTH";
795 O <<
"first-order splice";
798 O <<
"branch-on-cond";
801 O <<
"TC > VF ? TC - VF : 0";
807 O <<
"branch-on-count";
810 O <<
"extract-from-end";
813 O <<
"compute-reduction-result";
840 "Only PHINodes can have extra operands");
853 auto *Phi = cast<PHINode>(&I);
856 if (Phi->getBasicBlockIndex(PredBB) == -1)
857 Phi->addIncoming(V, PredBB);
859 Phi->setIncomingValueForBlock(PredBB, V);
874#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
877 O << Indent <<
"IR " << I;
880 O <<
" (extra operand" << (
getNumOperands() > 1 ?
"s" :
"") <<
": ";
911 assert(Variant !=
nullptr &&
"Can't create vector function.");
916 CI->getOperandBundlesAsDefs(OpBundles);
921 if (!V->getType()->isVoidTy())
934#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
937 O << Indent <<
"WIDEN-CALL ";
949 O <<
" @" << CalledFn->
getName() <<
"(";
955 O <<
" (using library function";
957 O <<
": " << Variant->
getName();
991 "Can't retrieve vector intrinsic or vector-predication intrinsics.");
996 CI->getOperandBundlesAsDefs(OpBundles);
1002 if (!V->getType()->isVoidTy())
1018 auto *V =
Op->getUnderlyingValue();
1065#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1068 O << Indent <<
"WIDEN-INTRINSIC ";
1098 Value *Mask =
nullptr;
1100 Mask = State.
get(VPMask);
1107 if (Opcode == Instruction::Sub)
1110 assert(Opcode == Instruction::Add &&
"only add or sub supported for now");
1146 {PtrTy, IncTy, MaskTy});
1154#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1157 O << Indent <<
"WIDEN-HISTOGRAM buckets: ";
1160 if (Opcode == Instruction::Sub)
1163 assert(Opcode == Instruction::Add);
1176 O << Indent <<
"WIDEN-SELECT ";
1202 State.
set(
this, Sel);
1226 [](
VPValue *
Op) {
return Op->getUnderlyingValue(); }))
1227 Operands.append(SI->op_begin(), SI->op_end());
1228 bool IsLogicalOr =
match(
this,
m_LogicalOr(m_VPValue(Op0), m_VPValue(Op1)));
1230 IsLogicalOr ? Instruction::Or : Instruction::And, VectorTy,
CostKind,
1231 {Op1VK, Op1VP}, {Op2VK, Op2VP},
Operands, SI);
1239 if (
auto *Cmp = dyn_cast<CmpInst>(SI->getCondition()))
1240 Pred = Cmp->getPredicate();
1246VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
1257#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1260 case OperationType::Cmp:
1263 case OperationType::DisjointOp:
1267 case OperationType::PossiblyExactOp:
1271 case OperationType::OverflowingBinOp:
1277 case OperationType::FPMathOp:
1280 case OperationType::GEPOp:
1288 case OperationType::NonNegOp:
1292 case OperationType::Other:
1302 auto &Builder = State.
Builder;
1304 case Instruction::Call:
1305 case Instruction::Br:
1306 case Instruction::PHI:
1307 case Instruction::GetElementPtr:
1308 case Instruction::Select:
1310 case Instruction::UDiv:
1311 case Instruction::SDiv:
1312 case Instruction::SRem:
1313 case Instruction::URem:
1314 case Instruction::Add:
1315 case Instruction::FAdd:
1316 case Instruction::Sub:
1317 case Instruction::FSub:
1318 case Instruction::FNeg:
1319 case Instruction::Mul:
1320 case Instruction::FMul:
1321 case Instruction::FDiv:
1322 case Instruction::FRem:
1323 case Instruction::Shl:
1324 case Instruction::LShr:
1325 case Instruction::AShr:
1326 case Instruction::And:
1327 case Instruction::Or:
1328 case Instruction::Xor: {
1336 if (
auto *VecOp = dyn_cast<Instruction>(V))
1344 case Instruction::Freeze: {
1348 State.
set(
this, Freeze);
1351 case Instruction::ICmp:
1352 case Instruction::FCmp: {
1354 bool FCmp = Opcode == Instruction::FCmp;
1382 "inferred type and type from generated instructions do not match");
1390 case Instruction::FNeg: {
1398 case Instruction::UDiv:
1399 case Instruction::SDiv:
1400 case Instruction::SRem:
1401 case Instruction::URem:
1404 case Instruction::Add:
1405 case Instruction::FAdd:
1406 case Instruction::Sub:
1407 case Instruction::FSub:
1408 case Instruction::Mul:
1409 case Instruction::FMul:
1410 case Instruction::FDiv:
1411 case Instruction::FRem:
1412 case Instruction::Shl:
1413 case Instruction::LShr:
1414 case Instruction::AShr:
1415 case Instruction::And:
1416 case Instruction::Or:
1417 case Instruction::Xor: {
1423 if (
RHS->isLiveIn())
1440 case Instruction::Freeze: {
1445 case Instruction::ICmp:
1446 case Instruction::FCmp: {
1468 "VPWidenEVLRecipe should not be used for scalars");
1471 Value *EVLArg = State.
get(EVL,
true);
1487 if (isa<FPMathOperator>(VPInst))
1488 setFlags(cast<Instruction>(VPInst));
1490 State.
set(
this, VPInst);
1495#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1498 O << Indent <<
"WIDEN ";
1507 O << Indent <<
"WIDEN ";
1517 auto &Builder = State.
Builder;
1524 State.
set(
this, Cast);
1526 if (
auto *CastOp = dyn_cast<Instruction>(Cast))
1541 if (isa<VPInterleaveRecipe>(R))
1543 if (
const auto *ReplicateRecipe = dyn_cast<VPReplicateRecipe>(R))
1546 const auto *WidenMemoryRecipe = dyn_cast<VPWidenMemoryRecipe>(R);
1547 if (WidenMemoryRecipe ==
nullptr)
1549 if (!WidenMemoryRecipe->isConsecutive())
1551 if (WidenMemoryRecipe->isReverse())
1553 if (WidenMemoryRecipe->isMasked())
1561 if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) &&
1563 if (
auto *StoreRecipe = dyn_cast<VPRecipeBase>(*
user_begin()))
1564 CCH = ComputeCCH(StoreRecipe);
1567 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
1568 Opcode == Instruction::FPExt) {
1584#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1587 O << Indent <<
"WIDEN-CAST ";
1611 auto *ValVTy = cast<VectorType>(Val->
getType());
1616 "Induction Step must be an integer or FP");
1624 Type *InitVecValSTy =
1635 Step = Builder.
CreateMul(InitVec, Step);
1636 return Builder.
CreateAdd(Val, Step,
"induction");
1640 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
1641 "Binary Opcode should be specified for FP induction");
1646 return Builder.
CreateBinOp(BinOp, Val, MulOp,
"induction");
1653 : ConstantFP::get(Ty,
C);
1657 assert(!State.
Lane &&
"Int or FP induction being replicated.");
1664 "Types must match");
1673 if (
ID.getInductionBinOp() && isa<FPMathOperator>(
ID.getInductionBinOp()))
1679 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
1680 "Expected either an induction phi-node or a truncate of it!");
1683 auto CurrIP = Builder.
saveIP();
1686 if (isa<TruncInst>(EntryVal)) {
1687 assert(Start->getType()->isIntegerTy() &&
1688 "Truncation requires an integer type");
1689 auto *TruncType = cast<IntegerType>(EntryVal->
getType());
1691 Start = Builder.
CreateCast(Instruction::Trunc, Start, TruncType);
1703 AddOp = Instruction::Add;
1704 MulOp = Instruction::Mul;
1706 AddOp =
ID.getInductionOpcode();
1707 MulOp = Instruction::FMul;
1714 SplatVF = State.
get(SplatVFOperand);
1737 State.
set(
this, VecInd);
1740 Builder.
CreateBinOp(AddOp, VecInd, SplatVF,
"vec.ind.next"));
1741 if (isa<TruncInst>(EntryVal))
1754#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1759 O <<
" = WIDEN-INDUCTION ";
1763 O <<
" (truncated to " << *TI->getType() <<
")";
1775 auto *CanIV = cast<VPCanonicalIVPHIRecipe>(&*
getParent()->begin());
1776 return StartC && StartC->isZero() && StepC && StepC->isOne() &&
1780#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1785 O <<
" = DERIVED-IV ";
1809 assert(BaseIVTy == Step->
getType() &&
"Types of BaseIV and Step must match!");
1816 AddOp = Instruction::Add;
1817 MulOp = Instruction::Mul;
1819 AddOp = InductionOpcode;
1820 MulOp = Instruction::FMul;
1829 Type *VecIVTy =
nullptr;
1830 Value *UnitStepVec =
nullptr, *SplatStep =
nullptr, *SplatIV =
nullptr;
1839 unsigned StartLane = 0;
1842 StartLane = State.
Lane->getKnownLane();
1843 EndLane = StartLane + 1;
1850 auto *InitVec = Builder.
CreateAdd(SplatStartIdx, UnitStepVec);
1864 for (
unsigned Lane = StartLane; Lane < EndLane; ++Lane) {
1870 "Expected StartIdx to be folded to a constant when VF is not "
1878#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1883 O <<
" = SCALAR-STEPS ";
1897 if (areAllOperandsInvariant()) {
1933 if (isIndexLoopInvariant(
I - 1))
1944 "NewGEP is not a pointer vector");
1945 State.
set(
this, NewGEP);
1950#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1953 O << Indent <<
"WIDEN-GEP ";
1954 O << (isPointerLoopInvariant() ?
"Inv" :
"Var");
1956 O <<
"[" << (isIndexLoopInvariant(
I) ?
"Inv" :
"Var") <<
"]";
1960 O <<
" = getelementptr";
1971 return IsScalable && (IsReverse || CurrentPart > 0)
1977 auto &Builder = State.
Builder;
1981 CurrentPart, Builder);
1985 if (IndexTy != RunTimeVF->
getType())
1989 ConstantInt::get(IndexTy, -(int64_t)CurrentPart), RunTimeVF);
1991 Value *LastLane = Builder.
CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
1995 ResultPtr = Builder.
CreateGEP(IndexedTy, ResultPtr, LastLane,
"",
1998 State.
set(
this, ResultPtr,
true);
2001#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2006 O <<
" = reverse-vector-pointer";
2013 auto &Builder = State.
Builder;
2017 CurrentPart, Builder);
2024 State.
set(
this, ResultPtr,
true);
2027#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2032 O <<
" = vector-pointer ";
2058 Value *Result =
nullptr;
2059 for (
unsigned In = 0; In < NumIncoming; ++In) {
2072 State.
set(
this, Result, OnlyFirstLaneUsed);
2091#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2094 O << Indent <<
"BLEND ";
2116 assert(!State.
Lane &&
"Reduction being replicated.");
2150 PrevInChain = NewRed;
2151 NextInChain = NewRed;
2157 NewRed, PrevInChain);
2162 State.
set(
this, NextInChain,
true);
2166 assert(!State.
Lane &&
"Reduction being replicated.");
2168 auto &Builder = State.
Builder;
2184 Mask = State.
get(CondOp);
2200 State.
set(
this, NewRed,
true);
2207 auto *VectorTy = cast<VectorType>(
toVectorTy(ElementTy, VF));
2215 "Any-of reduction not implemented in VPlan-based cost model currently.");
2217 (!cast<VPReductionPHIRecipe>(
getOperand(0))->isInLoop() ||
2219 "In-loop reduction not implemented in VPlan-based cost model currently.");
2222 "Inferred type and recurrence type mismatch.");
2237#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2240 O << Indent <<
"REDUCE ";
2255 O <<
" (with final reduction value stored in invariant address sank "
2262 O << Indent <<
"REDUCE ";
2279 O <<
" (with final reduction value stored in invariant address sank "
2288 if (
auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
2289 return any_of(PredR->users(), [PredR](
const VPUser *U) {
2290 return !U->usesScalars(PredR);
2305#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2308 O << Indent << (IsUniform ?
"CLONE " :
"REPLICATE ");
2317 O <<
"@" << CB->getCalledFunction()->getName() <<
"(";
2337 "Codegen only implemented for first lane.");
2339 case Instruction::SExt:
2340 case Instruction::ZExt:
2341 case Instruction::Trunc: {
2352 State.
set(
this, generate(State),
VPLane(0));
2355#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2358 O << Indent <<
"SCALAR-CAST ";
2362 O <<
" to " << *ResultTy;
2367 assert(State.
Lane &&
"Branch on Mask works only on single instance.");
2369 unsigned Lane = State.
Lane->getKnownLane();
2371 Value *ConditionBit =
nullptr;
2374 ConditionBit = State.
get(BlockInMask);
2384 assert(isa<UnreachableInst>(CurrentTerminator) &&
2385 "Expected to replace unreachable terminator with conditional branch.");
2401 assert(State.
Lane &&
"Predicated instruction PHI works per instance.");
2406 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
2408 "operand must be VPReplicateRecipe");
2423 State.
reset(
this, VPhi);
2425 State.
set(
this, VPhi);
2437 Phi->addIncoming(ScalarPredInst, PredicatedBB);
2441 State.
set(
this, Phi, *State.
Lane);
2448#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2451 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
2461 const Align Alignment =
2473 "Inconsecutive memory access should not have the order.");
2494 cast<VectorType>(Ty), {},
CostKind, 0);
2505 auto &Builder = State.
Builder;
2507 Value *Mask =
nullptr;
2508 if (
auto *VPMask =
getMask()) {
2511 Mask = State.
get(VPMask);
2520 "wide.masked.gather");
2532 State.
set(
this, NewLI);
2535#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2538 O << Indent <<
"WIDEN ";
2550 Value *AllTrueMask =
2552 return Builder.
CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,
2553 {Operand, AllTrueMask, EVL},
nullptr,
Name);
2564 auto &Builder = State.
Builder;
2569 Value *Mask =
nullptr;
2571 Mask = State.
get(VPMask);
2581 nullptr,
"wide.masked.gather");
2586 Instruction::Load, DataTy,
Addr,
"vp.op.load"));
2594 State.
set(
this, Res);
2608 const Align Alignment =
2619 cast<VectorType>(Ty), {},
CostKind, 0);
2622#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2625 O << Indent <<
"WIDEN ";
2639 auto &Builder = State.
Builder;
2642 Value *Mask =
nullptr;
2643 if (
auto *VPMask =
getMask()) {
2646 Mask = State.
get(VPMask);
2651 Value *StoredVal = State.
get(StoredVPValue);
2670#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2673 O << Indent <<
"WIDEN store ";
2685 auto &Builder = State.
Builder;
2689 Value *StoredVal = State.
get(StoredValue);
2693 Value *Mask =
nullptr;
2695 Mask = State.
get(VPMask);
2702 if (CreateScatter) {
2704 Intrinsic::vp_scatter,
2705 {StoredVal, Addr, Mask, EVL});
2711 {StoredVal, Addr}));
2729 const Align Alignment =
2740 cast<VectorType>(Ty), {},
CostKind, 0);
2743#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2746 O << Indent <<
"WIDEN vp.store ";
2754 auto VF = DstVTy->getElementCount();
2755 auto *SrcVecTy = cast<VectorType>(V->getType());
2756 assert(VF == SrcVecTy->getElementCount() &&
"Vector dimensions do not match");
2757 Type *SrcElemTy = SrcVecTy->getElementType();
2758 Type *DstElemTy = DstVTy->getElementType();
2759 assert((
DL.getTypeSizeInBits(SrcElemTy) ==
DL.getTypeSizeInBits(DstElemTy)) &&
2760 "Vector elements must have same size");
2771 "Only one type should be a pointer type");
2773 "Only one type should be a floating point type");
2785 unsigned Factor = Vals.
size();
2786 assert(Factor > 1 &&
"Tried to interleave invalid number of vectors");
2790 for (
Value *Val : Vals)
2791 assert(Val->getType() == VecTy &&
"Tried to interleave mismatched types");
2796 if (VecTy->isScalableTy()) {
2798 return Builder.
CreateIntrinsic(WideVecTy, Intrinsic::vector_interleave2,
2807 const unsigned NumElts = VecTy->getElementCount().getFixedValue();
2841 assert(!State.
Lane &&
"Interleave group being replicated.");
2847 unsigned InterleaveFactor = Group->
getFactor();
2853 "Reversed masked interleave-group not supported.");
2857 if (
auto *
I = dyn_cast<Instruction>(ResAddr))
2873 bool InBounds =
false;
2875 InBounds = Gep->isInBounds();
2882 auto CreateGroupMask = [&BlockInMask, &State,
2883 &InterleaveFactor](
Value *MaskForGaps) ->
Value * {
2885 assert(!MaskForGaps &&
"Interleaved groups with gaps are not supported.");
2886 assert(InterleaveFactor == 2 &&
2887 "Unsupported deinterleave factor for scalable vectors");
2888 auto *ResBlockInMask = State.
get(BlockInMask);
2893 MaskTy, Intrinsic::vector_interleave2, Ops,
2894 nullptr,
"interleaved.mask");
2900 Value *ResBlockInMask = State.
get(BlockInMask);
2904 "interleaved.mask");
2906 ShuffledMask, MaskForGaps)
2912 if (isa<LoadInst>(Instr)) {
2913 Value *MaskForGaps =
nullptr;
2914 if (NeedsMaskForGaps) {
2917 assert(MaskForGaps &&
"Mask for Gaps is required but it is null");
2921 if (BlockInMask || MaskForGaps) {
2922 Value *GroupMask = CreateGroupMask(MaskForGaps);
2925 PoisonVec,
"wide.masked.vec");
2933 if (VecTy->isScalableTy()) {
2934 assert(InterleaveFactor == 2 &&
2935 "Unsupported deinterleave factor for scalable vectors");
2940 Intrinsic::vector_deinterleave2, VecTy, NewLoad,
2941 nullptr,
"strided.vec");
2943 for (
unsigned I = 0;
I < InterleaveFactor; ++
I) {
2951 if (Member->getType() != ScalarTy) {
2960 State.
set(VPDefs[J], StridedVec);
2970 for (
unsigned I = 0;
I < InterleaveFactor; ++
I) {
2983 if (Member->getType() != ScalarTy) {
2993 State.
set(VPDefs[J], StridedVec);
3003 Value *MaskForGaps =
3006 "masking gaps for scalable vectors is not yet supported.");
3010 unsigned StoredIdx = 0;
3011 for (
unsigned i = 0; i < InterleaveFactor; i++) {
3013 "Fail to get a member from an interleaved store group");
3023 Value *StoredVec = State.
get(StoredValues[StoredIdx]);
3031 if (StoredVec->
getType() != SubVT)
3040 if (BlockInMask || MaskForGaps) {
3041 Value *GroupMask = CreateGroupMask(MaskForGaps);
3043 IVec, ResAddr, Group->
getAlign(), GroupMask);
3051#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3054 O << Indent <<
"INTERLEAVE-GROUP with factor " << IG->getFactor() <<
" at ";
3055 IG->getInsertPos()->printAsOperand(O,
false);
3065 for (
unsigned i = 0; i < IG->getFactor(); ++i) {
3066 if (!IG->getMember(i))
3069 O <<
"\n" << Indent <<
" store ";
3071 O <<
" to index " << i;
3073 O <<
"\n" << Indent <<
" ";
3075 O <<
" = load from index " << i;
3086 unsigned InsertPosIdx = 0;
3087 for (
unsigned Idx = 0; IG->getFactor(); ++
Idx)
3088 if (
auto *Member = IG->getMember(
Idx)) {
3089 if (Member == InsertPos)
3096 auto *VectorTy = cast<VectorType>(
toVectorTy(ValTy, VF));
3100 unsigned InterleaveFactor = IG->getFactor();
3105 for (
unsigned IF = 0; IF < InterleaveFactor; IF++)
3106 if (IG->getMember(IF))
3111 InsertPos->
getOpcode(), WideVecTy, IG->getFactor(), Indices,
3114 if (!IG->isReverse())
3117 return Cost + IG->getNumMembers() *
3119 VectorTy, std::nullopt,
CostKind, 0);
3122#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3125 O << Indent <<
"EMIT ";
3127 O <<
" = CANONICAL-INDUCTION ";
3133 return IsScalarAfterVectorization &&
3140 "Not a pointer induction according to InductionDescriptor!");
3142 "Unexpected type.");
3144 "Recipe should have been replaced");
3150 Type *ScStValueType = ScalarStartValue->
getType();
3153 PHINode *NewPointerPhi =
nullptr;
3154 if (CurrentPart == 0) {
3155 auto *IVR = cast<VPHeaderPHIRecipe>(&
getParent()
3157 ->getVectorLoopRegion()
3158 ->getEntryBasicBlock()
3160 PHINode *CanonicalIV = cast<PHINode>(State.
get(IVR,
true));
3163 NewPointerPhi->
addIncoming(ScalarStartValue, VectorPH);
3170 NewPointerPhi = cast<PHINode>(
GEP->getPointerOperand());
3183 if (CurrentPart == 0) {
3187 Value *NumUnrolledElems =
3195 NewPointerPhi->
addIncoming(InductionGEP, VectorPH);
3202 RuntimeVF, ConstantInt::get(PhiType, CurrentPart));
3203 Value *StartOffset =
3210 "scalar step must be the same across all parts");
3214 State.
VF, ScalarStepValue)),
3219#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3223 "unexpected number of operands");
3224 O << Indent <<
"EMIT ";
3226 O <<
" = WIDEN-POINTER-INDUCTION ";
3240 assert(!State.
Lane &&
"cannot be used in per-lane");
3248 "Results must match");
3261#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3264 O << Indent <<
"EMIT ";
3266 O <<
" = EXPAND SCEV " << *Expr;
3284 Value *CanonicalVectorIV = Builder.
CreateAdd(VStart, VStep,
"vec.iv");
3285 State.
set(
this, CanonicalVectorIV);
3288#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3291 O << Indent <<
"EMIT ";
3293 O <<
" = WIDEN-CANONICAL-INDUCTION ";
3299 auto &Builder = State.
Builder;
3304 ? VectorInit->getType()
3310 auto *One = ConstantInt::get(IdxTy, 1);
3314 auto *LastIdx = Builder.
CreateSub(RuntimeVF, One);
3322 Phi->addIncoming(VectorInit, VectorPH);
3323 State.
set(
this, Phi);
3342 cast<VectorType>(VectorTy), Mask,
CostKind,
3346#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3349 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
3357 auto &Builder = State.
Builder;
3368 bool ScalarPHI = State.
VF.
isScalar() || IsInLoop;
3374 "recipe must be in the vector loop header");
3377 State.
set(
this, Phi, IsInLoop);
3381 Value *Iden =
nullptr;
3393 StartV = Iden = State.
get(StartVPV);
3416 if (CurrentPart == 0) {
3431 Phi = cast<PHINode>(State.
get(
this, IsInLoop));
3432 Value *StartVal = (CurrentPart == 0) ? StartV : Iden;
3433 Phi->addIncoming(StartVal, VectorPH);
3436#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3439 O << Indent <<
"WIDEN-REDUCTION-PHI ";
3449 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
3454 State.
set(
this, VecPhi);
3457#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3460 O << Indent <<
"WIDEN-PHI ";
3485 Phi->addIncoming(StartMask, VectorPH);
3487 State.
set(
this, Phi);
3490#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3493 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
3501#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3504 O << Indent <<
"EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
3516 Phi->addIncoming(Start, VectorPH);
3518 State.
set(
this, Phi,
true);
3521#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3524 O << Indent <<
"SCALAR-PHI ";
AMDGPU Lower Kernel Arguments
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static cl::opt< TargetTransformInfo::TargetCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(TargetTransformInfo::TCK_RecipThroughput), cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(TargetTransformInfo::TCK_Latency, "latency", "Instruction latency"), clEnumValN(TargetTransformInfo::TCK_CodeSize, "code-size", "Code size"), clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency")))
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
cl::opt< unsigned > ForceTargetInstructionCost("force-target-instruction-cost", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's expected cost for " "an instruction to a single constant value. Mostly " "useful for getting consistent testing."))
mir Rename Register Operands
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
static Instruction * createReverseEVL(IRBuilderBase &Builder, Value *Operand, Value *EVL, const Twine &Name)
Use all-true mask for reverse rather than actual mask, as it avoids a dependence w/o affecting the re...
static Value * interleaveVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name)
Return a vector containing interleaved elements from multiple smaller input vectors.
static Value * createBitOrPointerCast(IRBuilderBase &Builder, Value *V, VectorType *DstVTy, const DataLayout &DL)
cl::opt< unsigned > ForceTargetInstructionCost
static Value * getStepVector(Value *Val, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)
This function adds (0 * Step, 1 * Step, 2 * Step, ...) to each vector element of Val.
static Type * getGEPIndexTy(bool IsScalable, bool IsReverse, unsigned CurrentPart, IRBuilderBase &Builder)
static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)
A helper function that returns an integer or floating-point constant with value C.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
InstListType::iterator iterator
Instruction iterators...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
static StringRef getPredicateName(Predicate P)
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
constexpr bool isVector() const
One or more elements.
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
void print(raw_ostream &O) const
Print fast-math flags to O.
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
ArrayRef< Type * > params() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
bool willReturn() const
Determine if the function will return.
bool doesNotThrow() const
Determine if the function cannot unwind.
Type * getReturnType() const
Returns the type of the ret val.
bool hasNoUnsignedSignedWrap() const
bool hasNoUnsignedWrap() const
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Common base class shared among various IRBuilders.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
ConstantInt * getTrue()
Get the constant value for i1 true.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateFreeze(Value *V, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
BasicBlock * GetInsertBlock() const
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Value * CreateVectorReverse(Value *V, const Twine &Name="")
Return a vector value that contains the vector V reversed.
Value * CreateFCmpFMF(CmpInst::Predicate P, Value *LHS, Value *RHS, FMFSource FMFSource, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
InsertPoint saveIP() const
Returns the current insert point.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateLogicalAnd(Value *Cond1, Value *Cond2, const Twine &Name="")
void restoreIP(InsertPoint IP)
Sets the current insert point to a previously-saved location.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
@ IK_PtrInduction
Pointer induction var. Step = C.
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
static InstructionCost getInvalid(CostType Val=0)
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
void addMetadata(InstTy *NewInst) const
Add metadata (e.g.
BlockT * getHeader() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
FastMathFlags getFastMathFlags() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Type * getRecurrenceType() const
Returns the type of the recurrence.
TrackingVH< Value > getRecurrenceStartValue() const
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
static bool isFindLastIVRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class uses information about analyze scalars to rewrite expressions in canonical form.
Type * getType() const
Return the LLVM type of this SCEV expression.
This class represents the LLVM 'select' instruction.
This class provides computation of slot numbers for LLVM Assembly writing.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
static IntegerType * getInt1Ty(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeID getTypeID() const
Return the type id for the type.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
value_op_iterator value_op_end()
Value * getOperand(unsigned i) const
value_op_iterator value_op_begin()
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenMemoryRecipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account when normalized the first incoming value wi...
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool isNormalized() const
A normalized blend is one that has an odd number of operands, whereby the first operand does not have...
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
const VPBlocksTy & getPredecessors() const
const VPBasicBlock * getEntryBasicBlock() const
VPValue * getMask() const
Return the mask used by this recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPBranchOnMaskRecipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
void dump() const
Dump the VPDef to stderr (for debugging).
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
unsigned getVPDefID() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPValue * getStartValue() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a vectorized histogram operation.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPHistogramRecipe.
VPValue * getMask() const
Return the mask operand if one was provided, or a null pointer if all lanes should be executed uncond...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPIRInstruction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
@ ResumePhi
Creates a scalar phi in a leaf VPBB with a single predecessor in VPlan.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
bool opcodeMayReadOrWriteFromMemory() const
Returns true if the underlying opcode may read from or write to memory.
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
bool isVectorToScalar() const
Returns true if this VPInstruction produces a scalar value from a vector, e.g.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
bool isSingleScalar() const
Returns true if this VPInstruction's operands are single scalars and the result is also a single scal...
void execute(VPTransformState &State) override
Generate the instruction.
VPValue * getAddr() const
Return the address accessed by this recipe.
VPValue * getMask() const
Return the mask used by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
Instruction * getInsertPos() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPInterleaveRecipe.
unsigned getNumStoreOperands() const
Returns the number of stored operands of this interleave group.
static bool isVPIntrinsic(Intrinsic::ID)
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)
static VPLane getFirstLane()
void execute(VPTransformState &State) override
Generates phi nodes for live-outs (from a replicate region) as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
virtual InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const
Compute the cost of this recipe either using a recipe's specialized implementation or using the legac...
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this recipe, taking into account if the cost computation should be skipped and the...
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
NonNegFlagsTy NonNegFlags
GEPNoWrapFlags getGEPNoWrapFlags() const
void setFlags(Instruction *I) const
Set the IR flags for I.
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
DisjointFlagsTy DisjointFlags
bool hasNoUnsignedWrap() const
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
bool hasNoSignedWrap() const
FastMathFlags getFastMathFlags() const
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPValue * getEVL() const
The VPValue of the explicit vector length.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool isConditional() const
Return true if the in-loop reduction is conditional.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of VPReductionRecipe.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
const RecurrenceDescriptor & getRecurrenceDescriptor() const
Return the recurrence decriptor for the in-loop reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
bool isOrdered() const
Return true if the in-loop reduction is ordered.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPReplicateRecipe.
unsigned getOpcode() const
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
LLVM_DUMP_METHOD void dump() const
Print this VPSingleDefRecipe to dbgs() (for debugging).
This class can be used to assign names to VPValues.
LLVMContext & getContext()
Return the LLVMContext used by the analysis.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
VPValue * getUnrollPartOperand(VPUser &U) const
Return the VPValue operand containing the unroll part or null if there is no such operand.
unsigned getUnrollPart(VPUser &U) const
Return the unroll part.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
virtual bool onlyFirstLaneUsed(const VPValue *Op) const
Returns true if the VPUser only uses the first lane of operand Op.
bool isDefinedOutsideLoopRegions() const
Returns true if the VPValue is defined outside any loop region.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
friend class VPInstruction
bool hasMoreThanOneUniqueUser() const
Returns true if the value has more than one unique user.
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
user_iterator user_begin()
unsigned getNumUsers() const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Function * getCalledScalarFunction() const
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCallRecipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
operand_range arg_operands()
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenCastRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override final
Print the recipe.
void execute(VPTransformState &State) override final
Produce a vp-intrinsic using the opcode and operands of the recipe, processing EVL elements.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
PHINode * getPHINode() const
VPValue * getStepValue()
Returns the step value of the induction.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
Type * getScalarType() const
Returns the scalar type of the induction.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getSplatVFValue()
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
StringRef getIntrinsicName() const
Return to name of the intrinsic as string.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Return the scalar return type of the intrinsic.
void execute(VPTransformState &State) override
Produce a widened version of the vector intrinsic.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this vector intrinsic.
bool IsMasked
Whether the memory access is masked.
bool Reverse
Whether the consecutive accessed addresses are in reverse order.
bool isConsecutive() const
Return whether the loaded-from / stored-to addresses are consecutive.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenMemoryRecipe.
bool Consecutive
Whether the accessed addresses are consecutive.
VPValue * getMask() const
Return the mask used by this recipe.
VPValue * getAddr() const
Return the address accessed by this recipe.
bool isReverse() const
Return whether the consecutive loaded/stored addresses are in reverse order.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
VPValue * getFirstUnrolledPartOperand()
Returns the VPValue representing the value of this induction at the first unrolled part,...
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenRecipe.
void execute(VPTransformState &State) override
Produce a widened instruction using the opcode and operands of the recipe, processing State....
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getOpcode() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
VectorBuilder & setEVL(Value *NewExplicitVectorLength)
VectorBuilder & setMask(Value *NewMask)
Value * createVectorInstruction(unsigned Opcode, Type *ReturnTy, ArrayRef< Value * > VecOpArray, const Twine &Name=Twine())
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
static VectorType * getDoubleElementsVectorType(VectorType *VTy)
This static method returns a VectorType with twice as many elements as the input type and the same el...
Type * getElementType() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
const ParentTy * getParent() const
self_iterator getIterator()
base_list_type::iterator iterator
iterator erase(iterator where)
pointer remove(iterator &IT)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})
Look up the Function declaration of the intrinsic id in the Module M.
StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
bool match(Val *V, const Pattern &P)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
bool isUniformAfterVectorization(const VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstPartUsed(const VPValue *Def)
Returns true if only the first part of Def is used.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Value * createSimpleReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)
Create a reduction of the given vector.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
unsigned getLoadStoreAddressSpace(const Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)
Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)
Create a mask that filters the members of an interleave group where there are gaps.
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)
Create a mask with replicated elements.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
Value * createOrderedReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, Value *Start)
Create an ordered reduction intrinsic using the given recurrence descriptor Desc.
Value * createReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, PHINode *OrigPhi=nullptr)
Create a generic reduction using a recurrence descriptor Desc Fast-math-flags are propagated using th...
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
@ SMax
Signed integer max implemented in terms of select(cmp()).
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic has a scalar operand.
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF)
Given information about an recurrence kind, return the identity for the @llvm.vector....
DWARFExpression::Operation Op
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
Type * toVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx, const TargetTransformInfo *TTI)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
This struct is a compact representation of a valid (non-zero power of two) alignment.
Struct to hold various analysis needed for cost computations.
TargetTransformInfo::OperandValueInfo getOperandInfo(VPValue *V) const
Returns the OperandInfo for V, if it is a live-in.
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
const TargetLibraryInfo & TLI
const TargetTransformInfo & TTI
SmallPtrSet< Instruction *, 8 > SkipCostComputation
void execute(VPTransformState &State) override
Generate the phi nodes.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this first-order recurrence phi recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the wide load or gather.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenLoadEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate a wide load or gather.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isInvariantCond() const
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenSelectRecipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
VPValue * getStoredValue() const
Return the address accessed by this recipe.
void execute(VPTransformState &State) override
Generate the wide store or scatter.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const override
Return the cost of this VPWidenStoreEVLRecipe.
VPValue * getEVL() const
Return the EVL operand.
void execute(VPTransformState &State) override
Generate a wide store or scatter.
VPValue * getStoredValue() const
Return the value stored by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.