36 GetIntOrFpInductionDescriptor,
40 Plan->getVectorLoopRegion());
41 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
43 if (!VPBB->getParent())
46 auto EndIter = Term ? Term->getIterator() : VPBB->end();
51 VPValue *VPV = Ingredient.getVPSingleValue();
55 if (
auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
56 auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
57 const auto *
II = GetIntOrFpInductionDescriptor(Phi);
61 VPValue *Start = Plan->getOrAddLiveIn(
II->getStartValue());
65 Phi, Start, Step, &Plan->getVF(), *
II, Ingredient.getDebugLoc());
67 assert(isa<VPInstruction>(&Ingredient) &&
68 "only VPInstructions expected here");
69 assert(!isa<PHINode>(Inst) &&
"phis should be handled above");
71 if (
LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
73 *Load, Ingredient.getOperand(0),
nullptr ,
75 Ingredient.getDebugLoc());
76 }
else if (
StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
78 *Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
79 nullptr ,
false ,
false ,
80 Ingredient.getDebugLoc());
83 }
else if (
CallInst *CI = dyn_cast<CallInst>(Inst)) {
86 {Ingredient.op_begin(), Ingredient.op_end() - 1}, CI->getType(),
88 }
else if (
SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
90 }
else if (
auto *CI = dyn_cast<CastInst>(Inst)) {
92 CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), *CI);
103 "Only recpies with zero or one defined values expected");
104 Ingredient.eraseFromParent();
111 bool Changed =
false;
115 for (
VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Iter)) {
122 for (
auto &Recipe : *VPBB) {
125 dyn_cast_or_null<VPSingleDefRecipe>(
Op->getDefiningRecipe()))
126 WorkList.
insert(std::make_pair(VPBB, Def));
132 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
135 std::tie(SinkTo, SinkCandidate) = WorkList[
I];
136 if (SinkCandidate->
getParent() == SinkTo ||
140 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
141 if (!ScalarVFOnly && RepR->isUniform())
143 }
else if (!isa<VPScalarIVStepsRecipe>(SinkCandidate))
146 bool NeedsDuplicating =
false;
151 auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
152 SinkCandidate](
VPUser *U) {
153 auto *UI = cast<VPRecipeBase>(U);
154 if (UI->getParent() == SinkTo)
156 NeedsDuplicating = UI->onlyFirstLaneUsed(SinkCandidate);
158 return NeedsDuplicating && isa<VPReplicateRecipe>(SinkCandidate);
160 if (!
all_of(SinkCandidate->
users(), CanSinkWithUser))
163 if (NeedsDuplicating) {
170 Clone->insertBefore(SinkCandidate);
172 return cast<VPRecipeBase>(&U)->getParent() != SinkTo;
178 dyn_cast_or_null<VPSingleDefRecipe>(
Op->getDefiningRecipe()))
179 WorkList.
insert(std::make_pair(SinkTo, Def));
188 auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry());
189 if (!EntryBB || EntryBB->size() != 1 ||
190 !isa<VPBranchOnMaskRecipe>(EntryBB->begin()))
193 return cast<VPBranchOnMaskRecipe>(&*EntryBB->begin())->getOperand(0);
198 auto *EntryBB = cast<VPBasicBlock>(R->getEntry());
199 if (EntryBB->getNumSuccessors() != 2)
202 auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]);
203 auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]);
204 if (!Succ0 || !Succ1)
207 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
209 if (Succ0->getSingleSuccessor() == Succ1)
211 if (Succ1->getSingleSuccessor() == Succ0)
226 for (
VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
228 if (!Region1->isReplicator())
230 auto *MiddleBasicBlock =
231 dyn_cast_or_null<VPBasicBlock>(Region1->getSingleSuccessor());
232 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
236 dyn_cast_or_null<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
237 if (!Region2 || !Region2->isReplicator())
242 if (!Mask1 || Mask1 != Mask2)
245 assert(Mask1 && Mask2 &&
"both region must have conditions");
251 if (TransformedRegions.
contains(Region1))
253 auto *MiddleBasicBlock = cast<VPBasicBlock>(Region1->getSingleSuccessor());
254 auto *Region2 = cast<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
258 if (!Then1 || !Then2)
277 cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
278 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
280 return cast<VPRecipeBase>(&U)->getParent() == Then2;
284 if (Phi1ToMove.getVPSingleValue()->getNumUsers() == 0) {
285 Phi1ToMove.eraseFromParent();
288 Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
297 TransformedRegions.
insert(Region1);
300 return !TransformedRegions.
empty();
307 std::string RegionName = (
Twine(
"pred.") + Instr->getOpcodeName()).str();
308 assert(Instr->getParent() &&
"Predicated instruction not in any basic block");
309 auto *BlockInMask = PredRecipe->
getMask();
326 RecipeWithoutMask->getDebugLoc());
346 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
349 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
350 if (RepR->isPredicated())
374 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
379 if (!VPBB->getParent())
382 dyn_cast_or_null<VPBasicBlock>(VPBB->getSinglePredecessor());
383 if (!PredVPBB || PredVPBB->getNumSuccessors() != 1 ||
384 isa<VPIRBasicBlock>(PredVPBB))
390 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(VPBB->getSinglePredecessor());
392 R.moveBefore(*PredVPBB, PredVPBB->
end());
394 auto *ParentRegion = cast_or_null<VPRegionBlock>(VPBB->getParent());
395 if (ParentRegion && ParentRegion->getExiting() == VPBB)
396 ParentRegion->setExiting(PredVPBB);
397 for (
auto *Succ :
to_vector(VPBB->successors())) {
403 return !WorkList.
empty();
410 bool ShouldSimplify =
true;
411 while (ShouldSimplify) {
426 auto *
IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
427 if (!
IV ||
IV->getTruncInst())
438 auto &Casts =
IV->getInductionDescriptor().getCastInsts();
442 for (
auto *U : FindMyCast->
users()) {
443 auto *UserCast = dyn_cast<VPSingleDefRecipe>(U);
444 if (UserCast && UserCast->getUnderlyingValue() == IRCast) {
445 FoundUserCast = UserCast;
449 FindMyCast = FoundUserCast;
461 WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
471 auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
473 if (!WidenOriginalIV || !WidenOriginalIV->isCanonical())
480 if (
any_of(WidenOriginalIV->users(),
481 [WidenOriginalIV](
VPUser *U) {
482 return !U->usesScalars(WidenOriginalIV);
497 auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
498 bool IsConditionalAssume =
499 RepR && RepR->isPredicated() &&
500 match(RepR->getUnderlyingInstr(), m_Intrinsic<Intrinsic::assume>());
501 if (IsConditionalAssume)
504 if (R.mayHaveSideEffects())
508 return all_of(R.definedValues(),
509 [](
VPValue *V) { return V->getNumUsers() == 0; });
535 Kind, FPBinOp, StartV, CanonicalIV, Step,
"offset.idx");
552 if (ResultTy != StepTy) {
567 for (
unsigned I = 0;
I !=
Users.size(); ++
I) {
569 if (isa<VPHeaderPHIRecipe>(Cur))
572 Users.insert(V->user_begin(), V->user_end());
574 return Users.takeVector();
595 auto *PhiR = dyn_cast<VPWidenInductionRecipe>(&Phi);
605 auto *Def = dyn_cast<VPSingleDefRecipe>(U);
606 auto *RepR = dyn_cast<VPReplicateRecipe>(U);
608 if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
609 Def->getNumUsers() == 0 || !Def->getUnderlyingValue() ||
610 (RepR && (RepR->isUniform() || RepR->isPredicated())))
619 Def->operands(),
true);
620 Clone->insertAfter(Def);
621 Def->replaceAllUsesWith(Clone);
626 if (
auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(&Phi)) {
633 VPValue *StepV = PtrIV->getOperand(1);
636 nullptr, StartV, StepV, PtrIV->getDebugLoc(), Builder);
647 auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(&Phi);
648 if (HasOnlyVectorVFs &&
none_of(WideIV->users(), [WideIV](
VPUser *U) {
649 return U->usesScalars(WideIV);
655 Plan,
ID.getKind(),
ID.getInductionOpcode(),
656 dyn_cast_or_null<FPMathOperator>(
ID.getInductionBinOp()),
657 WideIV->getTruncInst(), WideIV->getStartValue(), WideIV->getStepValue(),
658 WideIV->getDebugLoc(), Builder);
661 if (!HasOnlyVectorVFs)
662 WideIV->replaceAllUsesWith(Steps);
664 WideIV->replaceUsesWithIf(Steps, [WideIV](
VPUser &U,
unsigned) {
665 return U.usesScalars(WideIV);
674 auto *WideIV = dyn_cast<VPWidenInductionRecipe>(VPV);
678 auto *IntOrFpIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(WideIV);
679 return (IntOrFpIV && IntOrFpIV->getTruncInst()) ? nullptr : WideIV;
684 if (!Def || Def->getNumOperands() != 2)
686 WideIV = dyn_cast<VPWidenInductionRecipe>(Def->getOperand(0));
688 WideIV = dyn_cast<VPWidenInductionRecipe>(Def->getOperand(1));
692 auto IsWideIVInc = [&]() {
693 using namespace VPlanPatternMatch;
694 auto &
ID = WideIV->getInductionDescriptor();
697 VPValue *IVStep = WideIV->getStepValue();
698 switch (
ID.getInductionOpcode()) {
699 case Instruction::Add:
702 case Instruction::FAdd:
705 case Instruction::FSub:
708 case Instruction::Sub: {
713 m_Binary<Instruction::Sub>(m_VPValue(), m_VPValue(Step))) ||
718 return StepCI && IVStepCI &&
719 StepCI->getValue() == (-1 * IVStepCI->getValue());
728 return IsWideIVInc() ? WideIV :
nullptr;
733 using namespace VPlanPatternMatch;
735 if (ExitVPBBs.
size() != 1)
743 "predecessor must be the middle block");
748 auto *ExitIRI = cast<VPIRInstruction>(&R);
749 if (!isa<PHINode>(ExitIRI->getInstruction()))
753 if (!
match(ExitIRI->getOperand(0),
754 m_VPInstruction<VPInstruction::ExtractFromEnd>(
762 assert(EndValue &&
"end value must have been pre-computed");
765 ExitIRI->setOperand(0, EndValue);
770 VPValue *Step = WideIV->getStepValue();
774 B.createNaryOp(Instruction::Sub, {EndValue, Step}, {},
"ind.escape");
778 Escape =
B.createPtrAdd(EndValue,
779 B.createNaryOp(Instruction::Sub, {Zero, Step}),
782 const auto &
ID = WideIV->getInductionDescriptor();
783 Escape =
B.createNaryOp(
784 ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
787 {EndValue, Step}, {ID.getInductionBinOp()->getFastMathFlags()});
791 ExitIRI->setOperand(0, Escape);
802 auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(&R);
806 auto I = SCEV2VPV.
insert({ExpR->getSCEV(), ExpR});
809 ExpR->replaceAllUsesWith(
I.first->second);
810 ExpR->eraseFromParent();
819 while (!WorkList.
empty()) {
821 if (!Seen.
insert(Cur).second)
828 WorkList.
append(R->op_begin(), R->op_end());
829 R->eraseFromParent();
837 if (
auto *Blend = dyn_cast<VPBlendRecipe>(&R)) {
840 if (Blend->isNormalized() || !
match(Blend->getMask(0), m_False()))
841 UniqueValues.
insert(Blend->getIncomingValue(0));
842 for (
unsigned I = 1;
I != Blend->getNumIncomingValues(); ++
I)
843 if (!
match(Blend->getMask(
I), m_False()))
844 UniqueValues.
insert(Blend->getIncomingValue(
I));
846 if (UniqueValues.
size() == 1) {
847 Blend->replaceAllUsesWith(*UniqueValues.
begin());
848 Blend->eraseFromParent();
852 if (Blend->isNormalized())
858 unsigned StartIndex = 0;
859 for (
unsigned I = 0;
I != Blend->getNumIncomingValues(); ++
I) {
864 if (Mask->getNumUsers() == 1 && !
match(Mask, m_False())) {
871 OperandsWithMask.
push_back(Blend->getIncomingValue(StartIndex));
873 for (
unsigned I = 0;
I != Blend->getNumIncomingValues(); ++
I) {
876 OperandsWithMask.
push_back(Blend->getIncomingValue(
I));
877 OperandsWithMask.
push_back(Blend->getMask(
I));
881 cast<PHINode>(Blend->getUnderlyingValue()), OperandsWithMask);
882 NewBlend->insertBefore(&R);
884 VPValue *DeadMask = Blend->getMask(StartIndex);
886 Blend->eraseFromParent();
893 VPValue *Trunc = R.getVPSingleValue();
896 if (TruncTy == ATy) {
900 if (isa<VPReplicateRecipe>(&R))
904 unsigned ExtOpcode =
match(R.getOperand(0),
m_SExt(m_VPValue()))
909 if (
auto *UnderlyingExt = R.getOperand(0)->getUnderlyingValue()) {
911 VPC->setUnderlyingValue(UnderlyingExt);
913 VPC->insertBefore(&R);
917 VPC->insertBefore(&R);
925 R.getParent()->getPlan()->getCanonicalIV()->getScalarType());
928 auto *R = cast<VPRecipeBase>(U);
929 for (
VPValue *VPV : R->definedValues())
943 X == X1 &&
Y == Y1) {
944 R.getVPSingleValue()->replaceAllUsesWith(
X);
950 return R.getVPSingleValue()->replaceAllUsesWith(
A);
953 return R.getVPSingleValue()->replaceAllUsesWith(
A);
962 return R.getVPSingleValue()->replaceAllUsesWith(R.getOperand(1));
971 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
981 assert(Plan.
hasVF(BestVF) &&
"BestVF is not available in Plan");
982 assert(Plan.
hasUF(BestUF) &&
"BestUF is not available in Plan");
985 auto *Term = &ExitingVPBB->
back();
992 if (!
match(Term, m_BranchOnCount(m_VPValue(), m_VPValue())) &&
994 m_BranchOnCond(
m_Not(m_ActiveLaneMask(m_VPValue(), m_VPValue())))))
998 const SCEV *TripCount =
1000 assert(!isa<SCEVCouldNotCompute>(TripCount) &&
1001 "Trip count SCEV must be computable");
1004 if (TripCount->
isZero() ||
1011 auto *Header = cast<VPBasicBlock>(VectorRegion->
getEntry());
1015 IsaPred<VPCanonicalIVPHIRecipe, VPFirstOrderRecurrencePHIRecipe>)) {
1017 auto *HeaderPhiR = cast<VPHeaderPHIRecipe>(&HeaderR);
1018 HeaderPhiR->replaceAllUsesWith(HeaderPhiR->getStartValue());
1019 HeaderPhiR->eraseFromParent();
1028 B->setParent(
nullptr);
1043 Term->eraseFromParent();
1064 auto TryToPushSinkCandidate = [&](
VPRecipeBase *SinkCandidate) {
1067 if (SinkCandidate == Previous)
1070 if (isa<VPHeaderPHIRecipe>(SinkCandidate) ||
1071 !Seen.
insert(SinkCandidate).second ||
1075 if (SinkCandidate->mayHaveSideEffects())
1084 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
1087 "only recipes with a single defined value expected");
1090 if (!TryToPushSinkCandidate(cast<VPRecipeBase>(
User)))
1102 if (SinkCandidate == FOR)
1105 SinkCandidate->moveAfter(Previous);
1106 Previous = SinkCandidate;
1124 for (
VPUser *U : FOR->users()) {
1125 auto *R = cast<VPRecipeBase>(U);
1130 [&VPDT, HoistPoint](
VPUser *U) {
1131 auto *R = cast<VPRecipeBase>(U);
1132 return HoistPoint == R ||
1133 VPDT.properlyDominates(HoistPoint, R);
1135 "HoistPoint must dominate all users of FOR");
1137 auto NeedsHoisting = [HoistPoint, &VPDT,
1139 VPRecipeBase *HoistCandidate = HoistCandidateV->getDefiningRecipe();
1140 if (!HoistCandidate)
1146 "CFG in VPlan should still be flat, without replicate regions");
1148 if (!Visited.
insert(HoistCandidate).second)
1153 if (!EnclosingLoopRegion || isa<VPHeaderPHIRecipe>(HoistCandidate))
1160 return HoistCandidate;
1174 for (
unsigned I = 0;
I != HoistCandidates.
size(); ++
I) {
1177 "only recipes with a single defined value expected");
1178 if (!CanHoist(Current))
1189 if (
auto *R = NeedsHoisting(
Op))
1201 HoistCandidate->moveBefore(*HoistPoint->
getParent(),
1216 if (
auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
1221 VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
1224 while (
auto *PrevPhi =
1225 dyn_cast_or_null<VPFirstOrderRecurrencePHIRecipe>(Previous)) {
1226 assert(PrevPhi->getParent() == FOR->getParent());
1228 Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
1238 if (isa<VPHeaderPHIRecipe>(Previous))
1244 auto *RecurSplice = cast<VPInstruction>(
1246 {FOR, FOR->getBackedgeValue()}));
1248 FOR->replaceAllUsesWith(RecurSplice);
1251 RecurSplice->setOperand(0, FOR);
1259 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
1268 if (
auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(U)) {
1269 RecWithFlags->dropPoisonGeneratingFlags();
1283 auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
1284 return RepR && RepR->getOpcode() == Instruction::Alloca;
1291 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1294 if (CannotHoistRecipe(R))
1298 if (R.mayHaveSideEffects() || R.mayReadFromMemory() || R.isPhi() ||
1300 return !Op->isDefinedOutsideLoopRegions();
1303 R.moveBefore(*Preheader, Preheader->
end());
1313 unsigned NumProcessedRecipes = 0;
1323 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1330 VPValue *ResultVPV = R.getVPSingleValue();
1332 unsigned NewResSizeInBits = MinBWs.
lookup(UI);
1333 if (!NewResSizeInBits)
1337 NumProcessedRecipes++;
1343 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(&R)) {
1353 if (!
Op->isLiveIn())
1355 auto *UV = dyn_cast_or_null<Instruction>(
Op->getUnderlyingValue());
1358 IsaPred<VPWidenRecipe, VPWidenSelectRecipe>)) {
1361 ProcessedTruncs[
Op] =
nullptr;
1362 NumProcessedRecipes += 1;
1372 (void)OldResSizeInBits;
1380 if (
auto *VPW = dyn_cast<VPRecipeWithIRFlags>(&R))
1381 VPW->dropPoisonGeneratingFlags();
1384 if (OldResSizeInBits != NewResSizeInBits &&
1385 !
match(&R, m_Binary<Instruction::ICmp>(m_VPValue(), m_VPValue()))) {
1389 Ext->insertAfter(&R);
1391 Ext->setOperand(0, ResultVPV);
1392 assert(OldResSizeInBits > NewResSizeInBits &&
"Nothing to shrink?");
1395 match(&R, m_Binary<Instruction::ICmp>(m_VPValue(), m_VPValue())) &&
1396 "Only ICmps should not need extending the result.");
1399 assert(!isa<VPWidenStoreRecipe>(&R) &&
"stores cannot be narrowed");
1400 if (isa<VPWidenLoadRecipe>(&R))
1404 unsigned StartIdx = isa<VPWidenSelectRecipe>(&R) ? 1 : 0;
1405 for (
unsigned Idx = StartIdx;
Idx != R.getNumOperands(); ++
Idx) {
1406 auto *
Op = R.getOperand(
Idx);
1407 unsigned OpSizeInBits =
1409 if (OpSizeInBits == NewResSizeInBits)
1411 assert(OpSizeInBits > NewResSizeInBits &&
"nothing to truncate");
1412 auto [ProcessedIter, IterIsEmpty] =
1413 ProcessedTruncs.
insert({
Op,
nullptr});
1417 : ProcessedIter->second;
1418 R.setOperand(
Idx, NewOp);
1421 ProcessedIter->second = NewOp;
1422 if (!
Op->isLiveIn()) {
1427 auto *OpInst = dyn_cast<Instruction>(
Op->getLiveInIRValue());
1428 bool IsContained = MinBWs.
contains(OpInst);
1429 NumProcessedRecipes += IsContained;
1437 assert(MinBWs.
size() == NumProcessedRecipes &&
1438 "some entries in MinBWs haven't been processed");
1495 VPValue *StartV = CanonicalIVPHI->getStartValue();
1497 auto *CanonicalIVIncrement =
1498 cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
1501 CanonicalIVIncrement->dropPoisonGeneratingFlags();
1502 DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
1512 VPValue *TripCount, *IncrementValue;
1517 IncrementValue = CanonicalIVIncrement;
1523 IncrementValue = CanonicalIVPHI;
1534 DL,
"active.lane.mask.entry");
1539 LaneMaskPhi->insertAfter(CanonicalIVPHI);
1545 auto *InLoopIncrement =
1547 {IncrementValue}, {
false,
false},
DL);
1549 {InLoopIncrement, TripCount},
DL,
1550 "active.lane.mask.next");
1567 auto *FoundWidenCanonicalIVUser =
1569 [](
VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); });
1571 [](
VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); }) <=
1573 "Must have at most one VPWideCanonicalIVRecipe");
1575 auto *WideCanonicalIV =
1576 cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
1577 WideCanonicalIVs.
push_back(WideCanonicalIV);
1584 auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
1585 if (WidenOriginalIV && WidenOriginalIV->isCanonical())
1586 WideCanonicalIVs.
push_back(WidenOriginalIV);
1592 for (
auto *Wide : WideCanonicalIVs) {
1594 auto *HeaderMask = dyn_cast<VPInstruction>(U);
1598 assert(HeaderMask->getOperand(0) == Wide &&
1599 "WidenCanonicalIV must be the first operand of the compare");
1607 VPlan &Plan,
bool UseActiveLaneMaskForControlFlow,
1610 UseActiveLaneMaskForControlFlow) &&
1611 "DataAndControlFlowWithoutRuntimeCheck implies "
1612 "UseActiveLaneMaskForControlFlow");
1614 auto *FoundWidenCanonicalIVUser =
1616 [](
VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); });
1617 assert(FoundWidenCanonicalIVUser &&
1618 "Must have widened canonical IV when tail folding!");
1619 auto *WideCanonicalIV =
1620 cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
1622 if (UseActiveLaneMaskForControlFlow) {
1629 "active.lane.mask");
1636 HeaderMask->replaceAllUsesWith(LaneMask);
1653 assert(OrigMask &&
"Unmasked recipe when folding tail");
1654 return HeaderMask == OrigMask ? nullptr : OrigMask;
1659 VPValue *NewMask = GetNewMask(L->getMask());
1667 unsigned Opcode = W->getOpcode();
1673 VPValue *NewMask = GetNewMask(Red->getCondOp());
1676 .Case<VPWidenIntrinsicRecipe, VPWidenCastRecipe>(
1679 if (
auto *CallR = dyn_cast<VPWidenIntrinsicRecipe>(CR)) {
1683 auto *CastR = cast<VPWidenCastRecipe>(CR);
1692 "Expected VP intrinsic to have mask and EVL");
1719 Intrinsic::vp_merge, {&AllOneMask,
LHS,
RHS, &EVL},
1733 if (
auto *R = dyn_cast<VPReverseVectorPointerRecipe>(U))
1734 R->setOperand(1, &EVL);
1741 auto *CurRecipe = cast<VPRecipeBase>(U);
1748 assert(NumDefVal == CurRecipe->getNumDefinedValues() &&
1749 "New recipe must define the same number of values as the "
1753 "Only supports recipes with a single definition or without users.");
1755 if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(EVLRecipe)) {
1756 VPValue *CurVPV = CurRecipe->getVPSingleValue();
1767 R->eraseFromParent();
1813 VPlan &Plan,
const std::optional<unsigned> &MaxSafeElements) {
1818 bool ContainsWidenInductions =
any_of(
1820 IsaPred<VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe>);
1821 if (ContainsWidenInductions)
1825 VPValue *StartV = CanonicalIVPHI->getStartValue();
1829 EVLPhi->insertAfter(CanonicalIVPHI);
1830 VPBuilder Builder(Header, Header->getFirstNonPhi());
1834 if (MaxSafeElements) {
1837 ConstantInt::get(CanonicalIVPHI->getScalarType(), *MaxSafeElements));
1844 auto *CanonicalIVIncrement =
1845 cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
1847 if (
unsigned IVSize = CanonicalIVPHI->getScalarType()->getScalarSizeInBits();
1850 IVSize < 32 ? Instruction::Trunc : Instruction::ZExt, OpVPEVL,
1851 CanonicalIVPHI->getScalarType(), CanonicalIVIncrement->
getDebugLoc());
1856 {CanonicalIVIncrement->hasNoUnsignedWrap(),
1857 CanonicalIVIncrement->hasNoSignedWrap()},
1858 CanonicalIVIncrement->
getDebugLoc(),
"index.evl.next");
1859 NextEVLIV->insertBefore(CanonicalIVIncrement);
1860 EVLPhi->addOperand(NextEVLIV);
1866 CanonicalIVPHI->replaceAllUsesWith(EVLPhi);
1867 CanonicalIVIncrement->setOperand(0, CanonicalIVPHI);
1878 auto CollectPoisonGeneratingInstrsInBackwardSlice([&](
VPRecipeBase *Root) {
1883 while (!Worklist.
empty()) {
1884 VPRecipeBase *CurRec = Worklist.pop_back_val();
1886 if (!Visited.insert(CurRec).second)
1893 if (isa<VPWidenMemoryRecipe, VPInterleaveRecipe, VPScalarIVStepsRecipe,
1894 VPHeaderPHIRecipe>(CurRec))
1900 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(CurRec)) {
1902 using namespace llvm::VPlanPatternMatch;
1908 if (match(RecWithFlags, m_BinaryOr(m_VPValue(A), m_VPValue(B))) &&
1909 RecWithFlags->isDisjoint()) {
1910 VPBuilder Builder(RecWithFlags);
1911 VPInstruction *New = Builder.createOverflowingOp(
1912 Instruction::Add, {A, B}, {false, false},
1913 RecWithFlags->getDebugLoc());
1914 New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
1915 RecWithFlags->replaceAllUsesWith(New);
1916 RecWithFlags->eraseFromParent();
1919 RecWithFlags->dropPoisonGeneratingFlags();
1921 Instruction *Instr = dyn_cast_or_null<Instruction>(
1922 CurRec->getVPSingleValue()->getUnderlyingValue());
1924 assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
1925 "found instruction with poison generating flags not covered by "
1926 "VPRecipeWithIRFlags");
1940 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
1942 if (
auto *WidenRec = dyn_cast<VPWidenMemoryRecipe>(&Recipe)) {
1943 Instruction &UnderlyingInstr = WidenRec->getIngredient();
1944 VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
1945 if (AddrDef && WidenRec->isConsecutive() &&
1946 BlockNeedsPredication(UnderlyingInstr.
getParent()))
1947 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
1948 }
else if (
auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(&Recipe)) {
1949 VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
1953 InterleaveRec->getInterleaveGroup();
1954 bool NeedPredication =
false;
1956 I < NumMembers; ++
I) {
1959 NeedPredication |= BlockNeedsPredication(Member->getParent());
1962 if (NeedPredication)
1963 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
1975 if (InterleaveGroups.empty())
1983 for (
const auto *IG : InterleaveGroups) {
1985 for (
unsigned i = 0; i < IG->getFactor(); ++i)
1986 if (
auto *SI = dyn_cast_or_null<StoreInst>(IG->getMember(i))) {
1987 auto *StoreR = cast<VPWidenStoreRecipe>(RecipeBuilder.
getRecipe(SI));
1988 StoredValues.
push_back(StoreR->getStoredValue());
1991 bool NeedsMaskForGaps =
1992 IG->requiresScalarEpilogue() && !ScalarEpilogueAllowed;
1996 cast<VPWidenMemoryRecipe>(RecipeBuilder.
getRecipe(IRInsertPos));
2000 cast<VPWidenMemoryRecipe>(RecipeBuilder.
getRecipe(IG->getMember(0)));
2006 bool InBounds =
false;
2007 if (
auto *Gep = dyn_cast<GetElementPtrInst>(
2009 InBounds = Gep->isInBounds();
2015 assert(IG->getIndex(IRInsertPos) != 0 &&
2016 "index of insert position shouldn't be zero");
2020 IG->getIndex(IRInsertPos),
2025 Addr = InBounds ?
B.createInBoundsPtrAdd(InsertPos->getAddr(), OffsetVPV)
2026 :
B.createPtrAdd(InsertPos->getAddr(), OffsetVPV);
2029 InsertPos->getMask(), NeedsMaskForGaps);
2030 VPIG->insertBefore(InsertPos);
2033 for (
unsigned i = 0; i < IG->getFactor(); ++i)
2036 if (!Member->getType()->isVoidTy()) {
2047 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2050 if (!isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(&R))
2052 auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
2054 isa<VPCanonicalIVPHIRecipe>(PhiR) ?
"index" :
"evl.based.iv";
2057 PhiR->getDebugLoc(),
Name);
2058 ScalarR->insertBefore(PhiR);
2059 PhiR->replaceAllUsesWith(ScalarR);
2060 PhiR->eraseFromParent();
2069 auto *LatchVPBB = cast<VPBasicBlock>(LoopRegion->
getExiting());
2070 VPBuilder Builder(LatchVPBB->getTerminator());
2072 VPValue *IsEarlyExitTaken =
nullptr;
2078 auto *EarlyExitingBranch =
2080 BasicBlock *TrueSucc = EarlyExitingBranch->getSuccessor(0);
2081 BasicBlock *FalseSucc = EarlyExitingBranch->getSuccessor(1);
2089 VPEarlyExitBlock = cast<VPIRBasicBlock>(MiddleVPBB->getSuccessors()[0]);
2092 !OrigLoop->
contains(TrueSucc) ? TrueSucc : FalseSucc);
2096 OrigLoop->
contains(TrueSucc) ? TrueSucc : FalseSucc);
2097 auto *EarlyExitTakenCond = Builder.
createNot(EarlyExitNotTakenCond);
2112 auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
2114 "Unexpected terminator");
2115 auto *IsLatchExitTaken =
2117 LatchExitingBranch->getOperand(1));
2119 Instruction::Or, {IsEarlyExitTaken, IsLatchExitTaken});
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
iv Induction Variable Users
static bool mergeBlocksIntoPredecessors(Loop &L, DominatorTree &DT, LoopInfo &LI, MemorySSAUpdater *MSSAU, ScalarEvolution &SE)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file implements the TypeSwitch template, which mimics a switch() statement whose cases are type ...
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
Class for arbitrary precision integers.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
This class represents a function call, abstracting a target machine's calling convention.
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
static ConstantInt * getTrue(LLVMContext &Context)
This class represents an Operation in the Expression.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
static constexpr ElementCount getFixed(ScalarTy MinVal)
Utility class for floating point operations which can have information about relaxed accuracy require...
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
A struct for saving information about induction variables.
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_PtrInduction
Pointer induction var. Step = C.
@ IK_IntInduction
Integer induction variable. Step = C.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
The group of interleaved loads/stores sharing the same stride and close to each other.
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
uint32_t getNumMembers() const
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getUniqueExitBlock() const
If getUniqueExitBlocks would return exactly one block, return that block.
Represents a single loop in the control flow graph.
This class implements a map that also provides access to all stored values in a deterministic order.
bool contains(const KeyT &Key) const
ValueT lookup(const KeyT &Key) const
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
RecurKind getRecurrenceKind() const
This class represents an analyzed expression in the program.
bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
The main scalar evolution driver.
const SCEV * getElementCount(Type *Ty, ElementCount EC)
bool isKnownPredicate(CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVMContext & getContext() const
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This class implements a switch-like dispatch statement for a value of 'T' using dyn_cast functionalit...
TypeSwitch< T, ResultT > & Case(CallableT &&caseFn)
Add a case on the given type.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isPointerTy() const
True if this is an instance of PointerType.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPRegionBlock * getEnclosingLoopRegion()
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPRecipeBase & back() const
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
void swapSuccessors()
Swap successors of the block. The block must have exactly 2 successors.
VPBlockBase * getSinglePredecessor() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleHierarchicalPredecessor()
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
static void insertOnEdge(VPBlockBase *From, VPBlockBase *To, VPBlockBase *BlockPtr)
Inserts BlockPtr on the edge between From and To.
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To, unsigned PredIdx=-1u, unsigned SuccIdx=-1u)
Connect VPBlockBases From and To bi-directionally.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
A recipe for generating conditional branches on the bits of a mask.
RAII object that stores the current insertion point and restores it when the object is destroyed.
VPlan-based builder utility analogous to IRBuilder.
VPValue * createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B, DebugLoc DL={}, const Twine &Name="")
Create a new ICmp VPInstruction with predicate Pred and operands A and B.
VPDerivedIVRecipe * createDerivedIV(InductionDescriptor::InductionKind Kind, FPMathOperator *FPBinOp, VPValue *Start, VPValue *Current, VPValue *Step, const Twine &Name="")
Convert the input value Current to the corresponding value of an induction with Start and Step values...
VPInstruction * createPtrAdd(VPValue *Ptr, VPValue *Offset, DebugLoc DL={}, const Twine &Name="")
VPScalarCastRecipe * createScalarCast(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, DebugLoc DL)
static VPBuilder getToInsertAfter(VPRecipeBase *R)
Create a VPBuilder to insert after R.
VPScalarIVStepsRecipe * createScalarIVSteps(Instruction::BinaryOps InductionOpcode, FPMathOperator *FPBinOp, VPValue *IV, VPValue *Step)
VPInstruction * createOverflowingOp(unsigned Opcode, std::initializer_list< VPValue * > Operands, VPRecipeWithIRFlags::WrapFlagsTy WrapFlags, DebugLoc DL={}, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPValue * createNot(VPValue *Operand, DebugLoc DL={}, const Twine &Name="")
VPValue * createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL={}, const Twine &Name="", std::optional< FastMathFlags > FMFs=std::nullopt)
void setInsertPoint(VPBasicBlock *TheBB)
This specifies that created VPInstructions should be appended to the end of the specified block.
Canonical scalar induction phi of the vector loop.
Type * getScalarType() const
Returns the scalar type of the induction.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Template specialization of the standard LLVM dominator tree utility for VPBlockBases.
bool properlyDominates(const VPRecipeBase *A, const VPRecipeBase *B)
Returns true if A properly dominates B.
A recipe for generating the phi node for the current index of elements, adjusted in accordance with E...
A special type of VPBasicBlock that wraps an existing IR basic block.
This is a concrete Recipe that models a single VPlan-level instruction.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
static std::optional< unsigned > getMaskParamPos(Intrinsic::ID IntrinsicID)
static std::optional< unsigned > getVectorLengthParamPos(Intrinsic::ID IntrinsicID)
static Intrinsic::ID getForOpcode(unsigned OC)
The llvm.vp.* intrinsics for this instruction Opcode.
static Intrinsic::ID getForIntrinsic(Intrinsic::ID Id)
The llvm.vp.
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Helper class to create VPRecipies from IR instructions.
VPValue * getBlockInMask(BasicBlock *BB) const
Returns the entry mask for the block BB.
VPRecipeBase * getRecipe(Instruction *I)
Return the recipe created for given ingredient.
A recipe to represent inloop reduction operations with vector-predication intrinsics,...
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
const VPBlockBase * getExiting() const
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
VPScalarCastRecipe is a recipe to create scalar cast instructions.
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
Recipe to generate a scalar PHI.
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
An analysis for type-inference for VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
operand_iterator op_end()
operand_iterator op_begin()
void addOperand(VPValue *Operand)
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
void replaceAllUsesWith(VPValue *New)
unsigned getNumUsers() const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
A Recipe for widening the canonical induction variable of the vector loop.
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for widening operations with vector-predication intrinsics with explicit vector length (EVL)...
A recipe for handling GEP instructions.
Base class for widened induction (VPWidenIntOrFpInductionRecipe and VPWidenPointerInductionRecipe),...
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
A recipe for widening vector intrinsics.
VPValue * getMask() const
Return the mask used by this recipe.
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPBasicBlock * getEntry()
VPRegionBlock * createVPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting, const std::string &Name="", bool IsReplicator=false)
Create a new VPRegionBlock with Entry, Exiting and Name.
VPValue & getVF()
Returns the VF of the vector loop region.
VPValue * getTripCount() const
The trip count of the original loop.
bool hasVF(ElementCount VF)
bool hasUF(unsigned UF) const
void setVF(ElementCount VF)
auto getExitBlocks()
Return an iterator range over the VPIRBasicBlock wrapping the exit blocks of the VPlan,...
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
const VPBasicBlock * getMiddleBlock() const
Returns the 'middle' block of the plan, that is the block that selects whether to execute the scalar ...
VPBasicBlock * createVPBasicBlock(const Twine &Name, VPRecipeBase *Recipe=nullptr)
Create a new VPBasicBlock with Name and containing Recipe if present.
VPIRBasicBlock * createVPIRBasicBlock(BasicBlock *IRBB)
Create a VPIRBasicBlock from IRBB containing VPIRInstructions for all instructions in IRBB,...
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
bool hasScalarVFOnly() const
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
VPBasicBlock * getVectorPreheader()
Returns the preheader of the vector loop region, if one exists, or null otherwise.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
StringRef getName() const
Return a constant reference to the value's name.
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
BinaryOp_match< LHS, RHS, Instruction::Mul, true > m_c_Mul(const LHS &L, const RHS &R)
Matches a Mul with LHS and RHS in either order.
bool isUniformAfterVectorization(const VPValue *VPV)
Returns true if VPV is uniform after vectorization.
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
const SCEV * getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE)
Return the SCEV expression for V.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
bool isHeaderMask(const VPValue *V, VPlan &Plan)
Return true if V is a header mask in Plan.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
iterator_range< df_iterator< VPBlockShallowTraversalWrapper< VPBlockBase * > > > vp_depth_first_shallow(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order.
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
std::unique_ptr< VPlan > VPlanPtr
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
DWARFExpression::Operation Op
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Type * getLoadStoreType(const Value *I)
A helper function that returns the type of a load or store instruction.
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
@ Default
The result values are uniform if and only if all operands are uniform.
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
A recipe for handling first-order recurrence phis.
A recipe for widening load operations with vector-predication intrinsics, using the address to load f...
A recipe for widening load operations, using the address to load from and an optional mask.
A recipe for widening select instructions.
A recipe for widening store operations with vector-predication intrinsics, using the value to store,...
A recipe for widening store operations, using the stored value, the address to store to and an option...