33 GetIntOrFpInductionDescriptor,
37 Plan->getVectorLoopRegion());
38 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
40 if (!VPBB->getParent())
43 auto EndIter = Term ? Term->getIterator() : VPBB->end();
48 VPValue *VPV = Ingredient.getVPSingleValue();
52 if (
auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
53 auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
54 const auto *
II = GetIntOrFpInductionDescriptor(Phi);
58 VPValue *Start = Plan->getOrAddLiveIn(
II->getStartValue());
63 assert(isa<VPInstruction>(&Ingredient) &&
64 "only VPInstructions expected here");
65 assert(!isa<PHINode>(Inst) &&
"phis should be handled above");
67 if (
LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
69 *Load, Ingredient.getOperand(0),
nullptr ,
71 Ingredient.getDebugLoc());
72 }
else if (
StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
74 *Store, Ingredient.getOperand(1), Ingredient.getOperand(0),
75 nullptr ,
false ,
false ,
76 Ingredient.getDebugLoc());
79 }
else if (
CallInst *CI = dyn_cast<CallInst>(Inst)) {
83 }
else if (
SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
85 }
else if (
auto *CI = dyn_cast<CastInst>(Inst)) {
87 CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), *CI);
98 "Only recpies with zero or one defined values expected");
99 Ingredient.eraseFromParent();
106 bool Changed =
false;
110 for (
VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Iter)) {
117 for (
auto &Recipe : *VPBB) {
120 dyn_cast_or_null<VPSingleDefRecipe>(
Op->getDefiningRecipe()))
121 WorkList.
insert(std::make_pair(VPBB, Def));
127 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
130 std::tie(SinkTo, SinkCandidate) = WorkList[
I];
131 if (SinkCandidate->
getParent() == SinkTo ||
135 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
136 if (!ScalarVFOnly && RepR->isUniform())
138 }
else if (!isa<VPScalarIVStepsRecipe>(SinkCandidate))
141 bool NeedsDuplicating =
false;
146 auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
147 SinkCandidate](
VPUser *U) {
148 auto *UI = dyn_cast<VPRecipeBase>(U);
151 if (UI->getParent() == SinkTo)
153 NeedsDuplicating = UI->onlyFirstLaneUsed(SinkCandidate);
155 return NeedsDuplicating && isa<VPReplicateRecipe>(SinkCandidate);
157 if (!
all_of(SinkCandidate->
users(), CanSinkWithUser))
160 if (NeedsDuplicating) {
167 Clone->insertBefore(SinkCandidate);
169 return cast<VPRecipeBase>(&U)->getParent() != SinkTo;
175 dyn_cast_or_null<VPSingleDefRecipe>(
Op->getDefiningRecipe()))
176 WorkList.
insert(std::make_pair(SinkTo, Def));
185 auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry());
186 if (!EntryBB || EntryBB->size() != 1 ||
187 !isa<VPBranchOnMaskRecipe>(EntryBB->begin()))
190 return cast<VPBranchOnMaskRecipe>(&*EntryBB->begin())->getOperand(0);
195 auto *EntryBB = cast<VPBasicBlock>(R->getEntry());
196 if (EntryBB->getNumSuccessors() != 2)
199 auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]);
200 auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]);
201 if (!Succ0 || !Succ1)
204 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
206 if (Succ0->getSingleSuccessor() == Succ1)
208 if (Succ1->getSingleSuccessor() == Succ0)
223 for (
VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
225 if (!Region1->isReplicator())
227 auto *MiddleBasicBlock =
228 dyn_cast_or_null<VPBasicBlock>(Region1->getSingleSuccessor());
229 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
233 dyn_cast_or_null<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
234 if (!Region2 || !Region2->isReplicator())
239 if (!Mask1 || Mask1 != Mask2)
242 assert(Mask1 && Mask2 &&
"both region must have conditions");
248 if (DeletedRegions.
contains(Region1))
250 auto *MiddleBasicBlock = cast<VPBasicBlock>(Region1->getSingleSuccessor());
251 auto *Region2 = cast<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
255 if (!Then1 || !Then2)
274 cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
275 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
277 auto *UI = dyn_cast<VPRecipeBase>(&U);
278 return UI && UI->getParent() == Then2;
281 Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
290 DeletedRegions.
insert(Region1);
295 return !DeletedRegions.
empty();
302 std::string RegionName = (
Twine(
"pred.") + Instr->getOpcodeName()).str();
303 assert(Instr->getParent() &&
"Predicated instruction not in any basic block");
304 auto *BlockInMask = PredRecipe->
getMask();
336 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
339 if (
auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
340 if (RepR->isPredicated())
366 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
371 if (VPBB->getNumSuccessors() == 0 && !VPBB->getParent())
374 dyn_cast_or_null<VPBasicBlock>(VPBB->getSinglePredecessor());
375 if (!PredVPBB || PredVPBB->getNumSuccessors() != 1)
381 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(VPBB->getSinglePredecessor());
383 R.moveBefore(*PredVPBB, PredVPBB->
end());
385 auto *ParentRegion = cast_or_null<VPRegionBlock>(VPBB->getParent());
386 if (ParentRegion && ParentRegion->getExiting() == VPBB)
387 ParentRegion->setExiting(PredVPBB);
388 for (
auto *Succ :
to_vector(VPBB->successors())) {
394 return !WorkList.
empty();
401 bool ShouldSimplify =
true;
402 while (ShouldSimplify) {
417 auto *
IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
418 if (!
IV ||
IV->getTruncInst())
429 auto &Casts =
IV->getInductionDescriptor().getCastInsts();
433 for (
auto *U : FindMyCast->
users()) {
434 auto *UserCast = dyn_cast<VPSingleDefRecipe>(U);
435 if (UserCast && UserCast->getUnderlyingValue() == IRCast) {
436 FoundUserCast = UserCast;
440 FindMyCast = FoundUserCast;
452 WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
462 auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
464 if (!WidenOriginalIV || !WidenOriginalIV->isCanonical())
471 if (
any_of(WidenOriginalIV->users(),
472 [WidenOriginalIV](
VPUser *U) {
473 return !U->usesScalars(WidenOriginalIV);
488 auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
489 bool IsConditionalAssume =
490 RepR && RepR->isPredicated() &&
491 match(RepR->getUnderlyingInstr(), m_Intrinsic<Intrinsic::assume>());
492 if (IsConditionalAssume)
495 if (R.mayHaveSideEffects())
499 return all_of(R.definedValues(),
500 [](
VPValue *V) { return V->getNumUsers() == 0; });
526 if (!CanonicalIV->
isCanonical(Kind, StartV, Step)) {
528 HeaderVPBB->
insert(BaseIV, IP);
541 HeaderVPBB->
insert(BaseIV, IP);
547 if (ResultTy != StepTy) {
558 BaseIV, Step, InductionOpcode,
560 HeaderVPBB->
insert(Steps, IP);
581 if (
auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(&Phi)) {
588 VPValue *StepV = PtrIV->getOperand(1);
591 SE,
nullptr, StartV, StepV, InsertPt);
594 {PtrIV->getStartValue(), Steps},
597 Recipe->insertAfter(Steps);
598 PtrIV->replaceAllUsesWith(Recipe);
604 auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
607 if (HasOnlyVectorVFs &&
none_of(WideIV->users(), [WideIV](
VPUser *U) {
608 return U->usesScalars(WideIV);
614 Plan,
ID.getKind(),
ID.getInductionOpcode(),
615 dyn_cast_or_null<FPMathOperator>(
ID.getInductionBinOp()), SE,
616 WideIV->getTruncInst(), WideIV->getStartValue(), WideIV->getStepValue(),
620 if (!HasOnlyVectorVFs)
621 WideIV->replaceAllUsesWith(Steps);
623 WideIV->replaceUsesWithIf(Steps, [WideIV](
VPUser &U,
unsigned) {
624 return U.usesScalars(WideIV);
636 auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(&R);
640 auto I = SCEV2VPV.
insert({ExpR->getSCEV(), ExpR});
643 ExpR->replaceAllUsesWith(
I.first->second);
644 ExpR->eraseFromParent();
653 while (!WorkList.
empty()) {
655 if (!Seen.
insert(Cur).second)
662 WorkList.
append(R->op_begin(), R->op_end());
663 R->eraseFromParent();
670 assert(Plan.
hasVF(BestVF) &&
"BestVF is not available in Plan");
671 assert(Plan.
hasUF(BestUF) &&
"BestUF is not available in Plan");
674 auto *Term = &ExitingVPBB->
back();
681 if (!
match(Term, m_BranchOnCount(m_VPValue(), m_VPValue())) &&
683 m_BranchOnCond(
m_Not(m_ActiveLaneMask(m_VPValue(), m_VPValue())))))
692 if (TripCount->
isZero() ||
702 Term->eraseFromParent();
715 auto *
Region = dyn_cast_or_null<VPRegionBlock>(R->getParent()->getParent());
718 Region->getNumPredecessors() == 1 &&
"Expected SESE region!");
719 assert(R->getParent()->size() == 1 &&
720 "A recipe in an original replicator region must be the only "
721 "recipe in its block");
734 for (
auto &R : *
A->getParent()) {
744 if (ParentA == ParentB)
745 return LocalComesBefore(
A,
B);
748 "No replicate regions expected at this point");
750 "No replicate regions expected at this point");
765 auto TryToPushSinkCandidate = [&](
VPRecipeBase *SinkCandidate) {
768 if (SinkCandidate == Previous)
771 if (isa<VPHeaderPHIRecipe>(SinkCandidate) ||
772 !Seen.
insert(SinkCandidate).second ||
776 if (SinkCandidate->mayHaveSideEffects())
785 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
788 "only recipes with a single defined value expected");
791 if (
auto *R = dyn_cast<VPRecipeBase>(
User))
792 if (!TryToPushSinkCandidate(R))
804 if (SinkCandidate == FOR)
807 SinkCandidate->moveAfter(Previous);
808 Previous = SinkCandidate;
821 if (
auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
830 dyn_cast_or_null<VPInstruction>(MiddleVPBB->
getTerminator())) {
831 if (
auto *Cmp = dyn_cast<VPInstruction>(Term->getOperand(0)))
840 VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
843 while (
auto *PrevPhi =
844 dyn_cast_or_null<VPFirstOrderRecurrencePHIRecipe>(Previous)) {
845 assert(PrevPhi->getParent() == FOR->getParent());
847 Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
856 if (isa<VPHeaderPHIRecipe>(Previous))
862 auto *RecurSplice = cast<VPInstruction>(
864 {FOR, FOR->getBackedgeValue()}));
866 FOR->replaceAllUsesWith(RecurSplice);
869 RecurSplice->setOperand(0, FOR);
943 auto *Penultimate = cast<VPInstruction>(MiddleBuilder.
createNaryOp(
945 {FOR->getBackedgeValue(),
946 Plan.getOrAddLiveIn(ConstantInt::get(IntTy, 2))},
947 {},
"vector.recur.extract.for.phi"));
948 RecurSplice->replaceUsesWithIf(
949 Penultimate, [](
VPUser &U,
unsigned) {
return isa<VPLiveOut>(&U); });
956 for (
unsigned I = 0;
I !=
Users.size(); ++
I) {
958 if (!Cur || isa<VPHeaderPHIRecipe>(Cur))
961 Users.insert(V->user_begin(), V->user_end());
963 return Users.takeVector();
969 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
978 if (
auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(U)) {
979 RecWithFlags->dropPoisonGeneratingFlags();
988 if (
auto *Blend = dyn_cast<VPBlendRecipe>(&R)) {
989 VPValue *Inc0 = Blend->getIncomingValue(0);
990 for (
unsigned I = 1;
I != Blend->getNumIncomingValues(); ++
I)
991 if (Inc0 != Blend->getIncomingValue(
I) &&
992 !
match(Blend->getMask(
I), m_False()))
995 Blend->eraseFromParent();
1001 VPValue *Trunc = R.getVPSingleValue();
1004 if (TruncTy == ATy) {
1008 if (isa<VPReplicateRecipe>(&R))
1012 unsigned ExtOpcode =
match(R.getOperand(0),
m_SExt(m_VPValue()))
1014 : Instruction::ZExt;
1017 if (
auto *UnderlyingExt = R.getOperand(0)->getUnderlyingValue()) {
1019 VPC->setUnderlyingValue(UnderlyingExt);
1021 VPC->insertBefore(&R);
1025 VPC->insertBefore(&R);
1033 R.getParent()->getPlan()->getCanonicalIV()->getScalarType(),
1036 for (
VPUser *U :
A->users()) {
1037 auto *R = dyn_cast<VPRecipeBase>(U);
1040 for (
VPValue *VPV : R->definedValues())
1054 X == X1 &&
Y == Y1) {
1055 R.getVPSingleValue()->replaceAllUsesWith(
X);
1060 return R.getVPSingleValue()->replaceAllUsesWith(
A);
1068 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
1081 unsigned NumProcessedRecipes = 0;
1090 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1097 VPValue *ResultVPV = R.getVPSingleValue();
1099 unsigned NewResSizeInBits = MinBWs.
lookup(UI);
1100 if (!NewResSizeInBits)
1104 NumProcessedRecipes++;
1110 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(&R)) {
1120 if (!
Op->isLiveIn())
1122 auto *UV = dyn_cast_or_null<Instruction>(
Op->getUnderlyingValue());
1125 return !isa<VPWidenRecipe, VPWidenSelectRecipe>(U);
1129 ProcessedTruncs[
Op] =
nullptr;
1130 NumProcessedRecipes += 1;
1140 (void)OldResSizeInBits;
1147 if (
auto *VPW = dyn_cast<VPRecipeWithIRFlags>(&R))
1148 VPW->dropPoisonGeneratingFlags();
1151 if (OldResSizeInBits != NewResSizeInBits &&
1152 !
match(&R, m_Binary<Instruction::ICmp>(m_VPValue(), m_VPValue()))) {
1156 Ext->insertAfter(&R);
1158 Ext->setOperand(0, ResultVPV);
1159 assert(OldResSizeInBits > NewResSizeInBits &&
"Nothing to shrink?");
1162 match(&R, m_Binary<Instruction::ICmp>(m_VPValue(), m_VPValue())) &&
1163 "Only ICmps should not need extending the result.");
1165 assert(!isa<VPWidenStoreRecipe>(&R) &&
"stores cannot be narrowed");
1166 if (isa<VPWidenLoadRecipe>(&R))
1170 unsigned StartIdx = isa<VPWidenSelectRecipe>(&R) ? 1 : 0;
1171 for (
unsigned Idx = StartIdx;
Idx != R.getNumOperands(); ++
Idx) {
1172 auto *
Op = R.getOperand(
Idx);
1173 unsigned OpSizeInBits =
1175 if (OpSizeInBits == NewResSizeInBits)
1177 assert(OpSizeInBits > NewResSizeInBits &&
"nothing to truncate");
1178 auto [ProcessedIter, IterIsEmpty] =
1179 ProcessedTruncs.
insert({
Op,
nullptr});
1183 : ProcessedIter->second;
1184 R.setOperand(
Idx, NewOp);
1187 ProcessedIter->second = NewOp;
1188 if (!
Op->isLiveIn()) {
1193 auto *OpInst = dyn_cast<Instruction>(
Op->getLiveInIRValue());
1194 bool IsContained = MinBWs.
contains(OpInst);
1195 NumProcessedRecipes += IsContained;
1203 assert(MinBWs.
size() == NumProcessedRecipes &&
1204 "some entries in MinBWs haven't been processed");
1259 VPValue *StartV = CanonicalIVPHI->getStartValue();
1261 auto *CanonicalIVIncrement =
1262 cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
1265 CanonicalIVIncrement->dropPoisonGeneratingFlags();
1266 DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
1276 VPValue *TripCount, *IncrementValue;
1281 IncrementValue = CanonicalIVIncrement;
1287 IncrementValue = CanonicalIVPHI;
1298 DL,
"active.lane.mask.entry");
1303 LaneMaskPhi->insertAfter(CanonicalIVPHI);
1309 auto *InLoopIncrement =
1311 {IncrementValue}, {
false,
false},
DL);
1313 {InLoopIncrement, TripCount},
DL,
1314 "active.lane.mask.next");
1331 auto *FoundWidenCanonicalIVUser =
1333 [](
VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); });
1335 [](
VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); }) <=
1337 "Must have at most one VPWideCanonicalIVRecipe");
1339 auto *WideCanonicalIV =
1340 cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
1341 WideCanonicalIVs.
push_back(WideCanonicalIV);
1348 auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
1349 if (WidenOriginalIV && WidenOriginalIV->isCanonical())
1350 WideCanonicalIVs.
push_back(WidenOriginalIV);
1356 for (
auto *Wide : WideCanonicalIVs) {
1358 auto *HeaderMask = dyn_cast<VPInstruction>(U);
1362 assert(HeaderMask->getOperand(0) == Wide &&
1363 "WidenCanonicalIV must be the first operand of the compare");
1371 VPlan &Plan,
bool UseActiveLaneMaskForControlFlow,
1374 UseActiveLaneMaskForControlFlow) &&
1375 "DataAndControlFlowWithoutRuntimeCheck implies "
1376 "UseActiveLaneMaskForControlFlow");
1378 auto FoundWidenCanonicalIVUser =
1380 [](
VPUser *U) { return isa<VPWidenCanonicalIVRecipe>(U); });
1381 assert(FoundWidenCanonicalIVUser &&
1382 "Must have widened canonical IV when tail folding!");
1383 auto *WideCanonicalIV =
1384 cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
1386 if (UseActiveLaneMaskForControlFlow) {
1393 "active.lane.mask");
1400 HeaderMask->replaceAllUsesWith(LaneMask);
1431 return isa<VPWidenIntOrFpInductionRecipe, VPWidenPointerInductionRecipe>(
1436 bool ContainsOutloopReductions =
1438 auto *R = dyn_cast<VPReductionPHIRecipe>(&Phi);
1439 return R && !R->isInLoop();
1441 if (ContainsWidenInductions || ContainsOutloopReductions)
1445 VPValue *StartV = CanonicalIVPHI->getStartValue();
1449 EVLPhi->insertAfter(CanonicalIVPHI);
1452 VPEVL->
insertBefore(*Header, Header->getFirstNonPhi());
1454 auto *CanonicalIVIncrement =
1455 cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
1457 if (
unsigned IVSize = CanonicalIVPHI->getScalarType()->getScalarSizeInBits();
1460 : Instruction::ZExt,
1461 OpVPEVL, CanonicalIVPHI->getScalarType());
1466 {CanonicalIVIncrement->hasNoUnsignedWrap(),
1467 CanonicalIVIncrement->hasNoSignedWrap()},
1468 CanonicalIVIncrement->
getDebugLoc(),
"index.evl.next");
1469 NextEVLIV->insertBefore(CanonicalIVIncrement);
1470 EVLPhi->addOperand(NextEVLIV);
1475 auto *CurRecipe = dyn_cast<VPRecipeBase>(U);
1480 assert(OrigMask &&
"Unmasked recipe when folding tail");
1481 return HeaderMask == OrigMask ? nullptr : OrigMask;
1483 if (
auto *MemR = dyn_cast<VPWidenMemoryRecipe>(CurRecipe)) {
1484 VPValue *NewMask = GetNewMask(MemR->getMask());
1485 if (
auto *L = dyn_cast<VPWidenLoadRecipe>(MemR))
1487 else if (
auto *S = dyn_cast<VPWidenStoreRecipe>(MemR))
1491 }
else if (
auto *RedR = dyn_cast<VPReductionRecipe>(CurRecipe)) {
1493 GetNewMask(RedR->getCondOp()));
1498 assert(NumDefVal == CurRecipe->getNumDefinedValues() &&
1499 "New recipe must define the same number of values as the "
1503 "Only supports recipes with a single definition or without users.");
1505 if (isa<VPSingleDefRecipe, VPWidenLoadEVLRecipe>(NewRecipe)) {
1506 VPValue *CurVPV = CurRecipe->getVPSingleValue();
1509 CurRecipe->eraseFromParent();
1516 CanonicalIVPHI->replaceAllUsesWith(EVLPhi);
1517 CanonicalIVIncrement->setOperand(0, CanonicalIVPHI);
1528 auto collectPoisonGeneratingInstrsInBackwardSlice([&](
VPRecipeBase *Root) {
1533 while (!Worklist.
empty()) {
1534 VPRecipeBase *CurRec = Worklist.back();
1535 Worklist.pop_back();
1537 if (!Visited.insert(CurRec).second)
1544 if (isa<VPWidenMemoryRecipe>(CurRec) || isa<VPInterleaveRecipe>(CurRec) ||
1545 isa<VPScalarIVStepsRecipe>(CurRec) || isa<VPHeaderPHIRecipe>(CurRec))
1551 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(CurRec)) {
1553 using namespace llvm::VPlanPatternMatch;
1559 if (match(RecWithFlags, m_BinaryOr(m_VPValue(A), m_VPValue(B))) &&
1560 RecWithFlags->isDisjoint()) {
1561 VPBuilder Builder(RecWithFlags);
1562 VPInstruction *New = Builder.createOverflowingOp(
1563 Instruction::Add, {A, B}, {false, false},
1564 RecWithFlags->getDebugLoc());
1565 New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
1566 RecWithFlags->replaceAllUsesWith(New);
1567 RecWithFlags->eraseFromParent();
1570 RecWithFlags->dropPoisonGeneratingFlags();
1572 Instruction *Instr = dyn_cast_or_null<Instruction>(
1573 CurRec->getVPSingleValue()->getUnderlyingValue());
1575 assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
1576 "found instruction with poison generating flags not covered by "
1577 "VPRecipeWithIRFlags");
1591 for (
VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
1593 if (
auto *WidenRec = dyn_cast<VPWidenMemoryRecipe>(&Recipe)) {
1594 Instruction &UnderlyingInstr = WidenRec->getIngredient();
1595 VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
1596 if (AddrDef && WidenRec->isConsecutive() &&
1597 BlockNeedsPredication(UnderlyingInstr.
getParent()))
1598 collectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
1599 }
else if (
auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(&Recipe)) {
1600 VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
1604 InterleaveRec->getInterleaveGroup();
1605 bool NeedPredication =
false;
1607 I < NumMembers; ++
I) {
1610 NeedPredication |= BlockNeedsPredication(Member->getParent());
1613 if (NeedPredication)
1614 collectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
ReachingDefAnalysis InstSet & ToRemove
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
iv Induction Variable Users
static bool mergeBlocksIntoPredecessors(Loop &L, DominatorTree &DT, LoopInfo &LI, MemorySSAUpdater *MSSAU, ScalarEvolution &SE)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static const uint32_t IV[8]
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
This class represents a function call, abstracting a target machine's calling convention.
@ ICMP_ULE
unsigned less or equal
static ConstantInt * getTrue(LLVMContext &Context)
This class represents an Operation in the Expression.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Core dominator tree base class.
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Utility class for floating point operations which can have information about relaxed accuracy require...
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags.
Convenience struct for specifying and reasoning about fast-math flags.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
A struct for saving information about induction variables.
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_IntInduction
Integer induction variable. Step = C.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
The group of interleaved loads/stores sharing the same stride and close to each other.
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
uint32_t getNumMembers() const
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
This class implements a map that also provides access to all stored values in a deterministic order.
bool contains(const KeyT &Key) const
ValueT lookup(const KeyT &Key) const
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
RecurKind getRecurrenceKind() const
This class represents an analyzed expression in the program.
bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
The main scalar evolution driver.
bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
const SCEV * getElementCount(Type *Ty, ElementCount EC)
LLVMContext & getContext() const
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
RecipeListTy::iterator iterator
Instruction iterators...
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPRecipeBase & back() const
void insert(VPRecipeBase *Recipe, iterator InsertPt)
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
VPBlockBase * getSinglePredecessor() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleHierarchicalPredecessor()
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
A recipe for generating conditional branches on the bits of a mask.
VPlan-based builder utility analogous to IRBuilder.
static VPBuilder getToInsertAfter(VPRecipeBase *R)
Create a VPBuilder to insert after R.
VPInstruction * createOverflowingOp(unsigned Opcode, std::initializer_list< VPValue * > Operands, VPRecipeWithIRFlags::WrapFlagsTy WrapFlags, DebugLoc DL={}, const Twine &Name="")
VPInstruction * createNaryOp(unsigned Opcode, ArrayRef< VPValue * > Operands, Instruction *Inst=nullptr, const Twine &Name="")
Create an N-ary operation with Opcode, Operands and set Inst as its underlying Instruction.
VPValue * createNot(VPValue *Operand, DebugLoc DL={}, const Twine &Name="")
void setInsertPoint(VPBasicBlock *TheBB)
This specifies that created VPInstructions should be appended to the end of the specified block.
Canonical scalar induction phi of the vector loop.
Type * getScalarType() const
Returns the scalar type of the induction.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
A recipe for converting the input value IV value to the corresponding value of an IV with different s...
A recipe for generating the phi node for the current index of elements, adjusted in accordance with E...
This is a concrete Recipe that models a single VPlan-level instruction.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
A recipe to represent inloop reduction operations with vector-predication intrinsics,...
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
VPScalarCastRecipe is a recipe to create scalar cast instructions.
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
An analysis for type-inference for VPValues.
LLVMContext & getContext()
Return the LLVMContext used by the analysis.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
operand_iterator op_end()
operand_iterator op_begin()
void addOperand(VPValue *Operand)
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
void replaceAllUsesWith(VPValue *New)
unsigned getNumUsers() const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
void replaceUsesWithIf(VPValue *New, llvm::function_ref< bool(VPUser &U, unsigned Idx)> ShouldReplace)
Go through the uses list for this VPValue and make each use point to New if the callback ShouldReplac...
A recipe for widening Call instructions.
A Recipe for widening the canonical induction variable of the vector loop.
VPWidenCastRecipe is a recipe to create vector cast instructions.
A recipe for handling GEP instructions.
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPBasicBlock * getEntry()
VPValue * getTripCount() const
The trip count of the original loop.
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
bool hasVF(ElementCount VF)
bool hasUF(unsigned UF) const
void setVF(ElementCount VF)
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
bool hasScalarVFOnly() const
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
StringRef getName() const
Return a constant reference to the value's name.
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
CastInst_match< OpTy, TruncInst > m_Trunc(const OpTy &Op)
Matches Trunc.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
BinaryOp_match< LHS, RHS, Instruction::Mul, true > m_c_Mul(const LHS &L, const RHS &R)
Matches a Mul with LHS and RHS in either order.
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
bool isHeaderMask(VPValue *V, VPlan &Plan)
Return true if V is a header mask in Plan.
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
const SCEV * createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE, Loop *OrigLoop)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
std::unique_ptr< VPlan > VPlanPtr
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
DWARFExpression::Operation Op
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
@ DataAndControlFlowWithoutRuntimeCheck
Use predicate to control both data and control flow, but modify the trip count so that a runtime over...
A recipe for handling first-order recurrence phis.
A recipe for widening load operations with vector-predication intrinsics, using the address to load f...
A recipe for widening load operations, using the address to load from and an optional mask.
A recipe for widening select instructions.
A recipe for widening store operations with vector-predication intrinsics, using the value to store,...
A recipe for widening store operations, using the stored value, the address to store to and an option...