73#ifdef EXPENSIVE_CHECKS
105using namespace slpvectorizer;
107#define SV_NAME "slp-vectorizer"
108#define DEBUG_TYPE "SLP"
110STATISTIC(NumVectorInstructions,
"Number of vector instructions generated");
114 cl::desc(
"Run the SLP vectorization passes"));
118 cl::desc(
"Enable vectorization for wider vector utilization"));
122 cl::desc(
"Only vectorize if you gain more than this "
127 cl::desc(
"When true, SLP vectorizer bypasses profitability checks based on "
128 "heuristics and makes vectorization decision via cost modeling."));
132 cl::desc(
"Attempt to vectorize horizontal reductions"));
137 "Attempt to vectorize horizontal reductions feeding into a store"));
143 cl::desc(
"Allow optimization of original scalar identity operations on "
144 "matched horizontal reductions."));
148 cl::desc(
"Attempt to vectorize for this register size in bits"));
152 cl::desc(
"Maximum SLP vectorization factor (0=unlimited)"));
160 cl::desc(
"Limit the size of the SLP scheduling region per block"));
164 cl::desc(
"Attempt to vectorize for this register size in bits"));
168 cl::desc(
"Limit the recursion depth when building a vectorizable tree"));
172 cl::desc(
"Only vectorize small trees if they are fully vectorizable"));
178 cl::desc(
"The maximum look-ahead depth for operand reordering scores"));
187 cl::desc(
"The maximum look-ahead depth for searching best rooting option"));
191 cl::desc(
"The minimum number of loads, which should be considered strided, "
192 "if the stride is > 1 or is runtime value"));
196 cl::desc(
"The maximum stride, considered to be profitable."));
200 cl::desc(
"Display the SLP trees with Graphviz"));
204 cl::desc(
"Try to vectorize with non-power-of-2 number of elements."));
235 if (
SLPReVec && isa<FixedVectorType>(Ty))
237 return VectorType::isValidElementType(Ty) && !Ty->
isX86_FP80Ty() &&
243 assert(!isa<ScalableVectorType>(Ty) &&
244 "ScalableVectorType is not supported.");
245 if (
auto *VecTy = dyn_cast<FixedVectorType>(Ty))
246 return VecTy->getNumElements();
263 for (
unsigned I : seq<unsigned>(Mask.size()))
265 I * VecTyNumElements, VecTyNumElements)))
267 : Mask[
I] * VecTyNumElements + J;
274 return isa<Constant>(V) && !isa<ConstantExpr, GlobalValue>(V);
281 if (!isa<InsertElementInst, ExtractElementInst>(V) &&
282 !isa<ExtractValueInst, UndefValue>(V))
284 auto *
I = dyn_cast<Instruction>(V);
285 if (!
I || isa<ExtractValueInst>(
I))
287 if (!isa<FixedVectorType>(
I->getOperand(0)->getType()))
289 if (isa<ExtractElementInst>(
I))
291 assert(isa<InsertElementInst>(V) &&
"Expected only insertelement.");
307 return std::min<unsigned>(PartNumElems,
Size - Part * PartNumElems);
315 OS <<
"n=" << VL.
size() <<
" [" << *VL.
front() <<
", ..]";
331 for (
int I = 1, E = VL.
size();
I < E;
I++) {
332 auto *
II = dyn_cast<Instruction>(VL[
I]);
336 if (BB !=
II->getParent())
353 Value *FirstNonUndef =
nullptr;
354 for (
Value *V : VL) {
355 if (isa<UndefValue>(V))
357 if (!FirstNonUndef) {
361 if (V != FirstNonUndef)
364 return FirstNonUndef !=
nullptr;
369 if (
auto *Cmp = dyn_cast<CmpInst>(
I))
370 return Cmp->isCommutative();
371 if (
auto *BO = dyn_cast<BinaryOperator>(
I))
372 return BO->isCommutative() ||
373 (BO->getOpcode() == Instruction::Sub &&
379 ICmpInst::Predicate Pred;
380 if (match(U.getUser(),
381 m_ICmp(Pred, m_Specific(U.get()), m_Zero())) &&
382 (Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_NE))
386 return match(U.getUser(),
387 m_Intrinsic<Intrinsic::abs>(
388 m_Specific(U.get()), m_ConstantInt(Flag))) &&
389 (!cast<Instruction>(U.get())->hasNoSignedWrap() ||
392 (BO->getOpcode() == Instruction::FSub &&
395 return match(U.getUser(),
396 m_Intrinsic<Intrinsic::fabs>(m_Specific(U.get())));
398 return I->isCommutative();
404 static_assert(std::is_same_v<T, InsertElementInst> ||
405 std::is_same_v<T, ExtractElementInst>,
408 if (
const auto *IE = dyn_cast<T>(Inst)) {
409 const auto *VT = dyn_cast<FixedVectorType>(IE->getType());
412 const auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
415 if (CI->getValue().uge(VT->getNumElements()))
417 Index *= VT->getNumElements();
418 Index += CI->getZExtValue();
429 if (
auto Index = getInsertExtractIndex<InsertElementInst>(Inst,
Offset))
431 if (
auto Index = getInsertExtractIndex<ExtractElementInst>(Inst,
Offset))
436 const auto *
IV = dyn_cast<InsertValueInst>(Inst);
440 Type *CurrentType =
IV->getType();
441 for (
unsigned I :
IV->indices()) {
442 if (
const auto *ST = dyn_cast<StructType>(CurrentType)) {
443 Index *= ST->getNumElements();
444 CurrentType = ST->getElementType(
I);
445 }
else if (
const auto *AT = dyn_cast<ArrayType>(CurrentType)) {
446 Index *= AT->getNumElements();
447 CurrentType = AT->getElementType();
480 if (MaskArg == UseMask::UndefsAsMask)
484 if (MaskArg == UseMask::FirstArg &&
Value < VF)
485 UseMask.reset(
Value);
486 else if (MaskArg == UseMask::SecondArg &&
Value >= VF)
487 UseMask.reset(
Value - VF);
495template <
bool IsPoisonOnly = false>
499 using T = std::conditional_t<IsPoisonOnly, PoisonValue, UndefValue>;
502 auto *VecTy = dyn_cast<FixedVectorType>(
V->getType());
505 auto *
C = dyn_cast<Constant>(V);
507 if (!UseMask.empty()) {
509 while (
auto *
II = dyn_cast<InsertElementInst>(
Base)) {
511 if (isa<T>(
II->getOperand(1)))
518 if (*
Idx < UseMask.size() && !UseMask.test(*
Idx))
526 Res &= isUndefVector<IsPoisonOnly>(
Base, SubMask);
533 for (
unsigned I = 0, E = VecTy->getNumElements();
I != E; ++
I) {
534 if (
Constant *Elem =
C->getAggregateElement(
I))
536 (UseMask.empty() || (
I < UseMask.size() && !UseMask.test(
I))))
564static std::optional<TargetTransformInfo::ShuffleKind>
566 const auto *It =
find_if(VL, IsaPred<ExtractElementInst>);
570 std::accumulate(VL.
begin(), VL.
end(), 0u, [](
unsigned S,
Value *V) {
571 auto *EI = dyn_cast<ExtractElementInst>(V);
574 auto *VTy = dyn_cast<FixedVectorType>(EI->getVectorOperandType());
577 return std::max(S, VTy->getNumElements());
580 Value *Vec1 =
nullptr;
581 Value *Vec2 =
nullptr;
583 auto *EE = dyn_cast<ExtractElementInst>(V);
586 Value *Vec = EE->getVectorOperand();
587 if (isa<UndefValue>(Vec))
592 ShuffleMode CommonShuffleMode =
Unknown;
594 for (
unsigned I = 0, E = VL.
size();
I < E; ++
I) {
596 if (isa<UndefValue>(VL[
I]))
598 auto *EI = cast<ExtractElementInst>(VL[
I]);
599 if (isa<ScalableVectorType>(EI->getVectorOperandType()))
601 auto *Vec = EI->getVectorOperand();
603 if (isUndefVector</*isPoisonOnly=*/true>(Vec).all())
606 if (isa<UndefValue>(Vec)) {
609 if (isa<UndefValue>(EI->getIndexOperand()))
611 auto *
Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
617 unsigned IntIdx =
Idx->getValue().getZExtValue();
624 if (!Vec1 || Vec1 == Vec) {
626 }
else if (!Vec2 || Vec2 == Vec) {
632 if (CommonShuffleMode == Permute)
636 if (Mask[
I] %
Size !=
I) {
637 CommonShuffleMode = Permute;
640 CommonShuffleMode =
Select;
643 if (CommonShuffleMode ==
Select && Vec2)
654 assert((Opcode == Instruction::ExtractElement ||
655 Opcode == Instruction::ExtractValue) &&
656 "Expected extractelement or extractvalue instruction.");
657 if (Opcode == Instruction::ExtractElement) {
658 auto *CI = dyn_cast<ConstantInt>(E->
getOperand(1));
661 return CI->getZExtValue();
663 auto *EI = cast<ExtractValueInst>(E);
664 if (EI->getNumIndices() != 1)
666 return *EI->idx_begin();
672struct InstructionsState {
674 Value *OpValue =
nullptr;
685 unsigned getAltOpcode()
const {
690 bool isAltShuffle()
const {
return AltOp != MainOp; }
693 unsigned CheckedOpcode =
I->getOpcode();
694 return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
697 InstructionsState() =
delete;
699 : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {}
708 auto *
I = dyn_cast<Instruction>(
Op);
709 if (
I && S.isOpcodeOrAlt(
I))
728 unsigned BaseIndex = 0);
736 (!isa<Instruction>(BaseOp0) && !isa<Instruction>(Op0) &&
737 !isa<Instruction>(BaseOp1) && !isa<Instruction>(Op1)) ||
738 BaseOp0 == Op0 || BaseOp1 == Op1 ||
749 "Assessing comparisons of different types?");
759 return (BasePred == Pred &&
761 (BasePred == SwappedPred &&
770 unsigned BaseIndex) {
773 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
775 bool IsCastOp = isa<CastInst>(VL[BaseIndex]);
776 bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]);
777 bool IsCmpOp = isa<CmpInst>(VL[BaseIndex]);
779 IsCmpOp ? cast<CmpInst>(VL[BaseIndex])->getPredicate()
781 unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode();
782 unsigned AltOpcode = Opcode;
783 unsigned AltIndex = BaseIndex;
785 bool SwappedPredsCompatible = [&]() {
789 UniquePreds.
insert(BasePred);
790 UniqueNonSwappedPreds.
insert(BasePred);
791 for (
Value *V : VL) {
792 auto *
I = dyn_cast<CmpInst>(V);
798 UniqueNonSwappedPreds.
insert(CurrentPred);
799 if (!UniquePreds.
contains(CurrentPred) &&
800 !UniquePreds.
contains(SwappedCurrentPred))
801 UniquePreds.
insert(CurrentPred);
806 return UniqueNonSwappedPreds.
size() > 2 && UniquePreds.
size() == 2;
810 auto *IBase = cast<Instruction>(VL[BaseIndex]);
813 if (
auto *
CallBase = dyn_cast<CallInst>(IBase)) {
817 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
819 for (
int Cnt = 0, E = VL.
size(); Cnt < E; Cnt++) {
820 auto *
I = cast<Instruction>(VL[Cnt]);
821 unsigned InstOpcode =
I->getOpcode();
822 if (IsBinOp && isa<BinaryOperator>(
I)) {
823 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
827 AltOpcode = InstOpcode;
831 }
else if (IsCastOp && isa<CastInst>(
I)) {
832 Value *Op0 = IBase->getOperand(0);
834 Value *Op1 =
I->getOperand(0);
837 if (InstOpcode == Opcode || InstOpcode == AltOpcode)
839 if (Opcode == AltOpcode) {
842 "Cast isn't safe for alternation, logic needs to be updated!");
843 AltOpcode = InstOpcode;
848 }
else if (
auto *Inst = dyn_cast<CmpInst>(VL[Cnt]); Inst && IsCmpOp) {
849 auto *BaseInst = cast<CmpInst>(VL[BaseIndex]);
850 Type *Ty0 = BaseInst->getOperand(0)->getType();
851 Type *Ty1 = Inst->getOperand(0)->getType();
853 assert(InstOpcode == Opcode &&
"Expected same CmpInst opcode.");
860 if ((E == 2 || SwappedPredsCompatible) &&
861 (BasePred == CurrentPred || BasePred == SwappedCurrentPred))
866 auto *AltInst = cast<CmpInst>(VL[AltIndex]);
867 if (AltIndex != BaseIndex) {
870 }
else if (BasePred != CurrentPred) {
873 "CmpInst isn't safe for alternation, logic needs to be updated!");
878 if (BasePred == CurrentPred || BasePred == SwappedCurrentPred ||
879 AltPred == CurrentPred || AltPred == SwappedCurrentPred)
882 }
else if (InstOpcode == Opcode || InstOpcode == AltOpcode) {
883 if (
auto *Gep = dyn_cast<GetElementPtrInst>(
I)) {
884 if (Gep->getNumOperands() != 2 ||
885 Gep->getOperand(0)->getType() != IBase->getOperand(0)->getType())
886 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
887 }
else if (
auto *EI = dyn_cast<ExtractElementInst>(
I)) {
889 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
890 }
else if (
auto *LI = dyn_cast<LoadInst>(
I)) {
891 auto *BaseLI = cast<LoadInst>(IBase);
892 if (!LI->isSimple() || !BaseLI->isSimple())
893 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
894 }
else if (
auto *Call = dyn_cast<CallInst>(
I)) {
895 auto *
CallBase = cast<CallInst>(IBase);
897 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
899 !std::equal(Call->op_begin() + Call->getBundleOperandsStartIndex(),
900 Call->op_begin() + Call->getBundleOperandsEndIndex(),
903 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
906 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
909 if (Mappings.
size() != BaseMappings.
size() ||
910 Mappings.
front().ISA != BaseMappings.
front().ISA ||
911 Mappings.
front().ScalarName != BaseMappings.
front().ScalarName ||
912 Mappings.
front().VectorName != BaseMappings.
front().VectorName ||
913 Mappings.
front().Shape.VF != BaseMappings.
front().Shape.VF ||
914 Mappings.
front().Shape.Parameters !=
915 BaseMappings.
front().Shape.Parameters)
916 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
921 return InstructionsState(VL[BaseIndex],
nullptr,
nullptr);
924 return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]),
925 cast<Instruction>(VL[AltIndex]));
941 case Instruction::Load: {
942 LoadInst *LI = cast<LoadInst>(UserInst);
945 case Instruction::Store: {
946 StoreInst *SI = cast<StoreInst>(UserInst);
947 return (SI->getPointerOperand() == Scalar);
949 case Instruction::Call: {
950 CallInst *CI = cast<CallInst>(UserInst);
953 return isVectorIntrinsicWithScalarOpAtArg(ID, Arg.index()) &&
954 Arg.value().get() == Scalar;
966 if (
LoadInst *LI = dyn_cast<LoadInst>(
I))
973 if (
LoadInst *LI = dyn_cast<LoadInst>(
I))
974 return LI->isSimple();
976 return SI->isSimple();
978 return !
MI->isVolatile();
986 bool ExtendingManyInputs =
false) {
990 (!ExtendingManyInputs || SubMask.
size() > Mask.size() ||
992 (SubMask.
size() == Mask.size() &&
993 std::all_of(std::next(Mask.begin(), Mask.size() / 2), Mask.end(),
994 [](
int Idx) { return Idx == PoisonMaskElem; }))) &&
995 "SubMask with many inputs support must be larger than the mask.");
997 Mask.append(SubMask.
begin(), SubMask.
end());
1001 int TermValue = std::min(Mask.size(), SubMask.
size());
1002 for (
int I = 0, E = SubMask.
size();
I < E; ++
I) {
1004 (!ExtendingManyInputs &&
1005 (SubMask[
I] >= TermValue || Mask[SubMask[
I]] >= TermValue)))
1007 NewMask[
I] = Mask[SubMask[
I]];
1023 const unsigned Sz = Order.
size();
1026 for (
unsigned I = 0;
I < Sz; ++
I) {
1028 UnusedIndices.
reset(Order[
I]);
1030 MaskedIndices.
set(
I);
1032 if (MaskedIndices.
none())
1035 "Non-synced masked/available indices.");
1039 assert(
Idx >= 0 &&
"Indices must be synced.");
1051 for (
unsigned Lane : seq<unsigned>(VL.
size()))
1052 if (cast<Instruction>(VL[Lane])->
getOpcode() == Opcode1)
1053 OpcodeMask.
set(Lane);
1062 const unsigned E = Indices.
size();
1064 for (
unsigned I = 0;
I < E; ++
I)
1065 Mask[Indices[
I]] =
I;
1071 assert(!Mask.empty() &&
"Expected non-empty mask.");
1075 for (
unsigned I = 0, E = Prev.
size();
I < E; ++
I)
1077 Scalars[Mask[
I]] = Prev[
I];
1085 auto *
I = dyn_cast<Instruction>(V);
1090 auto *IO = dyn_cast<Instruction>(V);
1093 return isa<PHINode>(IO) || IO->getParent() != I->getParent();
1102 auto *
I = dyn_cast<Instruction>(V);
1106 return !
I->mayReadOrWriteMemory() && !
I->hasNUsesOrMore(
UsesLimit) &&
1108 auto *IU = dyn_cast<Instruction>(U);
1111 return IU->getParent() != I->getParent() || isa<PHINode>(IU);
1127 return !VL.
empty() &&
1131namespace slpvectorizer {
1136 struct ScheduleData;
1161 : BatchAA(*Aa),
F(Func), SE(Se),
TTI(Tti), TLI(TLi), LI(Li), DT(Dt),
1162 AC(AC), DB(DB),
DL(
DL), ORE(ORE),
1216 return !VectorizableTree.
empty() &&
1217 !VectorizableTree.
front()->UserTreeIndices.empty();
1222 assert(!VectorizableTree.
empty() &&
"No graph to get the first node from");
1223 return VectorizableTree.
front()->Scalars;
1229 return MinBWs.
at(VectorizableTree.
front().get()).second;
1244 VectorizableTree.
clear();
1245 ScalarToTreeEntry.clear();
1246 MultiNodeScalars.clear();
1248 NonScheduledFirst.
clear();
1249 EntryToLastInstruction.clear();
1250 ExternalUses.
clear();
1251 ExternalUsesAsGEPs.clear();
1252 for (
auto &Iter : BlocksSchedules) {
1253 BlockScheduling *BS = Iter.second.get();
1257 ReductionBitWidth = 0;
1258 CastMaxMinBWSizes.reset();
1259 ExtraBitWidthNodes.
clear();
1260 InstrElementSize.clear();
1261 UserIgnoreList =
nullptr;
1262 PostponedGathers.
clear();
1263 ValueToGatherNodes.
clear();
1320 return MaxVecRegSize;
1325 return MinVecRegSize;
1333 unsigned MaxVF =
MaxVFOption.getNumOccurrences() ?
1335 return MaxVF ? MaxVF : UINT_MAX;
1379 bool TryRecursiveCheck =
true)
const;
1403 OS <<
"{User:" << (
UserTE ? std::to_string(
UserTE->Idx) :
"null")
1404 <<
" EdgeIdx:" <<
EdgeIdx <<
"}";
1426 : TLI(TLI),
DL(
DL), SE(SE), R(R), NumLanes(NumLanes),
1427 MaxLevel(MaxLevel) {}
1481 if (isa<LoadInst>(V1)) {
1483 auto AllUsersAreInternal = [U1, U2,
this](
Value *V1,
Value *V2) {
1488 auto AllUsersVectorized = [U1, U2,
this](
Value *V) {
1490 return U == U1 || U == U2 || R.getTreeEntry(U) != nullptr;
1493 return AllUsersVectorized(V1) && AllUsersVectorized(V2);
1496 if (R.TTI->isLegalBroadcastLoad(V1->getType(),
1498 ((
int)V1->getNumUses() == NumLanes ||
1499 AllUsersAreInternal(V1, V2)))
1505 auto CheckSameEntryOrFail = [&]() {
1506 if (
const TreeEntry *TE1 = R.getTreeEntry(V1);
1507 TE1 && TE1 == R.getTreeEntry(V2))
1512 auto *LI1 = dyn_cast<LoadInst>(V1);
1513 auto *LI2 = dyn_cast<LoadInst>(V2);
1515 if (LI1->getParent() != LI2->getParent() || !LI1->isSimple() ||
1517 return CheckSameEntryOrFail();
1520 LI1->getType(), LI1->getPointerOperand(), LI2->getType(),
1521 LI2->getPointerOperand(),
DL, SE,
true);
1522 if (!Dist || *Dist == 0) {
1525 R.TTI->isLegalMaskedGather(
1528 return CheckSameEntryOrFail();
1532 if (std::abs(*Dist) > NumLanes / 2)
1541 auto *C1 = dyn_cast<Constant>(V1);
1542 auto *C2 = dyn_cast<Constant>(V2);
1556 if (isa<UndefValue>(V2))
1560 Value *EV2 =
nullptr;
1573 int Dist = Idx2 - Idx1;
1576 if (std::abs(Dist) == 0)
1578 if (std::abs(Dist) > NumLanes / 2)
1585 return CheckSameEntryOrFail();
1588 auto *I1 = dyn_cast<Instruction>(V1);
1589 auto *I2 = dyn_cast<Instruction>(V2);
1591 if (I1->getParent() != I2->getParent())
1592 return CheckSameEntryOrFail();
1599 if (S.getOpcode() &&
1600 (S.MainOp->getNumOperands() <= 2 || !MainAltOps.
empty() ||
1601 !S.isAltShuffle()) &&
1603 return cast<Instruction>(V)->getNumOperands() ==
1604 S.MainOp->getNumOperands();
1610 if (isa<UndefValue>(V2))
1613 return CheckSameEntryOrFail();
1647 int ShallowScoreAtThisLevel =
1656 auto *I1 = dyn_cast<Instruction>(
LHS);
1657 auto *I2 = dyn_cast<Instruction>(
RHS);
1658 if (CurrLevel == MaxLevel || !(I1 && I2) || I1 == I2 ||
1660 (((isa<LoadInst>(I1) && isa<LoadInst>(I2)) ||
1661 (I1->getNumOperands() > 2 && I2->getNumOperands() > 2) ||
1662 (isa<ExtractElementInst>(I1) && isa<ExtractElementInst>(I2))) &&
1663 ShallowScoreAtThisLevel))
1664 return ShallowScoreAtThisLevel;
1665 assert(I1 && I2 &&
"Should have early exited.");
1672 for (
unsigned OpIdx1 = 0, NumOperands1 = I1->getNumOperands();
1673 OpIdx1 != NumOperands1; ++OpIdx1) {
1675 int MaxTmpScore = 0;
1676 unsigned MaxOpIdx2 = 0;
1677 bool FoundBest =
false;
1681 ? I2->getNumOperands()
1682 : std::min(I2->getNumOperands(), OpIdx1 + 1);
1683 assert(FromIdx <= ToIdx &&
"Bad index");
1684 for (
unsigned OpIdx2 = FromIdx; OpIdx2 != ToIdx; ++OpIdx2) {
1686 if (Op2Used.
count(OpIdx2))
1691 I1, I2, CurrLevel + 1, std::nullopt);
1694 TmpScore > MaxTmpScore) {
1695 MaxTmpScore = TmpScore;
1702 Op2Used.
insert(MaxOpIdx2);
1703 ShallowScoreAtThisLevel += MaxTmpScore;
1706 return ShallowScoreAtThisLevel;
1737 struct OperandData {
1738 OperandData() =
default;
1739 OperandData(
Value *V,
bool APO,
bool IsUsed)
1740 : V(V), APO(APO), IsUsed(IsUsed) {}
1750 bool IsUsed =
false;
1759 enum class ReorderingMode {
1776 const Loop *L =
nullptr;
1779 OperandData &getData(
unsigned OpIdx,
unsigned Lane) {
1780 return OpsVec[OpIdx][Lane];
1784 const OperandData &getData(
unsigned OpIdx,
unsigned Lane)
const {
1785 return OpsVec[OpIdx][Lane];
1790 for (
unsigned OpIdx = 0, NumOperands = getNumOperands();
1791 OpIdx != NumOperands; ++OpIdx)
1792 for (
unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
1794 OpsVec[OpIdx][Lane].IsUsed =
false;
1798 void swap(
unsigned OpIdx1,
unsigned OpIdx2,
unsigned Lane) {
1799 std::swap(OpsVec[OpIdx1][Lane], OpsVec[OpIdx2][Lane]);
1811 int getSplatScore(
unsigned Lane,
unsigned OpIdx,
unsigned Idx)
const {
1812 Value *IdxLaneV = getData(
Idx, Lane).V;
1813 if (!isa<Instruction>(IdxLaneV) || IdxLaneV == getData(OpIdx, Lane).V)
1816 for (
unsigned Ln = 0, E = getNumLanes(); Ln < E; ++Ln) {
1819 Value *OpIdxLnV = getData(OpIdx, Ln).V;
1820 if (!isa<Instruction>(OpIdxLnV))
1822 Uniques.
insert(OpIdxLnV);
1824 int UniquesCount = Uniques.
size();
1825 int UniquesCntWithIdxLaneV =
1826 Uniques.
contains(IdxLaneV) ? UniquesCount : UniquesCount + 1;
1827 Value *OpIdxLaneV = getData(OpIdx, Lane).V;
1828 int UniquesCntWithOpIdxLaneV =
1829 Uniques.
contains(OpIdxLaneV) ? UniquesCount : UniquesCount + 1;
1830 if (UniquesCntWithIdxLaneV == UniquesCntWithOpIdxLaneV)
1833 UniquesCntWithOpIdxLaneV) -
1834 (
PowerOf2Ceil(UniquesCntWithIdxLaneV) - UniquesCntWithIdxLaneV);
1843 int getExternalUseScore(
unsigned Lane,
unsigned OpIdx,
unsigned Idx)
const {
1844 Value *IdxLaneV = getData(
Idx, Lane).V;
1845 Value *OpIdxLaneV = getData(OpIdx, Lane).V;
1854 auto *IdxLaneI = dyn_cast<Instruction>(IdxLaneV);
1855 if (!IdxLaneI || !isa<Instruction>(OpIdxLaneV))
1857 return R.areAllUsersVectorized(IdxLaneI)
1865 static const int ScoreScaleFactor = 10;
1873 int Lane,
unsigned OpIdx,
unsigned Idx,
1883 int SplatScore = getSplatScore(Lane, OpIdx,
Idx);
1884 if (Score <= -SplatScore) {
1889 Score += SplatScore;
1895 Score *= ScoreScaleFactor;
1896 Score += getExternalUseScore(Lane, OpIdx,
Idx);
1914 std::optional<unsigned>
1915 getBestOperand(
unsigned OpIdx,
int Lane,
int LastLane,
1918 unsigned NumOperands = getNumOperands();
1921 Value *OpLastLane = getData(OpIdx, LastLane).V;
1924 ReorderingMode RMode = ReorderingModes[OpIdx];
1925 if (RMode == ReorderingMode::Failed)
1926 return std::nullopt;
1929 bool OpIdxAPO = getData(OpIdx, Lane).APO;
1935 std::optional<unsigned>
Idx;
1939 BestScoresPerLanes.
try_emplace(std::make_pair(OpIdx, Lane), 0)
1945 bool IsUsed = RMode == ReorderingMode::Splat ||
1946 RMode == ReorderingMode::Constant ||
1947 RMode == ReorderingMode::Load;
1949 for (
unsigned Idx = 0;
Idx != NumOperands; ++
Idx) {
1951 OperandData &OpData = getData(
Idx, Lane);
1953 bool OpAPO = OpData.APO;
1962 if (OpAPO != OpIdxAPO)
1967 case ReorderingMode::Load:
1968 case ReorderingMode::Opcode: {
1969 bool LeftToRight = Lane > LastLane;
1970 Value *OpLeft = (LeftToRight) ? OpLastLane :
Op;
1971 Value *OpRight = (LeftToRight) ?
Op : OpLastLane;
1972 int Score = getLookAheadScore(OpLeft, OpRight, MainAltOps, Lane,
1973 OpIdx,
Idx, IsUsed);
1974 if (Score >
static_cast<int>(BestOp.Score) ||
1975 (Score > 0 && Score ==
static_cast<int>(BestOp.Score) &&
1978 BestOp.Score = Score;
1979 BestScoresPerLanes[std::make_pair(OpIdx, Lane)] = Score;
1983 case ReorderingMode::Constant:
1984 if (isa<Constant>(
Op) ||
1985 (!BestOp.Score && L && L->isLoopInvariant(
Op))) {
1987 if (isa<Constant>(
Op)) {
1989 BestScoresPerLanes[std::make_pair(OpIdx, Lane)] =
1992 if (isa<UndefValue>(
Op) || !isa<Constant>(
Op))
1996 case ReorderingMode::Splat:
1997 if (
Op == OpLastLane || (!BestOp.Score && isa<Constant>(
Op))) {
1998 IsUsed =
Op == OpLastLane;
1999 if (
Op == OpLastLane) {
2001 BestScoresPerLanes[std::make_pair(OpIdx, Lane)] =
2007 case ReorderingMode::Failed:
2013 getData(*BestOp.Idx, Lane).IsUsed = IsUsed;
2017 return std::nullopt;
2024 unsigned getBestLaneToStartReordering()
const {
2025 unsigned Min = UINT_MAX;
2026 unsigned SameOpNumber = 0;
2037 for (
int I = getNumLanes();
I > 0; --
I) {
2038 unsigned Lane =
I - 1;
2039 OperandsOrderData NumFreeOpsHash =
2040 getMaxNumOperandsThatCanBeReordered(Lane);
2043 if (NumFreeOpsHash.NumOfAPOs < Min) {
2044 Min = NumFreeOpsHash.NumOfAPOs;
2045 SameOpNumber = NumFreeOpsHash.NumOpsWithSameOpcodeParent;
2047 HashMap[NumFreeOpsHash.Hash] = std::make_pair(1, Lane);
2048 }
else if (NumFreeOpsHash.NumOfAPOs == Min &&
2049 NumFreeOpsHash.NumOpsWithSameOpcodeParent < SameOpNumber) {
2052 SameOpNumber = NumFreeOpsHash.NumOpsWithSameOpcodeParent;
2053 HashMap[NumFreeOpsHash.Hash] = std::make_pair(1, Lane);
2054 }
else if (NumFreeOpsHash.NumOfAPOs == Min &&
2055 NumFreeOpsHash.NumOpsWithSameOpcodeParent == SameOpNumber) {
2056 auto *It = HashMap.
find(NumFreeOpsHash.Hash);
2057 if (It == HashMap.
end())
2058 HashMap[NumFreeOpsHash.Hash] = std::make_pair(1, Lane);
2064 unsigned BestLane = 0;
2065 unsigned CntMin = UINT_MAX;
2067 if (
Data.second.first < CntMin) {
2068 CntMin =
Data.second.first;
2069 BestLane =
Data.second.second;
2076 struct OperandsOrderData {
2079 unsigned NumOfAPOs = UINT_MAX;
2082 unsigned NumOpsWithSameOpcodeParent = 0;
2096 OperandsOrderData getMaxNumOperandsThatCanBeReordered(
unsigned Lane)
const {
2097 unsigned CntTrue = 0;
2098 unsigned NumOperands = getNumOperands();
2108 bool AllUndefs =
true;
2109 unsigned NumOpsWithSameOpcodeParent = 0;
2113 for (
unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
2114 const OperandData &OpData = getData(OpIdx, Lane);
2119 if (
auto *
I = dyn_cast<Instruction>(OpData.V)) {
2121 I->getParent() != Parent) {
2122 if (NumOpsWithSameOpcodeParent == 0) {
2123 NumOpsWithSameOpcodeParent = 1;
2125 Parent =
I->getParent();
2127 --NumOpsWithSameOpcodeParent;
2130 ++NumOpsWithSameOpcodeParent;
2134 Hash,
hash_value((OpIdx + 1) * (OpData.V->getValueID() + 1)));
2135 AllUndefs = AllUndefs && isa<UndefValue>(OpData.V);
2139 OperandsOrderData
Data;
2140 Data.NumOfAPOs = std::max(CntTrue, NumOperands - CntTrue);
2141 Data.NumOpsWithSameOpcodeParent = NumOpsWithSameOpcodeParent;
2149 assert((empty() || VL.
size() == getNumLanes()) &&
2150 "Expected same number of lanes");
2151 assert(isa<Instruction>(VL[0]) &&
"Expected instruction");
2152 unsigned NumOperands = cast<Instruction>(VL[0])->getNumOperands();
2153 constexpr unsigned IntrinsicNumOperands = 2;
2154 if (isa<IntrinsicInst>(VL[0]))
2155 NumOperands = IntrinsicNumOperands;
2156 OpsVec.
resize(NumOperands);
2157 unsigned NumLanes = VL.
size();
2158 for (
unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
2159 OpsVec[OpIdx].
resize(NumLanes);
2160 for (
unsigned Lane = 0; Lane != NumLanes; ++Lane) {
2161 assert(isa<Instruction>(VL[Lane]) &&
"Expected instruction");
2172 bool IsInverseOperation = !
isCommutative(cast<Instruction>(VL[Lane]));
2173 bool APO = (OpIdx == 0) ?
false : IsInverseOperation;
2174 OpsVec[OpIdx][Lane] = {cast<Instruction>(VL[Lane])->getOperand(OpIdx),
2181 unsigned getNumOperands()
const {
return OpsVec.
size(); }
2184 unsigned getNumLanes()
const {
return OpsVec[0].
size(); }
2187 Value *getValue(
unsigned OpIdx,
unsigned Lane)
const {
2188 return getData(OpIdx, Lane).V;
2192 bool empty()
const {
return OpsVec.
empty(); }
2195 void clear() { OpsVec.
clear(); }
2200 bool shouldBroadcast(
Value *
Op,
unsigned OpIdx,
unsigned Lane) {
2201 bool OpAPO = getData(OpIdx, Lane).APO;
2202 bool IsInvariant = L && L->isLoopInvariant(
Op);
2204 for (
unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) {
2208 bool FoundCandidate =
false;
2209 for (
unsigned OpI = 0, OpE = getNumOperands(); OpI != OpE; ++OpI) {
2210 OperandData &
Data = getData(OpI, Ln);
2211 if (
Data.APO != OpAPO ||
Data.IsUsed)
2213 Value *OpILane = getValue(OpI, Lane);
2214 bool IsConstantOp = isa<Constant>(OpILane);
2223 ((Lns > 2 && isa<Constant>(
Data.V)) ||
2229 isa<Constant>(
Data.V)))) ||
2236 (IsInvariant && !isa<Constant>(
Data.V) &&
2238 L->isLoopInvariant(
Data.V))) {
2239 FoundCandidate =
true;
2246 if (!FoundCandidate)
2249 return getNumLanes() == 2 || Cnt > 1;
2254 bool canBeVectorized(
Instruction *
Op,
unsigned OpIdx,
unsigned Lane)
const {
2255 bool OpAPO = getData(OpIdx, Lane).APO;
2256 for (
unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) {
2259 if (
any_of(seq<unsigned>(getNumOperands()), [&](
unsigned OpI) {
2260 const OperandData &
Data = getData(OpI, Ln);
2261 if (
Data.APO != OpAPO ||
Data.IsUsed)
2263 Value *OpILn = getValue(OpI, Ln);
2264 return (L && L->isLoopInvariant(OpILn)) ||
2266 Op->getParent() == cast<Instruction>(OpILn)->getParent());
2276 : TLI(*R.TLI),
DL(*R.
DL), SE(*R.SE), R(R),
2280 appendOperandsOfVL(RootVL);
2287 assert(OpsVec[OpIdx].
size() == getNumLanes() &&
2288 "Expected same num of lanes across all operands");
2289 for (
unsigned Lane = 0, Lanes = getNumLanes(); Lane != Lanes; ++Lane)
2290 OpVL[Lane] = OpsVec[OpIdx][Lane].V;
2298 unsigned NumOperands = getNumOperands();
2299 unsigned NumLanes = getNumLanes();
2319 unsigned FirstLane = getBestLaneToStartReordering();
2322 for (
unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
2323 Value *OpLane0 = getValue(OpIdx, FirstLane);
2326 if (isa<LoadInst>(OpLane0))
2327 ReorderingModes[OpIdx] = ReorderingMode::Load;
2328 else if (
auto *OpILane0 = dyn_cast<Instruction>(OpLane0)) {
2330 if (shouldBroadcast(OpLane0, OpIdx, FirstLane) ||
2331 !canBeVectorized(OpILane0, OpIdx, FirstLane))
2332 ReorderingModes[OpIdx] = ReorderingMode::Splat;
2334 ReorderingModes[OpIdx] = ReorderingMode::Opcode;
2335 }
else if (isa<Constant>(OpLane0))
2336 ReorderingModes[OpIdx] = ReorderingMode::Constant;
2337 else if (isa<Argument>(OpLane0))
2339 ReorderingModes[OpIdx] = ReorderingMode::Splat;
2342 ReorderingModes[OpIdx] = ReorderingMode::Failed;
2349 auto &&SkipReordering = [
this]() {
2352 for (
const OperandData &
Data : Op0)
2355 if (
any_of(
Op, [&UniqueValues](
const OperandData &
Data) {
2374 if (SkipReordering())
2377 bool StrategyFailed =
false;
2385 for (
unsigned I = 0;
I < NumOperands; ++
I)
2386 MainAltOps[
I].push_back(getData(
I, FirstLane).V);
2388 for (
unsigned Distance = 1; Distance != NumLanes; ++Distance) {
2391 int Lane = FirstLane +
Direction * Distance;
2392 if (Lane < 0 || Lane >= (
int)NumLanes)
2395 assert(LastLane >= 0 && LastLane < (
int)NumLanes &&
2398 for (
unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
2400 std::optional<unsigned> BestIdx = getBestOperand(
2401 OpIdx, Lane, LastLane, ReorderingModes, MainAltOps[OpIdx]);
2408 swap(OpIdx, *BestIdx, Lane);
2411 StrategyFailed =
true;
2414 if (MainAltOps[OpIdx].
size() != 2) {
2415 OperandData &AltOp = getData(OpIdx, Lane);
2416 InstructionsState OpS =
2418 if (OpS.getOpcode() && OpS.isAltShuffle())
2425 if (!StrategyFailed)
2430#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2433 case ReorderingMode::Load:
2435 case ReorderingMode::Opcode:
2437 case ReorderingMode::Constant:
2439 case ReorderingMode::Splat:
2441 case ReorderingMode::Failed:
2462 const unsigned Indent = 2;
2465 OS <<
"Operand " << Cnt++ <<
"\n";
2466 for (
const OperandData &OpData : OpDataVec) {
2468 if (
Value *V = OpData.V)
2472 OS <<
", APO:" << OpData.APO <<
"}\n";
2494 int BestScore = Limit;
2495 std::optional<int>
Index;
2496 for (
int I : seq<int>(0, Candidates.size())) {
2498 Candidates[
I].second,
2501 if (Score > BestScore) {
2516 DeletedInstructions.insert(
I);
2521 template <
typename T>
2524 for (
T *V : DeadVals) {
2525 auto *
I = cast<Instruction>(V);
2526 DeletedInstructions.insert(
I);
2529 for (
T *V : DeadVals) {
2530 if (!V || !Processed.
insert(V).second)
2532 auto *
I = cast<Instruction>(V);
2535 if (
const TreeEntry *Entry = getTreeEntry(
I)) {
2536 Entries.push_back(Entry);
2537 auto It = MultiNodeScalars.find(
I);
2538 if (It != MultiNodeScalars.end())
2539 Entries.append(It->second.begin(), It->second.end());
2541 for (
Use &U :
I->operands()) {
2542 if (
auto *OpI = dyn_cast_if_present<Instruction>(U.get());
2543 OpI && !DeletedInstructions.contains(OpI) && OpI->hasOneUser() &&
2545 (Entries.empty() ||
none_of(Entries, [&](
const TreeEntry *Entry) {
2546 return Entry->VectorizedValue == OpI;
2550 I->dropAllReferences();
2552 for (
T *V : DeadVals) {
2553 auto *
I = cast<Instruction>(V);
2554 if (!
I->getParent())
2559 cast<Instruction>(U.getUser()));
2561 "trying to erase instruction with users.");
2562 I->removeFromParent();
2566 while (!DeadInsts.
empty()) {
2569 if (!VI || !VI->getParent())
2572 "Live instruction found in dead worklist!");
2573 assert(VI->use_empty() &&
"Instructions with uses are not dead.");
2580 for (
Use &OpU : VI->operands()) {
2581 Value *OpV = OpU.get();
2592 if (
auto *OpI = dyn_cast<Instruction>(OpV))
2593 if (!DeletedInstructions.contains(OpI) &&
2598 VI->removeFromParent();
2599 DeletedInstructions.insert(VI);
2607 return AnalyzedReductionsRoots.count(
I);
2612 AnalyzedReductionsRoots.insert(
I);
2626 AnalyzedReductionsRoots.clear();
2627 AnalyzedReductionVals.
clear();
2628 AnalyzedMinBWVals.
clear();
2640 return NonScheduledFirst.
contains(V);
2653 bool collectValuesToDemote(
const TreeEntry &E,
bool IsProfitableToDemoteRoot,
2657 unsigned &MaxDepthLevel,
2658 bool &IsProfitableToDemote,
2659 bool IsTruncRoot)
const;
2669 canReorderOperands(TreeEntry *UserTE,
2676 void reorderNodeWithReuses(TreeEntry &TE,
ArrayRef<int> Mask)
const;
2680 TreeEntry *getVectorizedOperand(TreeEntry *UserTE,
unsigned OpIdx) {
2682 TreeEntry *TE =
nullptr;
2684 TE = getTreeEntry(V);
2685 if (TE &&
is_contained(TE->UserTreeIndices, EdgeInfo(UserTE, OpIdx)))
2687 auto It = MultiNodeScalars.find(V);
2688 if (It != MultiNodeScalars.end()) {
2689 for (TreeEntry *E : It->second) {
2690 if (
is_contained(E->UserTreeIndices, EdgeInfo(UserTE, OpIdx))) {
2698 if (It != VL.
end()) {
2699 assert(
TE->isSame(VL) &&
"Expected same scalars.");
2707 const TreeEntry *getVectorizedOperand(
const TreeEntry *UserTE,
2708 unsigned OpIdx)
const {
2709 return const_cast<BoUpSLP *
>(
this)->getVectorizedOperand(
2710 const_cast<TreeEntry *
>(UserTE), OpIdx);
2714 bool areAllUsersVectorized(
2723 const TreeEntry *getOperandEntry(
const TreeEntry *E,
unsigned Idx)
const;
2727 getCastContextHint(
const TreeEntry &TE)
const;
2736 const EdgeInfo &EI);
2747 bool ResizeAllowed =
false)
const;
2758 Value *vectorizeOperand(TreeEntry *E,
unsigned NodeIdx,
bool PostponedPHIs);
2763 template <
typename BVTy,
typename ResTy,
typename...
Args>
2764 ResTy processBuildVector(
const TreeEntry *E,
Type *ScalarTy, Args &...Params);
2769 Value *createBuildVector(
const TreeEntry *E,
Type *ScalarTy);
2775 Instruction &getLastInstructionInBundle(
const TreeEntry *E);
2782 std::optional<TargetTransformInfo::ShuffleKind>
2794 unsigned NumParts)
const;
2806 std::optional<TargetTransformInfo::ShuffleKind>
2807 isGatherShuffledSingleRegisterEntry(
2824 isGatherShuffledEntry(
2827 unsigned NumParts,
bool ForOrder =
false);
2834 Type *ScalarTy)
const;
2838 void setInsertPointAfterBundle(
const TreeEntry *E);
2846 bool isFullyVectorizableTinyTree(
bool ForReduction)
const;
2859 collectUserStores(
const BoUpSLP::TreeEntry *TE)
const;
2875 findExternalStoreUsersReorderIndices(TreeEntry *TE)
const;
2879 TreeEntry(VecTreeTy &Container) : Container(Container) {}
2896 [Scalars](
Value *V,
int Idx) {
2897 return (isa<UndefValue>(V) &&
2898 Idx == PoisonMaskElem) ||
2899 (Idx != PoisonMaskElem && V == Scalars[Idx]);
2902 if (!ReorderIndices.empty()) {
2909 return IsSame(Scalars, Mask);
2910 if (VL.
size() == ReuseShuffleIndices.size()) {
2912 return IsSame(Scalars, Mask);
2916 return IsSame(Scalars, ReuseShuffleIndices);
2919 bool isOperandGatherNode(
const EdgeInfo &UserEI)
const {
2920 return isGather() && UserTreeIndices.front().EdgeIdx == UserEI.EdgeIdx &&
2921 UserTreeIndices.front().UserTE == UserEI.UserTE;
2925 bool hasEqualOperands(
const TreeEntry &TE)
const {
2926 if (
TE.getNumOperands() != getNumOperands())
2929 for (
unsigned I = 0, E = getNumOperands();
I < E; ++
I) {
2930 unsigned PrevCount =
Used.count();
2931 for (
unsigned K = 0;
K < E; ++
K) {
2934 if (getOperand(K) ==
TE.getOperand(
I)) {
2940 if (PrevCount ==
Used.count())
2949 unsigned getVectorFactor()
const {
2950 if (!ReuseShuffleIndices.empty())
2951 return ReuseShuffleIndices.size();
2952 return Scalars.
size();
2956 bool isGather()
const {
return State == NeedToGather; }
2990 VecTreeTy &Container;
3014 assert(Operands[OpIdx].empty() &&
"Already resized?");
3016 "Number of operands is greater than the number of scalars.");
3022 void setOperandsInOrder() {
3024 auto *I0 = cast<Instruction>(Scalars[0]);
3025 Operands.resize(I0->getNumOperands());
3026 unsigned NumLanes = Scalars.size();
3027 for (
unsigned OpIdx = 0, NumOperands = I0->getNumOperands();
3028 OpIdx != NumOperands; ++OpIdx) {
3030 for (
unsigned Lane = 0; Lane != NumLanes; ++Lane) {
3031 auto *
I = cast<Instruction>(Scalars[Lane]);
3032 assert(
I->getNumOperands() == NumOperands &&
3033 "Expected same number of operands");
3034 Operands[OpIdx][Lane] =
I->getOperand(OpIdx);
3058 unsigned getNumOperands()
const {
return Operands.size(); }
3061 Value *getSingleOperand(
unsigned OpIdx)
const {
3063 assert(!Operands[OpIdx].empty() &&
"No operand available");
3068 bool isAltShuffle()
const {
return MainOp != AltOp; }
3071 unsigned CheckedOpcode =
I->getOpcode();
3072 return (getOpcode() == CheckedOpcode ||
3073 getAltOpcode() == CheckedOpcode);
3080 auto *
I = dyn_cast<Instruction>(
Op);
3081 if (
I && isOpcodeOrAlt(
I))
3086 void setOperations(
const InstructionsState &S) {
3100 unsigned getOpcode()
const {
3101 return MainOp ? MainOp->
getOpcode() : 0;
3104 unsigned getAltOpcode()
const {
3110 int findLaneForValue(
Value *V)
const {
3111 unsigned FoundLane = std::distance(Scalars.begin(),
find(Scalars, V));
3112 assert(FoundLane < Scalars.size() &&
"Couldn't find extract lane");
3113 if (!ReorderIndices.
empty())
3114 FoundLane = ReorderIndices[FoundLane];
3115 assert(FoundLane < Scalars.size() &&
"Couldn't find extract lane");
3116 if (!ReuseShuffleIndices.
empty()) {
3117 FoundLane = std::distance(ReuseShuffleIndices.
begin(),
3118 find(ReuseShuffleIndices, FoundLane));
3132 bool isNonPowOf2Vec()
const {
3134 assert((!IsNonPowerOf2 || ReuseShuffleIndices.
empty()) &&
3135 "Reshuffling not supported with non-power-of-2 vectors yet.");
3136 return IsNonPowerOf2;
3143 for (
unsigned OpI = 0, OpE =
Operands.size(); OpI != OpE; ++OpI) {
3144 dbgs() <<
"Operand " << OpI <<
":\n";
3145 for (
const Value *V : Operands[OpI])
3148 dbgs() <<
"Scalars: \n";
3149 for (
Value *V : Scalars)
3151 dbgs() <<
"State: ";
3154 dbgs() <<
"Vectorize\n";
3156 case ScatterVectorize:
3157 dbgs() <<
"ScatterVectorize\n";
3159 case StridedVectorize:
3160 dbgs() <<
"StridedVectorize\n";
3163 dbgs() <<
"NeedToGather\n";
3166 dbgs() <<
"MainOp: ";
3168 dbgs() << *MainOp <<
"\n";
3171 dbgs() <<
"AltOp: ";
3173 dbgs() << *AltOp <<
"\n";
3176 dbgs() <<
"VectorizedValue: ";
3177 if (VectorizedValue)
3178 dbgs() << *VectorizedValue <<
"\n";
3181 dbgs() <<
"ReuseShuffleIndices: ";
3182 if (ReuseShuffleIndices.
empty())
3185 for (
int ReuseIdx : ReuseShuffleIndices)
3186 dbgs() << ReuseIdx <<
", ";
3188 dbgs() <<
"ReorderIndices: ";
3189 for (
unsigned ReorderIdx : ReorderIndices)
3190 dbgs() << ReorderIdx <<
", ";
3192 dbgs() <<
"UserTreeIndices: ";
3193 for (
const auto &EInfo : UserTreeIndices)
3194 dbgs() << EInfo <<
", ";
3201 void dumpTreeCosts(
const TreeEntry *E,
InstructionCost ReuseShuffleCost,
3204 dbgs() <<
"SLP: " << Banner <<
":\n";
3206 dbgs() <<
"SLP: Costs:\n";
3207 dbgs() <<
"SLP: ReuseShuffleCost = " << ReuseShuffleCost <<
"\n";
3208 dbgs() <<
"SLP: VectorCost = " << VecCost <<
"\n";
3209 dbgs() <<
"SLP: ScalarCost = " << ScalarCost <<
"\n";
3210 dbgs() <<
"SLP: ReuseShuffleCost + VecCost - ScalarCost = "
3211 << ReuseShuffleCost + VecCost - ScalarCost <<
"\n";
3217 std::optional<ScheduleData *> Bundle,
3218 const InstructionsState &S,
3219 const EdgeInfo &UserTreeIdx,
3222 TreeEntry::EntryState EntryState =
3223 Bundle ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
3224 return newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
3225 ReuseShuffleIndices, ReorderIndices);
3229 TreeEntry::EntryState EntryState,
3230 std::optional<ScheduleData *> Bundle,
3231 const InstructionsState &S,
3232 const EdgeInfo &UserTreeIdx,
3235 assert(((!Bundle && EntryState == TreeEntry::NeedToGather) ||
3236 (Bundle && EntryState != TreeEntry::NeedToGather)) &&
3237 "Need to vectorize gather entry?");
3238 VectorizableTree.
push_back(std::make_unique<TreeEntry>(VectorizableTree));
3239 TreeEntry *
Last = VectorizableTree.
back().get();
3240 Last->Idx = VectorizableTree.
size() - 1;
3241 Last->State = EntryState;
3242 Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
3243 ReuseShuffleIndices.end());
3244 if (ReorderIndices.
empty()) {
3246 Last->setOperations(S);
3249 Last->Scalars.assign(VL.
size(),
nullptr);
3252 if (Idx >= VL.size())
3253 return UndefValue::get(VL.front()->getType());
3257 Last->setOperations(S);
3258 Last->ReorderIndices.append(ReorderIndices.
begin(), ReorderIndices.
end());
3260 if (!
Last->isGather()) {
3261 for (
Value *V : VL) {
3262 const TreeEntry *
TE = getTreeEntry(V);
3264 "Scalar already in tree!");
3267 MultiNodeScalars.try_emplace(V).first->getSecond().push_back(
Last);
3270 ScalarToTreeEntry[
V] =
Last;
3273 ScheduleData *BundleMember = *Bundle;
3274 assert((BundleMember || isa<PHINode>(S.MainOp) ||
3277 "Bundle and VL out of sync");
3279 for (
Value *V : VL) {
3284 BundleMember->TE =
Last;
3285 BundleMember = BundleMember->NextInBundle;
3288 assert(!BundleMember &&
"Bundle and VL out of sync");
3291 bool AllConstsOrCasts =
true;
3294 auto *
I = dyn_cast<CastInst>(V);
3295 AllConstsOrCasts &=
I &&
I->getType()->isIntegerTy();
3298 if (AllConstsOrCasts)
3300 std::make_pair(std::numeric_limits<unsigned>::max(), 1);
3301 MustGather.
insert(VL.begin(), VL.end());
3304 if (UserTreeIdx.UserTE) {
3305 Last->UserTreeIndices.push_back(UserTreeIdx);
3306 assert((!
Last->isNonPowOf2Vec() ||
Last->ReorderIndices.empty()) &&
3307 "Reordering isn't implemented for non-power-of-2 nodes yet");
3314 TreeEntry::VecTreeTy VectorizableTree;
3319 for (
unsigned Id = 0, IdE = VectorizableTree.size(); Id != IdE; ++Id) {
3320 VectorizableTree[
Id]->dump();
3326 TreeEntry *getTreeEntry(
Value *V) {
return ScalarToTreeEntry.lookup(V); }
3328 const TreeEntry *getTreeEntry(
Value *V)
const {
3329 return ScalarToTreeEntry.lookup(V);
3338 bool areAltOperandsProfitable(
const InstructionsState &S,
3343 TreeEntry::EntryState getScalarsVectorizationState(
3376 using ValueToGatherNodesMap =
3378 ValueToGatherNodesMap ValueToGatherNodes;
3381 struct ExternalUser {
3405 AliasCacheKey
Key = std::make_pair(Inst1, Inst2);
3406 auto It = AliasCache.
find(Key);
3407 if (It != AliasCache.
end())
3412 AliasCache.
try_emplace(std::make_pair(Inst2, Inst1), Aliased);
3416 using AliasCacheKey = std::pair<Instruction *, Instruction *>;
3448 UserList ExternalUses;
3468 struct ScheduleData {
3471 enum { InvalidDeps = -1 };
3473 ScheduleData() =
default;
3475 void init(
int BlockSchedulingRegionID,
Value *OpVal) {
3476 FirstInBundle =
this;
3477 NextInBundle =
nullptr;
3478 NextLoadStore =
nullptr;
3479 IsScheduled =
false;
3480 SchedulingRegionID = BlockSchedulingRegionID;
3481 clearDependencies();
3488 if (hasValidDependencies()) {
3489 assert(UnscheduledDeps <= Dependencies &&
"invariant");
3491 assert(UnscheduledDeps == Dependencies &&
"invariant");
3495 assert(isSchedulingEntity() &&
3496 "unexpected scheduled state");
3497 for (
const ScheduleData *BundleMember =
this; BundleMember;
3498 BundleMember = BundleMember->NextInBundle) {
3499 assert(BundleMember->hasValidDependencies() &&
3500 BundleMember->UnscheduledDeps == 0 &&
3501 "unexpected scheduled state");
3502 assert((BundleMember ==
this || !BundleMember->IsScheduled) &&
3503 "only bundle is marked scheduled");
3507 assert(Inst->getParent() == FirstInBundle->Inst->getParent() &&
3508 "all bundle members must be in same basic block");
3514 bool hasValidDependencies()
const {
return Dependencies != InvalidDeps; }
3518 bool isSchedulingEntity()
const {
return FirstInBundle ==
this; }
3522 bool isPartOfBundle()
const {
3523 return NextInBundle !=
nullptr || FirstInBundle !=
this ||
TE;
3528 bool isReady()
const {
3529 assert(isSchedulingEntity() &&
3530 "can't consider non-scheduling entity for ready list");
3531 return unscheduledDepsInBundle() == 0 && !IsScheduled;
3537 int incrementUnscheduledDeps(
int Incr) {
3538 assert(hasValidDependencies() &&
3539 "increment of unscheduled deps would be meaningless");
3540 UnscheduledDeps += Incr;
3541 return FirstInBundle->unscheduledDepsInBundle();
3546 void resetUnscheduledDeps() {
3547 UnscheduledDeps = Dependencies;
3551 void clearDependencies() {
3552 Dependencies = InvalidDeps;
3553 resetUnscheduledDeps();
3554 MemoryDependencies.clear();
3555 ControlDependencies.clear();
3558 int unscheduledDepsInBundle()
const {
3559 assert(isSchedulingEntity() &&
"only meaningful on the bundle");
3561 for (
const ScheduleData *BundleMember =
this; BundleMember;
3562 BundleMember = BundleMember->NextInBundle) {
3563 if (BundleMember->UnscheduledDeps == InvalidDeps)
3565 Sum += BundleMember->UnscheduledDeps;
3571 if (!isSchedulingEntity()) {
3572 os <<
"/ " << *Inst;
3573 }
else if (NextInBundle) {
3575 ScheduleData *SD = NextInBundle;
3577 os <<
';' << *SD->Inst;
3578 SD = SD->NextInBundle;
3589 Value *OpValue =
nullptr;
3592 TreeEntry *
TE =
nullptr;
3596 ScheduleData *FirstInBundle =
nullptr;
3600 ScheduleData *NextInBundle =
nullptr;
3604 ScheduleData *NextLoadStore =
nullptr;
3618 int SchedulingRegionID = 0;
3621 int SchedulingPriority = 0;
3627 int Dependencies = InvalidDeps;
3633 int UnscheduledDeps = InvalidDeps;
3637 bool IsScheduled =
false;
3642 const BoUpSLP::ScheduleData &SD) {
3667 struct BlockScheduling {
3669 : BB(BB), ChunkSize(BB->
size()), ChunkPos(ChunkSize) {}
3673 ScheduleStart =
nullptr;
3674 ScheduleEnd =
nullptr;
3675 FirstLoadStoreInRegion =
nullptr;
3676 LastLoadStoreInRegion =
nullptr;
3677 RegionHasStackSave =
false;
3681 ScheduleRegionSizeLimit -= ScheduleRegionSize;
3684 ScheduleRegionSize = 0;
3688 ++SchedulingRegionID;
3692 if (BB !=
I->getParent())
3695 ScheduleData *SD = ScheduleDataMap.lookup(
I);
3696 if (SD && isInSchedulingRegion(SD))
3701 ScheduleData *getScheduleData(
Value *V) {
3702 if (
auto *
I = dyn_cast<Instruction>(V))
3703 return getScheduleData(
I);
3707 ScheduleData *getScheduleData(
Value *V,
Value *Key) {
3709 return getScheduleData(V);
3710 auto I = ExtraScheduleDataMap.find(V);
3711 if (
I != ExtraScheduleDataMap.end()) {
3712 ScheduleData *SD =
I->second.lookup(Key);
3713 if (SD && isInSchedulingRegion(SD))
3719 bool isInSchedulingRegion(ScheduleData *SD)
const {
3720 return SD->SchedulingRegionID == SchedulingRegionID;
3725 template <
typename ReadyListType>
3726 void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
3727 SD->IsScheduled =
true;
3730 for (ScheduleData *BundleMember = SD; BundleMember;
3731 BundleMember = BundleMember->NextInBundle) {
3732 if (BundleMember->Inst != BundleMember->OpValue)
3738 auto &&DecrUnsched = [
this, &ReadyList](
Instruction *
I) {
3739 doForAllOpcodes(
I, [&ReadyList](ScheduleData *OpDef) {
3740 if (OpDef && OpDef->hasValidDependencies() &&
3741 OpDef->incrementUnscheduledDeps(-1) == 0) {
3745 ScheduleData *DepBundle = OpDef->FirstInBundle;
3746 assert(!DepBundle->IsScheduled &&
3747 "already scheduled bundle gets ready");
3748 ReadyList.insert(DepBundle);
3750 <<
"SLP: gets ready (def): " << *DepBundle <<
"\n");
3758 if (TreeEntry *TE = BundleMember->TE) {
3760 int Lane = std::distance(
TE->Scalars.begin(),
3761 find(
TE->Scalars, BundleMember->Inst));
3762 assert(Lane >= 0 &&
"Lane not set");
3770 auto *
In = BundleMember->Inst;
3773 (isa<ExtractValueInst, ExtractElementInst, IntrinsicInst>(In) ||
3774 In->getNumOperands() ==
TE->getNumOperands()) &&
3775 "Missed TreeEntry operands?");
3778 for (
unsigned OpIdx = 0, NumOperands =
TE->getNumOperands();
3779 OpIdx != NumOperands; ++OpIdx)
3780 if (
auto *
I = dyn_cast<Instruction>(
TE->getOperand(OpIdx)[Lane]))
3785 for (
Use &U : BundleMember->Inst->operands())
3786 if (
auto *
I = dyn_cast<Instruction>(
U.get()))
3790 for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
3791 if (MemoryDepSD->hasValidDependencies() &&
3792 MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
3795 ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
3796 assert(!DepBundle->IsScheduled &&
3797 "already scheduled bundle gets ready");
3798 ReadyList.insert(DepBundle);
3800 <<
"SLP: gets ready (mem): " << *DepBundle <<
"\n");
3804 for (ScheduleData *DepSD : BundleMember->ControlDependencies) {
3805 if (DepSD->incrementUnscheduledDeps(-1) == 0) {
3808 ScheduleData *DepBundle = DepSD->FirstInBundle;
3809 assert(!DepBundle->IsScheduled &&
3810 "already scheduled bundle gets ready");
3811 ReadyList.insert(DepBundle);
3813 <<
"SLP: gets ready (ctl): " << *DepBundle <<
"\n");
3824 assert(ScheduleStart->getParent() == ScheduleEnd->getParent() &&
3825 ScheduleStart->comesBefore(ScheduleEnd) &&
3826 "Not a valid scheduling region?");
3828 for (
auto *
I = ScheduleStart;
I != ScheduleEnd;
I =
I->getNextNode()) {
3829 auto *SD = getScheduleData(
I);
3832 assert(isInSchedulingRegion(SD) &&
3833 "primary schedule data not in window?");
3834 assert(isInSchedulingRegion(SD->FirstInBundle) &&
3835 "entire bundle in window!");
3837 doForAllOpcodes(
I, [](ScheduleData *SD) { SD->verify(); });
3840 for (
auto *SD : ReadyInsts) {
3841 assert(SD->isSchedulingEntity() && SD->isReady() &&
3842 "item in ready list not ready?");
3847 void doForAllOpcodes(
Value *V,
3849 if (ScheduleData *SD = getScheduleData(V))
3851 auto I = ExtraScheduleDataMap.find(V);
3852 if (
I != ExtraScheduleDataMap.end())
3853 for (
auto &
P :
I->second)
3854 if (isInSchedulingRegion(
P.second))
3859 template <
typename ReadyListType>
3860 void initialFillReadyList(ReadyListType &ReadyList) {
3861 for (
auto *
I = ScheduleStart;
I != ScheduleEnd;
I =
I->getNextNode()) {
3862 doForAllOpcodes(
I, [&](ScheduleData *SD) {
3863 if (SD->isSchedulingEntity() && SD->hasValidDependencies() &&
3865 ReadyList.insert(SD);
3867 <<
"SLP: initially in ready list: " << *SD <<
"\n");
3882 std::optional<ScheduleData *>
3884 const InstructionsState &S);
3890 ScheduleData *allocateScheduleDataChunks();
3894 bool extendSchedulingRegion(
Value *V,
const InstructionsState &S);
3899 ScheduleData *PrevLoadStore,
3900 ScheduleData *NextLoadStore);
3904 void calculateDependencies(ScheduleData *SD,
bool InsertInReadyList,
3908 void resetSchedule();
3929 ExtraScheduleDataMap;
3942 ScheduleData *FirstLoadStoreInRegion =
nullptr;
3946 ScheduleData *LastLoadStoreInRegion =
nullptr;
3951 bool RegionHasStackSave =
false;
3954 int ScheduleRegionSize = 0;
3963 int SchedulingRegionID = 1;
3971 void scheduleBlock(BlockScheduling *BS);
3978 struct OrdersTypeDenseMapInfo {
3991 static unsigned getHashValue(
const OrdersType &V) {
4012 unsigned MaxVecRegSize;
4013 unsigned MinVecRegSize;
4028 unsigned ReductionBitWidth = 0;
4032 std::optional<std::pair<unsigned, unsigned>> CastMaxMinBWSizes;
4051 struct ChildIteratorType
4053 ChildIteratorType, SmallVector<BoUpSLP::EdgeInfo, 1>::iterator> {
4064 return R.VectorizableTree[0].get();
4068 return {
N->UserTreeIndices.begin(),
N->Container};
4072 return {
N->UserTreeIndices.end(),
N->Container};
4077 class nodes_iterator {
4088 bool operator!=(
const nodes_iterator &N2)
const {
return N2.It != It; }
4092 return nodes_iterator(R->VectorizableTree.begin());
4096 return nodes_iterator(R->VectorizableTree.end());
4099 static unsigned size(
BoUpSLP *R) {
return R->VectorizableTree.size(); }
4110 OS << Entry->Idx <<
".\n";
4113 for (
auto *V : Entry->Scalars) {
4115 if (
llvm::any_of(R->ExternalUses, [&](
const BoUpSLP::ExternalUser &EU) {
4116 return EU.Scalar == V;
4126 if (Entry->isGather())
4128 if (Entry->State == TreeEntry::ScatterVectorize ||
4129 Entry->State == TreeEntry::StridedVectorize)
4130 return "color=blue";
4139 for (
auto *
I : DeletedInstructions) {
4140 if (!
I->getParent()) {
4143 if (isa<PHINode>(
I))
4145 I->insertBefore(
F->getEntryBlock(),
4146 F->getEntryBlock().getFirstNonPHIIt());
4148 I->insertBefore(
F->getEntryBlock().getTerminator());
4151 for (
Use &U :
I->operands()) {
4152 auto *
Op = dyn_cast<Instruction>(U.get());
4153 if (
Op && !DeletedInstructions.count(
Op) &&
Op->hasOneUser() &&
4157 I->dropAllReferences();
4159 for (
auto *
I : DeletedInstructions) {
4161 "trying to erase instruction with users.");
4162 I->eraseFromParent();
4168#ifdef EXPENSIVE_CHECKS
4179 assert(!Mask.empty() && Reuses.
size() == Mask.size() &&
4180 "Expected non-empty mask.");
4183 for (
unsigned I = 0,
E = Prev.
size();
I <
E; ++
I)
4185 Reuses[Mask[
I]] = Prev[
I];
4193 bool BottomOrder =
false) {
4194 assert(!Mask.empty() &&
"Expected non-empty mask.");
4195 unsigned Sz = Mask.size();
4198 if (Order.
empty()) {
4200 std::iota(PrevOrder.
begin(), PrevOrder.
end(), 0);
4202 PrevOrder.
swap(Order);
4205 for (
unsigned I = 0;
I < Sz; ++
I)
4207 Order[
I] = PrevOrder[Mask[
I]];
4209 return Data.value() == Sz ||
Data.index() ==
Data.value();
4218 if (Order.
empty()) {
4220 std::iota(MaskOrder.
begin(), MaskOrder.
end(), 0);
4230 for (
unsigned I = 0;
I < Sz; ++
I)
4232 Order[MaskOrder[
I]] =
I;
4236std::optional<BoUpSLP::OrdersType>
4238 assert(TE.isGather() &&
"Expected gather node only.");
4242 Type *ScalarTy = GatheredScalars.
front()->getType();
4243 int NumScalars = GatheredScalars.
size();
4245 return std::nullopt;
4248 if (NumParts == 0 || NumParts >= NumScalars)
4254 tryToGatherExtractElements(GatheredScalars, ExtractMask, NumParts);
4256 isGatherShuffledEntry(&TE, GatheredScalars, Mask, Entries, NumParts,
4259 if (GatherShuffles.
empty() && ExtractShuffles.
empty())
4260 return std::nullopt;
4261 OrdersType CurrentOrder(NumScalars, NumScalars);
4262 if (GatherShuffles.
size() == 1 &&
4264 Entries.front().front()->isSame(TE.Scalars)) {
4267 std::iota(CurrentOrder.
begin(), CurrentOrder.
end(), 0);
4268 return CurrentOrder;
4272 return all_of(Mask, [&](
int I) {
4279 if ((ExtractShuffles.
empty() && IsSplatMask(Mask) &&
4280 (Entries.size() != 1 ||
4281 Entries.front().front()->ReorderIndices.empty())) ||
4282 (GatherShuffles.
empty() && IsSplatMask(ExtractMask)))
4283 return std::nullopt;
4288 for (
int I : seq<int>(0, NumParts)) {
4289 if (ShuffledSubMasks.
test(
I))
4291 const int VF = GetVF(
I);
4297 if (
any_of(Slice, [&](
int I) {
return I != NumScalars; })) {
4298 std::fill(Slice.
begin(), Slice.
end(), NumScalars);
4299 ShuffledSubMasks.
set(
I);
4303 int FirstMin = INT_MAX;
4304 int SecondVecFound =
false;
4305 for (
int K : seq<int>(Limit)) {
4306 int Idx = Mask[
I * PartSz + K];
4308 Value *V = GatheredScalars[
I * PartSz + K];
4310 SecondVecFound =
true;
4319 SecondVecFound =
true;
4323 FirstMin = (FirstMin / PartSz) * PartSz;
4325 if (SecondVecFound) {
4326 std::fill(Slice.
begin(), Slice.
end(), NumScalars);
4327 ShuffledSubMasks.
set(
I);
4330 for (
int K : seq<int>(Limit)) {
4331 int Idx = Mask[
I * PartSz + K];
4335 if (
Idx >= PartSz) {
4336 SecondVecFound =
true;
4339 if (CurrentOrder[
I * PartSz +
Idx] >
4340 static_cast<unsigned>(
I * PartSz + K) &&
4341 CurrentOrder[
I * PartSz +
Idx] !=
4342 static_cast<unsigned>(
I * PartSz +
Idx))
4343 CurrentOrder[
I * PartSz +
Idx] =
I * PartSz + K;
4346 if (SecondVecFound) {
4347 std::fill(Slice.
begin(), Slice.
end(), NumScalars);
4348 ShuffledSubMasks.
set(
I);
4354 if (!ExtractShuffles.
empty())
4355 TransformMaskToOrder(
4356 CurrentOrder, ExtractMask, PartSz, NumParts, [&](
unsigned I) {
4357 if (!ExtractShuffles[
I])
4360 unsigned Sz =
getNumElems(TE.getVectorFactor(), PartSz,
I);
4361 for (
unsigned Idx : seq<unsigned>(Sz)) {
4362 int K =
I * PartSz +
Idx;
4365 if (!TE.ReuseShuffleIndices.empty())
4366 K = TE.ReuseShuffleIndices[K];
4367 if (!TE.ReorderIndices.empty())
4368 K = std::distance(TE.ReorderIndices.begin(),
4369 find(TE.ReorderIndices, K));
4370 auto *EI = dyn_cast<ExtractElementInst>(TE.Scalars[K]);
4373 VF = std::max(VF, cast<VectorType>(EI->getVectorOperandType())
4375 .getKnownMinValue());
4380 if (GatherShuffles.
size() == 1 && NumParts != 1) {
4381 if (ShuffledSubMasks.
any())
4382 return std::nullopt;
4383 PartSz = NumScalars;
4386 if (!Entries.empty())
4387 TransformMaskToOrder(CurrentOrder, Mask, PartSz, NumParts, [&](
unsigned I) {
4388 if (!GatherShuffles[
I])
4390 return std::max(Entries[
I].front()->getVectorFactor(),
4391 Entries[
I].back()->getVectorFactor());
4394 count_if(CurrentOrder, [&](
int Idx) {
return Idx == NumScalars; });
4395 if (ShuffledSubMasks.
all() || (NumScalars > 2 && NumUndefs >= NumScalars / 2))
4396 return std::nullopt;
4397 return std::move(CurrentOrder);
4402 bool CompareOpcodes =
true) {
4405 auto *GEP1 = dyn_cast<GetElementPtrInst>(Ptr1);
4408 auto *GEP2 = dyn_cast<GetElementPtrInst>(Ptr2);
4411 return GEP1->getNumOperands() == 2 && GEP2->getNumOperands() == 2 &&
4415 getSameOpcode({GEP1->getOperand(1), GEP2->getOperand(1)}, TLI)
4420template <
typename T>
4422 Align CommonAlignment = cast<T>(VL.
front())->getAlign();
4424 CommonAlignment = std::min(CommonAlignment, cast<T>(V)->
getAlign());
4425 return CommonAlignment;
4430 unsigned Sz = Order.
size();
4432 return Pair.value() == Sz || Sz - Pair.index() - 1 == Pair.value();
4443static std::optional<Value *>
4449 const SCEV *PtrSCEVLowest =
nullptr;
4450 const SCEV *PtrSCEVHighest =
nullptr;
4456 return std::nullopt;
4458 if (!PtrSCEVLowest && !PtrSCEVHighest) {
4459 PtrSCEVLowest = PtrSCEVHighest = PtrSCEV;
4463 if (isa<SCEVCouldNotCompute>(Diff))
4464 return std::nullopt;
4466 PtrSCEVLowest = PtrSCEV;
4470 if (isa<SCEVCouldNotCompute>(Diff1))
4471 return std::nullopt;
4473 PtrSCEVHighest = PtrSCEV;
4479 if (isa<SCEVCouldNotCompute>(Dist))
4480 return std::nullopt;
4481 int Size =
DL.getTypeStoreSize(ElemTy);
4482 auto TryGetStride = [&](
const SCEV *Dist,
4483 const SCEV *Multiplier) ->
const SCEV * {
4484 if (
const auto *M = dyn_cast<SCEVMulExpr>(Dist)) {
4485 if (M->getOperand(0) == Multiplier)
4486 return M->getOperand(1);
4487 if (M->getOperand(1) == Multiplier)
4488 return M->getOperand(0);
4491 if (Multiplier == Dist)
4496 const SCEV *Stride =
nullptr;
4497 if (
Size != 1 || SCEVs.
size() > 2) {
4499 Stride = TryGetStride(Dist, Sz);
4501 return std::nullopt;
4503 if (!Stride || isa<SCEVConstant>(Stride))
4504 return std::nullopt;
4507 using DistOrdPair = std::pair<int64_t, int>;
4509 std::set<DistOrdPair,
decltype(Compare)> Offsets(Compare);
4511 bool IsConsecutive =
true;
4512 for (
const SCEV *PtrSCEV : SCEVs) {
4514 if (PtrSCEV != PtrSCEVLowest) {
4516 const SCEV *Coeff = TryGetStride(Diff, Stride);
4518 return std::nullopt;
4519 const auto *SC = dyn_cast<SCEVConstant>(Coeff);
4520 if (!SC || isa<SCEVCouldNotCompute>(SC))
4521 return std::nullopt;
4525 return std::nullopt;
4526 Dist = SC->getAPInt().getZExtValue();
4530 return std::nullopt;
4531 auto Res = Offsets.emplace(Dist, Cnt);
4533 return std::nullopt;
4535 IsConsecutive = IsConsecutive && std::next(Res.first) == Offsets.end();
4538 if (Offsets.size() != SCEVs.
size())
4539 return std::nullopt;
4540 SortedIndices.
clear();
4541 if (!IsConsecutive) {
4545 for (
const std::pair<int64_t, int> &Pair : Offsets) {
4546 SortedIndices[Cnt] = Pair.second;
4556static std::pair<InstructionCost, InstructionCost>
4572 if (
DL->getTypeSizeInBits(ScalarTy) !=
DL->getTypeAllocSizeInBits(ScalarTy))
4578 const unsigned Sz = VL.
size();
4580 auto *POIter = PointerOps.
begin();
4581 for (
Value *V : VL) {
4582 auto *L = cast<LoadInst>(V);
4585 *POIter = L->getPointerOperand();
4596 "supported with VectorizeNonPowerOf2");
4600 Align CommonAlignment = computeCommonAlignment<LoadInst>(VL);
4611 if (Order.
empty()) {
4612 Ptr0 = PointerOps.
front();
4613 PtrN = PointerOps.
back();
4615 Ptr0 = PointerOps[Order.
front()];
4616 PtrN = PointerOps[Order.
back()];
4618 std::optional<int> Diff =
4621 if (
static_cast<unsigned>(*Diff) == Sz - 1)
4624 bool IsPossibleStrided = *Diff % (Sz - 1) == 0;
4638 auto IsAnyPointerUsedOutGraph =
4639 IsPossibleStrided &&
any_of(PointerOps, [&](
Value *V) {
4640 return isa<Instruction>(V) &&
any_of(V->users(), [&](
User *U) {
4641 return !getTreeEntry(U) && !MustGather.contains(U);
4644 if (IsPossibleStrided && (IsAnyPointerUsedOutGraph ||
4646 (
static_cast<unsigned>(std::abs(*Diff)) <=
4649 static_cast<unsigned>(std::abs(*Diff)) > Sz) ||
4650 *Diff == -(
static_cast<int>(Sz) - 1))) {
4651 int Stride = *Diff /
static_cast<int>(Sz - 1);
4652 if (*Diff == Stride *
static_cast<int>(Sz - 1)) {
4664 else if (
Ptr != Ptr0)
4669 if (((Dist / Stride) * Stride) != Dist ||
4670 !Dists.
insert(Dist).second)
4673 if (Dists.
size() == Sz)
4679 auto CheckForShuffledLoads = [&, &
TTI = *
TTI](
Align CommonAlignment) {
4680 unsigned Sz =
DL->getTypeSizeInBits(ScalarTy);
4682 unsigned MaxVF = std::max<unsigned>(
bit_floor(VL.
size() / 2), MinVF);
4683 MaxVF = std::min(
getMaximumVF(Sz, Instruction::Load), MaxVF);
4684 for (
unsigned VF = MaxVF; VF >= MinVF; VF /= 2) {
4685 unsigned VectorizedCnt = 0;
4687 for (
unsigned Cnt = 0,
End = VL.
size(); Cnt + VF <=
End;
4688 Cnt += VF, ++VectorizedCnt) {
4706 if (VectorizedCnt == VL.
size() / VF) {
4709 auto [ScalarGEPCost, VectorGEPCost] =
getGEPCosts(
4710 TTI, PointerOps, PointerOps.
front(), Instruction::GetElementPtr,
4714 Instruction::Load, VecTy,
4716 false, CommonAlignment,
CostKind) +
4717 VectorGEPCost - ScalarGEPCost;
4721 auto *LI0 = cast<LoadInst>(VL[
I * VF]);
4724 auto [ScalarGEPCost, VectorGEPCost] =
4726 LI0->getPointerOperand(), Instruction::Load,
4729 Instruction::Load, SubVecTy, LI0->getAlign(),
4730 LI0->getPointerAddressSpace(),
CostKind,
4732 VectorGEPCost - ScalarGEPCost;
4736 auto [ScalarGEPCost, VectorGEPCost] =
4738 LI0->getPointerOperand(), Instruction::Load,
4742 Instruction::Load, SubVecTy, LI0->getPointerOperand(),
4743 false, CommonAlignment,
CostKind) +
4744 VectorGEPCost - ScalarGEPCost;
4748 auto [ScalarGEPCost, VectorGEPCost] =
getGEPCosts(
4750 LI0->getPointerOperand(), Instruction::GetElementPtr,
4754 Instruction::Load, SubVecTy, LI0->getPointerOperand(),
4755 false, CommonAlignment,
CostKind) +
4756 VectorGEPCost - ScalarGEPCost;
4761 "Expected only consecutive, strided or masked gather loads.");
4764 for (
int Idx : seq<int>(0, VL.
size()))
4773 if (MaskedGatherCost >= VecLdCost)
4783 bool ProfitableGatherPointers =
4786 return L->isLoopInvariant(V);
4788 if (ProfitableGatherPointers ||
all_of(PointerOps, [IsSorted](
Value *
P) {
4789 auto *
GEP = dyn_cast<GetElementPtrInst>(
P);
4791 (
GEP &&
GEP->getNumOperands() == 2 &&
4792 isa<Constant, Instruction>(
GEP->getOperand(1)));
4794 Align CommonAlignment = computeCommonAlignment<LoadInst>(VL);
4799 if (TryRecursiveCheck && CheckForShuffledLoads(CommonAlignment)) {
4818 "Expected list of pointer operands.");
4823 Bases[VL[0]].push_back(std::make_tuple(VL[0], 0U, 0U));
4828 std::optional<int> Diff =
4834 Base.second.emplace_back(
Ptr, *Diff, Cnt++);
4840 if (Bases.
size() > VL.
size() / 2 - 1)
4844 Bases[
Ptr].emplace_back(
Ptr, 0, Cnt++);
4850 bool AnyConsecutive =
false;
4851 for (
auto &
Base : Bases) {
4852 auto &Vec =
Base.second;
4853 if (Vec.size() > 1) {
4855 const std::tuple<Value *, int, unsigned> &
Y) {
4856 return std::get<1>(
X) < std::get<1>(
Y);
4858 int InitialOffset = std::get<1>(Vec[0]);
4860 return std::get<1>(
P.value()) == int(
P.index()) + InitialOffset;
4866 SortedIndices.
clear();
4867 if (!AnyConsecutive)
4875 for (
auto &
Base : Bases) {
4876 Value *Strip =
Base.first->stripInBoundsConstantOffsets();
4877 Value *Root = Strip;
4878 while (
auto *Gep = dyn_cast<GetElementPtrInst>(Root))
4879 Root = Gep->getOperand(0);
4882 auto *Begin = SortedBases.
begin();
4883 auto *
End = SortedBases.
end();
4884 while (Begin !=
End) {
4885 Value *Root = std::get<2>(*Begin);
4886 auto *Mid = std::stable_partition(
4887 Begin,
End, [&Root](
auto V) {
return std::get<2>(V) == Root; });
4889 for (
auto I = Begin;
I < Mid; ++
I)
4890 LessThan.try_emplace(std::get<1>(*
I));
4891 for (
auto I = Begin;
I < Mid; ++
I) {
4892 Value *V = std::get<1>(*
I);
4893 while (
auto *Gep = dyn_cast<GetElementPtrInst>(V)) {
4894 V = Gep->getOperand(0);
4895 if (LessThan.contains(V))
4896 LessThan[V][std::get<1>(*
I)] =
true;
4899 std::stable_sort(Begin, Mid, [&LessThan](
auto &V1,
auto &V2) {
4900 return LessThan[std::get<1>(V1)][std::get<1>(V2)];
4906 for (
auto Base : SortedBases)
4907 for (
auto &
T : Bases[std::get<0>(
Base)])
4911 "Expected SortedIndices to be the size of VL");
4915std::optional<BoUpSLP::OrdersType>
4917 assert(TE.isGather() &&
"Expected gather node only.");
4918 Type *ScalarTy = TE.Scalars[0]->getType();
4921 Ptrs.
reserve(TE.Scalars.size());
4922 for (
Value *V : TE.Scalars) {
4923 auto *L = dyn_cast<LoadInst>(V);
4924 if (!L || !L->isSimple())
4925 return std::nullopt;
4931 return std::move(Order);
4932 return std::nullopt;
4943 if (VU->
getType() != V->getType())
4946 if (!VU->
hasOneUse() && !V->hasOneUse())
4952 if (Idx1 == std::nullopt || Idx2 == std::nullopt)
4958 cast<VectorType>(VU->
getType())->getElementCount().getKnownMinValue());
4959 bool IsReusedIdx =
false;
4961 if (IE2 == VU && !IE1)
4963 if (IE1 == V && !IE2)
4964 return V->hasOneUse();
4965 if (IE1 && IE1 != V) {
4967 IsReusedIdx |= ReusedIdx.
test(Idx1);
4968 ReusedIdx.
set(Idx1);
4969 if ((IE1 != VU && !IE1->
hasOneUse()) || IsReusedIdx)
4972 IE1 = dyn_cast_or_null<InsertElementInst>(GetBaseOperand(IE1));
4974 if (IE2 && IE2 != VU) {
4976 IsReusedIdx |= ReusedIdx.
test(Idx2);
4977 ReusedIdx.
set(Idx2);
4978 if ((IE2 != V && !IE2->hasOneUse()) || IsReusedIdx)
4981 IE2 = dyn_cast_or_null<InsertElementInst>(GetBaseOperand(IE2));
4983 }
while (!IsReusedIdx && (IE1 || IE2));
4987std::optional<BoUpSLP::OrdersType>
4990 if (TE.isNonPowOf2Vec())
4991 return std::nullopt;
4995 if (!TE.ReuseShuffleIndices.empty()) {
4997 return std::nullopt;
5005 unsigned Sz = TE.Scalars.size();
5006 if (TE.isGather()) {
5007 if (std::optional<OrdersType> CurrentOrder =
5012 ::addMask(Mask, TE.ReuseShuffleIndices);
5013 OrdersType Res(TE.getVectorFactor(), TE.getVectorFactor());
5014 unsigned Sz = TE.Scalars.size();
5015 for (
int K = 0,
E = TE.getVectorFactor() / Sz; K <
E; ++K) {
5018 Res[
Idx + K * Sz] =
I + K * Sz;
5020 return std::move(Res);
5023 if (Sz == 2 && TE.getVectorFactor() == 4 &&
5025 2 * TE.getVectorFactor())) == 1)
5026 return std::nullopt;
5030 if (TE.ReorderIndices.empty())
5031 std::iota(ReorderMask.
begin(), ReorderMask.
end(), 0);
5034 ::addMask(ReorderMask, TE.ReuseShuffleIndices);
5035 unsigned VF = ReorderMask.
size();
5039 for (
unsigned I = 0;
I < VF;
I += Sz) {
5041 unsigned UndefCnt = 0;
5042 unsigned Limit = std::min(Sz, VF -
I);
5051 Val >=
static_cast<int>(NumParts) || UsedVals.
test(Val) ||
5053 return std::nullopt;
5055 for (
unsigned K = 0; K < NumParts; ++K)
5056 ResOrder[Val + Sz * K] =
I + K;
5058 return std::move(ResOrder);
5060 unsigned VF = TE.getVectorFactor();
5063 TE.ReuseShuffleIndices.end());
5064 if (TE.getOpcode() == Instruction::ExtractElement && !TE.isAltShuffle() &&
5066 std::optional<unsigned> Idx = getExtractIndex(cast<Instruction>(V));
5067 return Idx && *Idx < Sz;
5070 if (TE.ReorderIndices.empty())
5071 std::iota(ReorderMask.
begin(), ReorderMask.
end(), 0);
5074 for (
unsigned I = 0;
I < VF; ++
I) {
5075 int &
Idx = ReusedMask[
I];
5078 Value *V = TE.Scalars[ReorderMask[
Idx]];
5080 Idx = std::distance(ReorderMask.
begin(),
find(ReorderMask, *EI));
5086 std::iota(ResOrder.
begin(), ResOrder.
end(), 0);
5087 auto *It = ResOrder.
begin();
5088 for (
unsigned K = 0; K < VF; K += Sz) {
5092 std::iota(SubMask.begin(), SubMask.end(), 0);
5094 transform(CurrentOrder, It, [K](
unsigned Pos) {
return Pos + K; });
5095 std::advance(It, Sz);
5098 return Data.index() ==
Data.value();
5100 return std::nullopt;
5101 return std::move(ResOrder);
5103 if (TE.State == TreeEntry::StridedVectorize && !TopToBottom &&
5104 any_of(TE.UserTreeIndices,
5106 return !Instruction::isBinaryOp(EI.UserTE->getOpcode());
5108 (TE.ReorderIndices.empty() ||
isReverseOrder(TE.ReorderIndices)))
5109 return std::nullopt;
5110 if ((TE.State == TreeEntry::Vectorize ||
5111 TE.State == TreeEntry::StridedVectorize) &&
5112 (isa<LoadInst, ExtractElementInst, ExtractValueInst>(TE.getMainOp()) ||
5113 (TopToBottom && isa<StoreInst, InsertElementInst>(TE.getMainOp()))) &&
5115 return TE.ReorderIndices;
5116 if (TE.State == TreeEntry::Vectorize && TE.getOpcode() == Instruction::PHI) {
5117 auto PHICompare = [&](
unsigned I1,
unsigned I2) {
5118 Value *V1 = TE.Scalars[I1];
5119 Value *V2 = TE.Scalars[I2];
5120 if (V1 == V2 || (V1->
getNumUses() == 0 && V2->getNumUses() == 0))
5126 auto *FirstUserOfPhi1 = cast<Instruction>(*V1->
user_begin());
5127 auto *FirstUserOfPhi2 = cast<Instruction>(*V2->user_begin());
5128 if (
auto *IE1 = dyn_cast<InsertElementInst>(FirstUserOfPhi1))
5129 if (
auto *IE2 = dyn_cast<InsertElementInst>(FirstUserOfPhi2)) {
5136 if (
auto *EE1 = dyn_cast<ExtractElementInst>(FirstUserOfPhi1))
5137 if (
auto *EE2 = dyn_cast<ExtractElementInst>(FirstUserOfPhi2)) {
5138 if (EE1->getOperand(0) != EE2->getOperand(0))
5144 auto IsIdentityOrder = [](
const OrdersType &Order) {
5145 for (
unsigned Idx : seq<unsigned>(0, Order.size()))
5150 if (!TE.ReorderIndices.empty())
5151 return TE.ReorderIndices;
5154 std::iota(Phis.begin(), Phis.end(), 0);
5156 for (
unsigned Id = 0, Sz = TE.Scalars.size(); Id < Sz; ++Id)
5159 for (
unsigned Id = 0, Sz = Phis.size(); Id < Sz; ++Id)
5160 ResOrder[Id] = PhiToId[Phis[Id]];
5161 if (IsIdentityOrder(ResOrder))
5162 return std::nullopt;
5163 return std::move(ResOrder);
5165 if (TE.isGather() && !TE.isAltShuffle() &&
allSameType(TE.Scalars)) {
5168 if ((TE.getOpcode() == Instruction::ExtractElement ||
5169 (
all_of(TE.Scalars, IsaPred<UndefValue, ExtractElementInst>) &&
5170 any_of(TE.Scalars, IsaPred<ExtractElementInst>))) &&
5172 auto *EE = dyn_cast<ExtractElementInst>(V);
5173 return !EE || isa<FixedVectorType>(EE->getVectorOperandType());
5178 bool Reuse = canReuseExtract(TE.Scalars, TE.getMainOp(), CurrentOrder,
5180 if (Reuse || !CurrentOrder.
empty())
5181 return std::move(CurrentOrder);
5189 int Sz = TE.Scalars.size();
5191 count_if(TE.Scalars, IsaPred<UndefValue>) == Sz - 1) {
5193 find_if(TE.Scalars, [](
Value *V) { return !isConstant(V); });
5194 if (It == TE.Scalars.begin())
5197 if (It != TE.Scalars.end()) {
5199 unsigned Idx = std::distance(TE.Scalars.begin(), It);
5214 if (InsertFirstCost + PermuteCost < InsertIdxCost) {
5217 return std::move(Order);
5222 return std::nullopt;
5223 if (TE.Scalars.size() >= 4)
5227 return CurrentOrder;
5229 return std::nullopt;
5239 for (
unsigned I = Sz,
E = Mask.size();
I <
E;
I += Sz) {
5241 if (Cluster != FirstCluster)
5247void BoUpSLP::reorderNodeWithReuses(TreeEntry &TE,
ArrayRef<int> Mask)
const {
5250 const unsigned Sz =
TE.Scalars.size();
5252 if (!
TE.isGather() ||
5259 addMask(NewMask,
TE.ReuseShuffleIndices);
5261 TE.ReorderIndices.clear();
5268 for (
auto *It =
TE.ReuseShuffleIndices.begin(),
5269 *
End =
TE.ReuseShuffleIndices.end();
5270 It !=
End; std::advance(It, Sz))
5271 std::iota(It, std::next(It, Sz), 0);
5277 "Expected same size of orders");
5278 unsigned Sz = Order.
size();
5280 for (
unsigned Idx : seq<unsigned>(0, Sz)) {
5281 if (Order[
Idx] != Sz)
5282 UsedIndices.
set(Order[
Idx]);
5284 if (SecondaryOrder.
empty()) {
5285 for (
unsigned Idx : seq<unsigned>(0, Sz))
5286 if (Order[
Idx] == Sz && !UsedIndices.
test(
Idx))
5289 for (
unsigned Idx : seq<unsigned>(0, Sz))
5290 if (SecondaryOrder[
Idx] != Sz && Order[
Idx] == Sz &&
5291 !UsedIndices.
test(SecondaryOrder[
Idx]))
5292 Order[
Idx] = SecondaryOrder[
Idx];
5312 ExternalUserReorderMap;
5317 const std::unique_ptr<TreeEntry> &TE) {
5320 findExternalStoreUsersReorderIndices(TE.get());
5321 if (!ExternalUserReorderIndices.
empty()) {
5322 VFToOrderedEntries[TE->getVectorFactor()].
insert(TE.get());
5324 std::move(ExternalUserReorderIndices));
5330 if (TE->isAltShuffle()) {
5333 unsigned Opcode0 = TE->getOpcode();
5334 unsigned Opcode1 = TE->getAltOpcode();
5337 if (TTIRef.isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask)) {
5338 VFToOrderedEntries[TE->getVectorFactor()].
insert(TE.get());
5344 if (std::optional<OrdersType> CurrentOrder =
5354 const TreeEntry *UserTE = TE.get();
5356 if (UserTE->UserTreeIndices.size() != 1)
5359 return EI.UserTE->State == TreeEntry::Vectorize &&
5360 EI.UserTE->isAltShuffle() && EI.UserTE->Idx != 0;
5363 UserTE = UserTE->UserTreeIndices.back().UserTE;
5366 VFToOrderedEntries[TE->getVectorFactor()].
insert(TE.get());
5367 if (!(TE->State == TreeEntry::Vectorize ||
5368 TE->State == TreeEntry::StridedVectorize) ||
5369 !TE->ReuseShuffleIndices.empty())
5370 GathersToOrders.
try_emplace(TE.get(), *CurrentOrder);
5371 if (TE->State == TreeEntry::Vectorize &&
5372 TE->getOpcode() == Instruction::PHI)
5373 PhisToOrders.
try_emplace(TE.get(), *CurrentOrder);
5378 for (
unsigned VF = VectorizableTree.front()->getVectorFactor(); VF > 1;
5380 auto It = VFToOrderedEntries.
find(VF);
5381 if (It == VFToOrderedEntries.
end())
5393 for (
const TreeEntry *OpTE : OrderedEntries) {
5396 if (!OpTE->ReuseShuffleIndices.empty() && !GathersToOrders.
count(OpTE))
5399 const auto &Order = [OpTE, &GathersToOrders, &AltShufflesToOrders,
5401 if (OpTE->isGather() || !OpTE->ReuseShuffleIndices.empty()) {
5402 auto It = GathersToOrders.find(OpTE);
5403 if (It != GathersToOrders.end())
5406 if (OpTE->isAltShuffle()) {
5407 auto It = AltShufflesToOrders.find(OpTE);
5408 if (It != AltShufflesToOrders.end())
5411 if (OpTE->State == TreeEntry::Vectorize &&
5412 OpTE->getOpcode() == Instruction::PHI) {
5413 auto It = PhisToOrders.
find(OpTE);
5414 if (It != PhisToOrders.
end())
5417 return OpTE->ReorderIndices;
5420 auto It = ExternalUserReorderMap.
find(OpTE);
5421 if (It != ExternalUserReorderMap.
end()) {
5422 const auto &ExternalUserReorderIndices = It->second;
5426 if (OpTE->getVectorFactor() != OpTE->Scalars.size()) {
5427 OrdersUses.insert(std::make_pair(
OrdersType(), 0)).first->second +=
5428 ExternalUserReorderIndices.size();
5430 for (
const OrdersType &ExtOrder : ExternalUserReorderIndices)
5431 ++OrdersUses.insert(std::make_pair(ExtOrder, 0)).first->second;
5438 if (OpTE->State == TreeEntry::Vectorize && !OpTE->isAltShuffle() &&
5439 OpTE->getOpcode() == Instruction::Store && !Order.empty()) {
5442 unsigned E = Order.size();
5445 return Idx == PoisonMaskElem ? E : static_cast<unsigned>(Idx);
5448 ++OrdersUses.insert(std::make_pair(CurrentOrder, 0)).first->second;
5450 ++OrdersUses.insert(std::make_pair(Order, 0)).first->second;
5453 if (OrdersUses.empty())
5456 const unsigned Sz = Order.size();
5457 for (
unsigned Idx : seq<unsigned>(0, Sz))
5458 if (
Idx != Order[
Idx] && Order[
Idx] != Sz)
5463 unsigned IdentityCnt = 0;
5464 unsigned FilledIdentityCnt = 0;
5466 for (
auto &Pair : OrdersUses) {
5467 if (Pair.first.empty() || IsIdentityOrder(Pair.first)) {
5468 if (!Pair.first.empty())
5469 FilledIdentityCnt += Pair.second;
5470 IdentityCnt += Pair.second;
5475 unsigned Cnt = IdentityCnt;
5476 for (
auto &Pair : OrdersUses) {
5480 if (Cnt < Pair.second ||
5481 (Cnt == IdentityCnt && IdentityCnt == FilledIdentityCnt &&
5482 Cnt == Pair.second && !BestOrder.
empty() &&
5483 IsIdentityOrder(BestOrder))) {
5485 BestOrder = Pair.first;
5492 if (IsIdentityOrder(BestOrder))
5498 unsigned E = BestOrder.
size();
5500 return I < E ? static_cast<int>(I) : PoisonMaskElem;
5503 for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
5505 if (TE->Scalars.size() != VF) {
5506 if (TE->ReuseShuffleIndices.size() == VF) {
5512 return EI.UserTE->Scalars.size() == VF ||
5513 EI.UserTE->Scalars.size() ==
5516 "All users must be of VF size.");
5519 reorderNodeWithReuses(*TE, Mask);
5523 if ((TE->State == TreeEntry::Vectorize ||
5524 TE->State == TreeEntry::StridedVectorize) &&
5527 !TE->isAltShuffle()) {
5531 if (isa<InsertElementInst, StoreInst>(TE->getMainOp()))
5532 TE->reorderOperands(Mask);
5535 TE->reorderOperands(Mask);
5536 assert(TE->ReorderIndices.empty() &&
5537 "Expected empty reorder sequence.");
5540 if (!TE->ReuseShuffleIndices.empty()) {
5547 addMask(NewReuses, TE->ReuseShuffleIndices);
5548 TE->ReuseShuffleIndices.swap(NewReuses);
5554bool BoUpSLP::canReorderOperands(
5555 TreeEntry *UserTE,
SmallVectorImpl<std::pair<unsigned, TreeEntry *>> &Edges,
5559 if (UserTE->isNonPowOf2Vec())
5562 for (
unsigned I = 0,
E = UserTE->getNumOperands();
I <
E; ++
I) {
5563 if (
any_of(Edges, [
I](
const std::pair<unsigned, TreeEntry *> &OpData) {
5564 return OpData.first ==
I &&
5565 (OpData.second->State == TreeEntry::Vectorize ||
5566 OpData.second->State == TreeEntry::StridedVectorize);
5569 if (TreeEntry *TE = getVectorizedOperand(UserTE,
I)) {
5571 if (
any_of(TE->UserTreeIndices,
5572 [UserTE](
const EdgeInfo &EI) { return EI.UserTE != UserTE; }))
5576 Edges.emplace_back(
I, TE);
5582 if (TE->State != TreeEntry::Vectorize &&
5583 TE->State != TreeEntry::StridedVectorize &&
5584 TE->ReuseShuffleIndices.empty() && TE->ReorderIndices.empty())
5588 TreeEntry *
Gather =
nullptr;
5590 [&
Gather, UserTE,
I](TreeEntry *TE) {
5591 assert(TE->State != TreeEntry::Vectorize &&
5592 TE->State != TreeEntry::StridedVectorize &&
5593 "Only non-vectorized nodes are expected.");
5594 if (
any_of(TE->UserTreeIndices,
5595 [UserTE,
I](
const EdgeInfo &EI) {
5596 return EI.UserTE == UserTE && EI.EdgeIdx == I;
5598 assert(TE->isSame(UserTE->getOperand(
I)) &&
5599 "Operand entry does not match operands.");
5620 for (
const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
5621 if (TE->State != TreeEntry::Vectorize &&
5622 TE->State != TreeEntry::StridedVectorize)
5624 if (std::optional<OrdersType> CurrentOrder =
5626 OrderedEntries.
insert(TE.get());
5627 if (!(TE->State == TreeEntry::Vectorize ||
5628 TE->State == TreeEntry::StridedVectorize) ||
5629 !TE->ReuseShuffleIndices.empty())
5630 GathersToOrders.
insert(TE.get());
5639 while (!OrderedEntries.
empty()) {
5644 for (TreeEntry *TE : OrderedEntries) {
5645 if (!(TE->State == TreeEntry::Vectorize ||
5646 TE->State == TreeEntry::StridedVectorize ||
5647 (TE->isGather() && GathersToOrders.
contains(TE))) ||
5648 TE->UserTreeIndices.empty() || !TE->ReuseShuffleIndices.empty() ||
5651 return EI.UserTE == TE->UserTreeIndices.front().UserTE;
5653 !Visited.
insert(TE).second) {
5659 for (
EdgeInfo &EI : TE->UserTreeIndices) {
5660 TreeEntry *UserTE = EI.
UserTE;
5661 auto It =
Users.find(UserTE);
5662 if (It ==
Users.end())
5663 It =
Users.insert({UserTE, {}}).first;
5664 It->second.emplace_back(EI.
EdgeIdx, TE);
5668 for (TreeEntry *TE : Filtered)
5669 OrderedEntries.remove(TE);
5671 std::pair<TreeEntry *, SmallVector<std::pair<unsigned, TreeEntry *>>>>
5673 sort(UsersVec, [](
const auto &Data1,
const auto &Data2) {
5674 return Data1.first->Idx > Data2.first->Idx;
5676 for (
auto &
Data : UsersVec) {
5679 if (!canReorderOperands(
Data.first,
Data.second, NonVectorized,
5681 for (
const std::pair<unsigned, TreeEntry *> &
Op :
Data.second)
5682 OrderedEntries.remove(
Op.second);
5695 for (
const auto &
Op :
Data.second) {
5696 TreeEntry *OpTE =
Op.second;
5697 if (!VisitedOps.
insert(OpTE).second)
5699 if (!OpTE->ReuseShuffleIndices.empty() && !GathersToOrders.
count(OpTE))
5701 const auto Order = [&]() ->
const OrdersType {
5702 if (OpTE->isGather() || !OpTE->ReuseShuffleIndices.empty())
5705 return OpTE->ReorderIndices;
5709 if (Order.size() == 1)
5712 Data.second, [OpTE](
const std::pair<unsigned, TreeEntry *> &
P) {
5713 return P.second == OpTE;
5716 if (OpTE->State == TreeEntry::Vectorize && !OpTE->isAltShuffle() &&
5717 OpTE->getOpcode() == Instruction::Store && !Order.empty()) {
5720 unsigned E = Order.size();
5723 return Idx == PoisonMaskElem ? E : static_cast<unsigned>(Idx);
5726 OrdersUses.insert(std::make_pair(CurrentOrder, 0)).first->second +=
5729 OrdersUses.insert(std::make_pair(Order, 0)).first->second += NumOps;
5731 auto Res = OrdersUses.insert(std::make_pair(
OrdersType(), 0));
5732 const auto AllowsReordering = [&](
const TreeEntry *TE) {
5734 if (TE->isNonPowOf2Vec())
5736 if (!TE->ReorderIndices.empty() || !TE->ReuseShuffleIndices.empty() ||
5737 (TE->State == TreeEntry::Vectorize && TE->isAltShuffle()) ||
5738 (IgnoreReorder && TE->Idx == 0))
5740 if (TE->isGather()) {
5749 for (
const EdgeInfo &EI : OpTE->UserTreeIndices) {
5750 TreeEntry *UserTE = EI.
UserTE;
5751 if (!VisitedUsers.
insert(UserTE).second)
5756 if (AllowsReordering(UserTE))
5764 if (
static_cast<unsigned>(
count_if(
5765 Ops, [UserTE, &AllowsReordering](
5766 const std::pair<unsigned, TreeEntry *> &
Op) {
5767 return AllowsReordering(
Op.second) &&
5770 return EI.UserTE == UserTE;
5772 })) <= Ops.
size() / 2)
5773 ++Res.first->second;
5776 if (OrdersUses.empty()) {
5777 for (
const std::pair<unsigned, TreeEntry *> &
Op :
Data.second)
5778 OrderedEntries.remove(
Op.second);
5782 const unsigned Sz = Order.size();
5783 for (
unsigned Idx : seq<unsigned>(0, Sz))
5784 if (
Idx != Order[
Idx] && Order[
Idx] != Sz)
5789 unsigned IdentityCnt = 0;
5790 unsigned VF =
Data.second.front().second->getVectorFactor();
5792 for (
auto &Pair : OrdersUses) {
5793 if (Pair.first.empty() || IsIdentityOrder(Pair.first)) {
5794 IdentityCnt += Pair.second;
5799 unsigned Cnt = IdentityCnt;
5800 for (
auto &Pair : OrdersUses) {
5804 if (Cnt < Pair.second) {
5806 BestOrder = Pair.first;
5813 if (IsIdentityOrder(BestOrder)) {
5814 for (
const std::pair<unsigned, TreeEntry *> &
Op :
Data.second)
5815 OrderedEntries.remove(
Op.second);
5824 unsigned E = BestOrder.
size();
5826 return I < E ? static_cast<int>(I) : PoisonMaskElem;
5828 for (
const std::pair<unsigned, TreeEntry *> &
Op :
Data.second) {
5829 TreeEntry *TE =
Op.second;
5830 OrderedEntries.remove(TE);
5831 if (!VisitedOps.
insert(TE).second)
5833 if (TE->ReuseShuffleIndices.size() == BestOrder.
size()) {
5834 reorderNodeWithReuses(*TE, Mask);
5838 if (TE->State != TreeEntry::Vectorize &&
5839 TE->State != TreeEntry::StridedVectorize &&
5840 (TE->State != TreeEntry::ScatterVectorize ||
5841 TE->ReorderIndices.empty()))
5843 assert((BestOrder.
size() == TE->ReorderIndices.size() ||
5844 TE->ReorderIndices.empty()) &&
5845 "Non-matching sizes of user/operand entries.");
5847 if (IgnoreReorder && TE == VectorizableTree.front().get())
5848 IgnoreReorder =
false;
5851 for (TreeEntry *
Gather : GatherOps) {
5853 "Unexpected reordering of gathers.");
5854 if (!
Gather->ReuseShuffleIndices.empty()) {
5860 OrderedEntries.remove(
Gather);
5864 if (
Data.first->State != TreeEntry::Vectorize ||
5865 !isa<ExtractElementInst, ExtractValueInst, LoadInst>(
5866 Data.first->getMainOp()) ||
5867 Data.first->isAltShuffle())
5868 Data.first->reorderOperands(Mask);
5869 if (!isa<InsertElementInst, StoreInst>(
Data.first->getMainOp()) ||
5870 Data.first->isAltShuffle() ||
5871 Data.first->State == TreeEntry::StridedVectorize) {
5875 if (
Data.first->ReuseShuffleIndices.empty() &&
5876 !
Data.first->ReorderIndices.empty() &&
5877 !
Data.first->isAltShuffle()) {
5880 OrderedEntries.insert(
Data.first);
5888 if (IgnoreReorder && !VectorizableTree.front()->ReorderIndices.empty() &&
5889 VectorizableTree.front()->ReuseShuffleIndices.empty())
5890 VectorizableTree.front()->ReorderIndices.clear();
5897 for (
auto &TEPtr : VectorizableTree) {
5898 TreeEntry *Entry = TEPtr.get();
5901 if (Entry->isGather())
5905 for (
int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
5906 Value *Scalar = Entry->Scalars[Lane];
5907 if (!isa<Instruction>(Scalar))
5910 auto It = ScalarToExtUses.
find(Scalar);
5911 if (It != ScalarToExtUses.
end() && !ExternalUses[It->second].User)
5915 const auto *ExtI = ExternallyUsedValues.
find(Scalar);
5916 if (ExtI != ExternallyUsedValues.
end()) {
5917 int FoundLane = Entry->findLaneForValue(Scalar);
5918 LLVM_DEBUG(
dbgs() <<
"SLP: Need to extract: Extra arg from lane "
5919 << FoundLane <<
" from " << *Scalar <<
".\n");
5920 ScalarToExtUses.
try_emplace(Scalar, ExternalUses.size());
5921 ExternalUses.emplace_back(Scalar,
nullptr, FoundLane);
5924 for (
User *U : Scalar->users()) {
5932 if (UserIgnoreList && UserIgnoreList->contains(UserInst))
5936 if (TreeEntry *UseEntry = getTreeEntry(U)) {
5940 if (UseEntry->State == TreeEntry::ScatterVectorize ||
5942 Scalar, cast<Instruction>(UseEntry->Scalars.front()), TLI)) {
5943 LLVM_DEBUG(
dbgs() <<
"SLP: \tInternal user will be removed:" << *U
5945 assert(!UseEntry->isGather() &&
"Bad state");
5949 if (It != ScalarToExtUses.
end()) {
5950 ExternalUses[It->second].User =
nullptr;
5955 if (U && Scalar->hasNUsesOrMore(
UsesLimit))
5957 int FoundLane = Entry->findLaneForValue(Scalar);
5959 <<
" from lane " << FoundLane <<
" from " << *Scalar
5961 It = ScalarToExtUses.
try_emplace(Scalar, ExternalUses.size()).first;
5962 ExternalUses.emplace_back(Scalar, U, FoundLane);
5971BoUpSLP::collectUserStores(
const BoUpSLP::TreeEntry *TE)
const {
5973 for (
unsigned Lane : seq<unsigned>(0, TE->Scalars.size())) {
5974 Value *V = TE->Scalars[Lane];
5980 for (
User *U : V->users()) {
5981 auto *SI = dyn_cast<StoreInst>(U);
5982 if (SI ==
nullptr || !SI->isSimple() ||
5986 if (getTreeEntry(U))
5990 auto &StoresVec = PtrToStoresMap[
Ptr];
5993 if (StoresVec.size() > Lane)
5996 if (!StoresVec.empty() &&
5997 SI->getParent() != StoresVec.back()->getParent())
6000 if (!StoresVec.empty() &&
6001 SI->getValueOperand()->getType() !=
6002 StoresVec.back()->getValueOperand()->getType())
6004 StoresVec.push_back(SI);
6007 return PtrToStoresMap;
6011 OrdersType &ReorderIndices)
const {
6019 StoreOffsetVec[0] = {S0, 0};
6022 for (
unsigned Idx : seq<unsigned>(1, StoresVec.
size())) {
6024 std::optional<int> Diff =
6026 SI->getPointerOperand(), *
DL, *SE,
6031 StoreOffsetVec[
Idx] = {StoresVec[
Idx], *Diff};
6036 stable_sort(StoreOffsetVec, [](
const std::pair<StoreInst *, int> &Pair1,
6037 const std::pair<StoreInst *, int> &Pair2) {
6038 int Offset1 = Pair1.second;
6039 int Offset2 = Pair2.second;
6040 return Offset1 < Offset2;
6044 for (
unsigned Idx : seq<unsigned>(1, StoreOffsetVec.size()))
6045 if (StoreOffsetVec[
Idx].second != StoreOffsetVec[
Idx - 1].second + 1)
6050 ReorderIndices.reserve(StoresVec.
size());
6053 [SI](
const std::pair<StoreInst *, int> &Pair) {
6054 return Pair.first ==
SI;
6056 StoreOffsetVec.begin();
6057 ReorderIndices.push_back(
Idx);
6062 auto IsIdentityOrder = [](
const OrdersType &Order) {
6063 for (
unsigned Idx : seq<unsigned>(0, Order.size()))
6068 if (IsIdentityOrder(ReorderIndices))
6069 ReorderIndices.clear();
6076 for (
unsigned Idx : Order)
6083BoUpSLP::findExternalStoreUsersReorderIndices(TreeEntry *TE)
const {
6084 unsigned NumLanes =
TE->Scalars.size();
6087 collectUserStores(TE);
6096 for (
const auto &Pair : PtrToStoresMap) {
6097 auto &StoresVec = Pair.second;
6099 if (StoresVec.size() != NumLanes)
6104 if (!canFormVector(StoresVec, ReorderIndices))
6109 ExternalReorderIndices.
push_back(ReorderIndices);
6111 return ExternalReorderIndices;
6117 UserIgnoreList = &UserIgnoreLst;
6120 buildTree_rec(Roots, 0,
EdgeInfo());
6127 buildTree_rec(Roots, 0,
EdgeInfo());
6134 Value *NeedsScheduling =
nullptr;
6135 for (
Value *V : VL) {
6138 if (!NeedsScheduling) {
6139 NeedsScheduling = V;
6144 return NeedsScheduling;
6155 bool AllowAlternate) {
6159 if (
auto *LI = dyn_cast<LoadInst>(V)) {
6162 SubKey =
hash_value(LoadsSubkeyGenerator(Key, LI));
6167 if (isa<ExtractElementInst, UndefValue>(V))
6169 if (
auto *EI = dyn_cast<ExtractElementInst>(V)) {
6171 !isa<UndefValue>(EI->getIndexOperand()))
6174 }
else if (
auto *
I = dyn_cast<Instruction>(V)) {
6177 if ((isa<BinaryOperator, CastInst>(
I)) &&
6187 : cast<CastInst>(
I)->getOperand(0)->getType()));
6189 if (isa<CastInst>(
I)) {
6190 std::pair<size_t, size_t> OpVals =
6196 }
else if (
auto *CI = dyn_cast<CmpInst>(
I)) {
6198 if (CI->isCommutative())
6204 }
else if (
auto *Call = dyn_cast<CallInst>(
I)) {
6218 }
else if (
auto *Gep = dyn_cast<GetElementPtrInst>(
I)) {
6219 if (Gep->getNumOperands() == 2 && isa<ConstantInt>(Gep->getOperand(1)))
6220 SubKey =
hash_value(Gep->getPointerOperand());
6224 !isa<ConstantInt>(
I->getOperand(1))) {
6232 return std::make_pair(Key, SubKey);
6242bool BoUpSLP::areAltOperandsProfitable(
const InstructionsState &S,
6244 unsigned Opcode0 = S.getOpcode();
6245 unsigned Opcode1 = S.getAltOpcode();
6249 Opcode0, Opcode1, OpcodeMask))
6252 for (
unsigned I : seq<unsigned>(0, S.MainOp->getNumOperands())) {
6256 Operands.back().push_back(cast<Instruction>(V)->getOperand(
I));
6260 for (
unsigned I : seq<unsigned>(0, VL.size() - 1)) {
6266 switch (Res.value_or(0)) {
6281 constexpr unsigned NumAltInsts = 3;
6282 unsigned NonInstCnt = 0;
6285 unsigned UndefCnt = 0;
6287 unsigned ExtraShuffleInsts = 0;
6296 return is_contained(Operands.back(), V);
6299 ++ExtraShuffleInsts;
6316 if (isa<Constant, ExtractElementInst>(V) ||
6317 getTreeEntry(V) || (L &&
L->isLoopInvariant(V))) {
6318 if (isa<UndefValue>(V))
6324 if (!Res.second && Res.first->second == 1)
6325 ++ExtraShuffleInsts;
6326 ++Res.first->getSecond();
6327 if (
auto *
I = dyn_cast<Instruction>(V))
6328 UniqueOpcodes.
insert(
I->getOpcode());
6329 else if (Res.second)
6332 return none_of(Uniques, [&](
const auto &
P) {
6333 return P.first->hasNUsesOrMore(
P.second + 1) &&
6335 return getTreeEntry(U) || Uniques.contains(U);
6344 (UndefCnt < (VL.size() - 1) * S.MainOp->getNumOperands() &&
6345 (UniqueOpcodes.
size() + NonInstCnt + ExtraShuffleInsts +
6346 NumAltInsts) < S.MainOp->getNumOperands() * VL.size());
6349BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
6352 assert(S.MainOp &&
"Expected instructions with same/alternate opcodes only.");
6354 unsigned ShuffleOrOp =
6355 S.isAltShuffle() ? (
unsigned)Instruction::ShuffleVector : S.getOpcode();
6356 auto *VL0 = cast<Instruction>(S.OpValue);
6357 switch (ShuffleOrOp) {
6358 case Instruction::PHI: {
6361 return TreeEntry::NeedToGather;
6364 for (
Value *
Incoming : cast<PHINode>(V)->incoming_values()) {
6366 if (Term &&
Term->isTerminator()) {
6368 <<
"SLP: Need to swizzle PHINodes (terminator use).\n");
6369 return TreeEntry::NeedToGather;
6373 return TreeEntry::Vectorize;
6375 case Instruction::ExtractValue:
6376 case Instruction::ExtractElement: {
6377 bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
6380 return TreeEntry::NeedToGather;
6381 if (Reuse || !CurrentOrder.empty())
6382 return TreeEntry::Vectorize;
6384 return TreeEntry::NeedToGather;
6386 case Instruction::InsertElement: {
6390 for (
Value *V : VL) {
6391 SourceVectors.
insert(cast<Instruction>(V)->getOperand(0));
6393 "Non-constant or undef index?");
6397 return !SourceVectors.contains(V);
6400 LLVM_DEBUG(
dbgs() <<
"SLP: Gather of insertelement vectors with "
6401 "different source vectors.\n");
6402 return TreeEntry::NeedToGather;
6405 return TreeEntry::Vectorize;
6407 case Instruction::Load: {
6416 return TreeEntry::Vectorize;
6418 return TreeEntry::ScatterVectorize;
6420 return TreeEntry::StridedVectorize;
6423 Type *ScalarTy = VL0->getType();
6424 if (
DL->getTypeSizeInBits(ScalarTy) !=
6425 DL->getTypeAllocSizeInBits(ScalarTy))
6426 LLVM_DEBUG(
dbgs() <<
"SLP: Gathering loads of non-packed type.\n");
6428 [](
Value *V) {
return !cast<LoadInst>(V)->isSimple(); }))
6433 return TreeEntry::NeedToGather;
6437 case Instruction::ZExt:
6438 case Instruction::SExt:
6439 case Instruction::FPToUI:
6440 case Instruction::FPToSI:
6441 case Instruction::FPExt:
6442 case Instruction::PtrToInt:
6443 case Instruction::IntToPtr:
6444 case Instruction::SIToFP:
6445 case Instruction::UIToFP:
6446 case Instruction::Trunc:
6447 case Instruction::FPTrunc:
6448 case Instruction::BitCast: {
6449 Type *SrcTy = VL0->getOperand(0)->getType();
6450 for (
Value *V : VL) {
6451 Type *Ty = cast<Instruction>(V)->getOperand(0)->getType();
6454 dbgs() <<
"SLP: Gathering casts with different src types.\n");
6455 return TreeEntry::NeedToGather;
6458 return TreeEntry::Vectorize;
6460 case Instruction::ICmp:
6461 case Instruction::FCmp: {
6465 Type *ComparedTy = VL0->getOperand(0)->getType();
6466 for (
Value *V : VL) {
6468 if ((
Cmp->getPredicate() != P0 &&
Cmp->getPredicate() != SwapP0) ||
6469 Cmp->getOperand(0)->getType() != ComparedTy) {
6470 LLVM_DEBUG(
dbgs() <<
"SLP: Gathering cmp with different predicate.\n");
6471 return TreeEntry::NeedToGather;
6474 return TreeEntry::Vectorize;
6476 case Instruction::Select:
6477 case Instruction::FNeg:
6478 case Instruction::Add:
6479 case Instruction::FAdd:
6480 case Instruction::Sub:
6481 case Instruction::FSub:
6482 case Instruction::Mul:
6483 case Instruction::FMul:
6484 case Instruction::UDiv:
6485 case Instruction::SDiv:
6486 case Instruction::FDiv:
6487 case Instruction::URem:
6488 case Instruction::SRem:
6489 case Instruction::FRem:
6490 case Instruction::Shl:
6491 case Instruction::LShr:
6492 case Instruction::AShr:
6493 case Instruction::And:
6494 case Instruction::Or:
6495 case Instruction::Xor:
6496 case Instruction::Freeze:
6497 return TreeEntry::Vectorize;
6498 case Instruction::GetElementPtr: {
6500 for (
Value *V : VL) {
6501 auto *
I = dyn_cast<GetElementPtrInst>(V);
6504 if (
I->getNumOperands() != 2) {
6505 LLVM_DEBUG(
dbgs() <<
"SLP: not-vectorizable GEP (nested indexes).\n");
6506 return TreeEntry::NeedToGather;
6512 Type *Ty0 = cast<GEPOperator>(VL0)->getSourceElementType();
6513 for (
Value *V : VL) {
6514 auto *
GEP = dyn_cast<GEPOperator>(V);
6517 Type *CurTy =
GEP->getSourceElementType();
6519 LLVM_DEBUG(
dbgs() <<
"SLP: not-vectorizable GEP (different types).\n");
6520 return TreeEntry::NeedToGather;
6525 Type *Ty1 = VL0->getOperand(1)->getType();
6526 for (
Value *V : VL) {
6527 auto *
I = dyn_cast<GetElementPtrInst>(V);
6530 auto *
Op =
I->getOperand(1);
6531 if ((!IsScatterVectorizeUserTE && !isa<ConstantInt>(
Op)) ||
6532 (
Op->getType() != Ty1 &&
6533 ((IsScatterVectorizeUserTE && !isa<ConstantInt>(
Op)) ||
6534 Op->getType()->getScalarSizeInBits() >
6535 DL->getIndexSizeInBits(
6536 V->getType()->getPointerAddressSpace())))) {
6538 dbgs() <<
"SLP: not-vectorizable GEP (non-constant indexes).\n");
6539 return TreeEntry::NeedToGather;
6543 return TreeEntry::Vectorize;
6545 case Instruction::Store: {
6547 llvm::Type *ScalarTy = cast<StoreInst>(VL0)->getValueOperand()->getType();
6550 if (
DL->getTypeSizeInBits(ScalarTy) !=
6551 DL->getTypeAllocSizeInBits(ScalarTy)) {
6552 LLVM_DEBUG(
dbgs() <<
"SLP: Gathering stores of non-packed type.\n");
6553 return TreeEntry::NeedToGather;
6557 for (
Value *V : VL) {
6558 auto *
SI = cast<StoreInst>(V);
6559 if (!
SI->isSimple()) {
6561 return TreeEntry::NeedToGather;
6570 if (CurrentOrder.empty()) {
6571 Ptr0 = PointerOps.
front();
6572 PtrN = PointerOps.
back();
6574 Ptr0 = PointerOps[CurrentOrder.front()];
6575 PtrN = PointerOps[CurrentOrder.back()];
6577 std::optional<int> Dist =
6580 if (
static_cast<unsigned>(*Dist) == VL.size() - 1)
6581 return TreeEntry::Vectorize;
6585 return TreeEntry::NeedToGather;
6587 case Instruction::Call: {
6590 CallInst *CI = cast<CallInst>(VL0);
6601 return TreeEntry::NeedToGather;
6606 for (
unsigned J = 0; J != NumArgs; ++J)
6609 for (
Value *V : VL) {
6610 CallInst *CI2 = dyn_cast<CallInst>(V);
6616 LLVM_DEBUG(
dbgs() <<
"SLP: mismatched calls:" << *CI <<
"!=" << *V
6618 return TreeEntry::NeedToGather;
6622 for (
unsigned J = 0; J != NumArgs; ++J) {
6625 if (ScalarArgs[J] != A1J) {
6627 <<
"SLP: mismatched arguments in call:" << *CI
6628 <<
" argument " << ScalarArgs[J] <<
"!=" << A1J <<
"\n");
6629 return TreeEntry::NeedToGather;
6638 LLVM_DEBUG(
dbgs() <<
"SLP: mismatched bundle operands in calls:" << *CI
6639 <<
"!=" << *V <<
'\n');
6640 return TreeEntry::NeedToGather;
6644 return TreeEntry::Vectorize;
6646 case Instruction::ShuffleVector: {
6649 if (!S.isAltShuffle()) {
6650 LLVM_DEBUG(
dbgs() <<
"SLP: ShuffleVector are not vectorized.\n");
6651 return TreeEntry::NeedToGather;
6656 <<
"SLP: ShuffleVector not vectorized, operands are buildvector and "
6657 "the whole alt sequence is not profitable.\n");
6658 return TreeEntry::NeedToGather;
6661 return TreeEntry::Vectorize;
6665 return TreeEntry::NeedToGather;
6679 PHIHandler() =
delete;
6681 : DT(DT), Main(Main), Phis(Phis),
6682 Operands(Main->getNumIncomingValues(),
6684 void buildOperands() {
6685 constexpr unsigned FastLimit = 4;
6695 auto *
P = cast<PHINode>(V);
6696 if (
P->getIncomingBlock(
I) == InBB)
6711 Blocks.try_emplace(InBB).first->second.push_back(
I);
6714 auto *
P = cast<PHINode>(V);
6715 for (
unsigned I : seq<unsigned>(0,
P->getNumIncomingValues())) {
6723 auto It =
Blocks.find(InBB);
6729 for (
const auto &
P :
Blocks) {
6730 if (
P.getSecond().size() <= 1)
6732 unsigned BasicI =
P.getSecond().front();
6735 [&](
const auto &Data) {
6736 return !Data.value() ||
6737 Data.value() ==
Operands[BasicI][Data.index()];
6739 "Expected empty operands list.");
6749 const EdgeInfo &UserTreeIdx) {
6755 auto TryToFindDuplicates = [&](
const InstructionsState &S,
6756 bool DoNotFail =
false) {
6759 for (
Value *V : VL) {
6766 auto Res = UniquePositions.try_emplace(V, UniqueValues.
size());
6771 size_t NumUniqueScalarValues = UniqueValues.
size();
6772 if (NumUniqueScalarValues == VL.size()) {
6773 ReuseShuffleIndices.
clear();
6776 if (UserTreeIdx.UserTE && UserTreeIdx.UserTE->isNonPowOf2Vec()) {
6777 LLVM_DEBUG(
dbgs() <<
"SLP: Reshuffling scalars not yet supported "
6778 "for nodes with padding.\n");
6779 newTreeEntry(VL, std::nullopt , S, UserTreeIdx);
6783 if (NumUniqueScalarValues <= 1 ||
6784 (UniquePositions.size() == 1 &&
all_of(UniqueValues,
6786 return isa<UndefValue>(V) ||
6789 !llvm::has_single_bit<uint32_t>(NumUniqueScalarValues)) {
6790 if (DoNotFail && UniquePositions.size() > 1 &&
6791 NumUniqueScalarValues > 1 && S.MainOp->isSafeToRemove() &&
6793 return isa<ExtractElementInst>(V) ||
6794 areAllUsersVectorized(cast<Instruction>(V),
6798 if (PWSz == VL.size()) {
6799 ReuseShuffleIndices.
clear();
6801 NonUniqueValueVL.
assign(UniqueValues.
begin(), UniqueValues.
end());
6802 NonUniqueValueVL.
append(PWSz - UniqueValues.
size(),
6803 UniqueValues.
back());
6804 VL = NonUniqueValueVL;
6809 newTreeEntry(VL, std::nullopt , S, UserTreeIdx);
6820 if (!EphValues.
empty()) {
6821 for (
Value *V : VL) {
6822 if (EphValues.
count(V)) {
6824 <<
") is ephemeral.\n");
6825 newTreeEntry(VL, std::nullopt , S, UserTreeIdx);
6835 !(S.MainOp && isa<Instruction>(S.MainOp) && S.MainOp == S.AltOp &&
6840 cast<Instruction>(
I)->getOpcode() ==
6841 cast<Instruction>(S.MainOp)->getOpcode();
6843 LLVM_DEBUG(
dbgs() <<
"SLP: Gathering due to max recursion depth.\n");
6844 if (TryToFindDuplicates(S))
6845 newTreeEntry(VL, std::nullopt , S, UserTreeIdx,
6846 ReuseShuffleIndices);
6851 if (S.getOpcode() == Instruction::ExtractElement &&
6852 isa<ScalableVectorType>(
6853 cast<ExtractElementInst>(S.OpValue)->getVectorOperandType())) {
6854 LLVM_DEBUG(
dbgs() <<
"SLP: Gathering due to scalable vector type.\n");
6855 if (TryToFindDuplicates(S))
6856 newTreeEntry(VL, std::nullopt , S, UserTreeIdx,
6857 ReuseShuffleIndices);
6862 if (!
SLPReVec && S.OpValue->getType()->isVectorTy() &&
6863 !isa<InsertElementInst>(S.OpValue)) {
6865 newTreeEntry(VL, std::nullopt , S, UserTreeIdx);
6869 if (
StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
6870 if (!
SLPReVec &&
SI->getValueOperand()->getType()->isVectorTy()) {
6871 LLVM_DEBUG(
dbgs() <<
"SLP: Gathering due to store vector type.\n");
6872 newTreeEntry(VL, std::nullopt , S, UserTreeIdx);
6881 auto &&NotProfitableForVectorization = [&S,
this,
6883 if (!S.getOpcode() || !S.isAltShuffle() || VL.size() > 2)
6892 for (
Value *V : VL) {
6893 auto *
I = cast<Instruction>(V);
6895 return isa<Instruction>(Op) || isVectorLikeInstWithConstOps(Op);
6899 if ((IsCommutative &&
6900 std::accumulate(InstsCount.begin(), InstsCount.end(), 0) < 2) ||
6902 all_of(InstsCount, [](
unsigned ICnt) {
return ICnt < 2; })))
6904 assert(VL.size() == 2 &&
"Expected only 2 alternate op instructions.");
6906 auto *
I1 = cast<Instruction>(VL.front());
6907 auto *I2 = cast<Instruction>(VL.back());
6910 I2->getOperand(
Op));
6911 if (
static_cast<unsigned>(
count_if(
6912 Candidates, [
this](
ArrayRef<std::pair<Value *, Value *>> Cand) {
6914 })) >= S.MainOp->getNumOperands() / 2)
6916 if (S.MainOp->getNumOperands() > 2)
6918 if (IsCommutative) {
6923 I2->getOperand((
Op + 1) % E));
6925 Candidates, [
this](
ArrayRef<std::pair<Value *, Value *>> Cand) {
6934 bool IsScatterVectorizeUserTE =
6935 UserTreeIdx.UserTE &&
6936 UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize;
6937 bool AreAllSameBlock = S.getOpcode() &&
allSameBlock(VL);
6938 bool AreScatterAllGEPSameBlock =
6939 (IsScatterVectorizeUserTE && S.OpValue->getType()->isPointerTy() &&
6943 auto *
I = dyn_cast<GetElementPtrInst>(V);
6947 BB =
I->getParent();
6948 return BB ==
I->getParent() &&
I->getNumOperands() == 2;
6951 sortPtrAccesses(VL, UserTreeIdx.UserTE->getMainOp()->getType(), *DL, *SE,
6953 bool AreAllSameInsts = AreAllSameBlock || AreScatterAllGEPSameBlock;
6955 (isa<InsertElementInst, ExtractValueInst, ExtractElementInst>(
6958 NotProfitableForVectorization(VL)) {
6959 LLVM_DEBUG(
dbgs() <<
"SLP: Gathering due to C,S,B,O, small shuffle. \n");
6960 if (TryToFindDuplicates(S))
6961 newTreeEntry(VL, std::nullopt , S, UserTreeIdx,
6962 ReuseShuffleIndices);
6970 if (TreeEntry *E = getTreeEntry(S.OpValue)) {
6971 LLVM_DEBUG(
dbgs() <<
"SLP: \tChecking bundle: " << *S.OpValue <<
".\n");
6972 if (!E->isSame(VL)) {
6973 auto It = MultiNodeScalars.
find(S.OpValue);
6974 if (It != MultiNodeScalars.
end()) {
6975 auto *TEIt =
find_if(It->getSecond(),
6976 [&](TreeEntry *ME) { return ME->isSame(VL); });
6977 if (TEIt != It->getSecond().end())
6987 LLVM_DEBUG(
dbgs() <<
"SLP: Gathering due to partial overlap.\n");
6988 if (TryToFindDuplicates(S))
6989 newTreeEntry(VL, std::nullopt , S, UserTreeIdx,
6990 ReuseShuffleIndices);
6996 E->UserTreeIndices.push_back(UserTreeIdx);
6997 LLVM_DEBUG(
dbgs() <<
"SLP: Perfect diamond merge at " << *S.OpValue
7004 for (
Value *V : VL) {
7005 if ((!IsScatterVectorizeUserTE && !isa<Instruction>(V)) ||
7008 if (getTreeEntry(V)) {
7010 <<
") is already in tree.\n");
7011 if (TryToFindDuplicates(S))
7012 newTreeEntry(VL, std::nullopt , S, UserTreeIdx,
7013 ReuseShuffleIndices);
7019 if (UserIgnoreList && !UserIgnoreList->empty()) {
7020 for (
Value *V : VL) {
7021 if (UserIgnoreList && UserIgnoreList->contains(V)) {
7022 LLVM_DEBUG(
dbgs() <<
"SLP: Gathering due to gathered scalar.\n");
7023 if (TryToFindDuplicates(S))
7024 newTreeEntry(VL, std::nullopt , S, UserTreeIdx,
7025 ReuseShuffleIndices);
7033 if (!AreAllSameBlock && AreScatterAllGEPSameBlock) {
7034 assert(S.OpValue->getType()->isPointerTy() &&
7035 count_if(VL, IsaPred<GetElementPtrInst>) >= 2 &&
7036 "Expected pointers only.");
7038 const auto *It =
find_if(VL, IsaPred<GetElementPtrInst>);
7039 assert(It != VL.end() &&
"Expected at least one GEP.");
7045 auto *VL0 = cast<Instruction>(S.OpValue);
7052 newTreeEntry(VL, std::nullopt , S, UserTreeIdx);
7061 newTreeEntry(VL, std::nullopt , S, UserTreeIdx);
7066 if (!TryToFindDuplicates(S,
true))
7072 TreeEntry::EntryState State = getScalarsVectorizationState(
7073 S, VL, IsScatterVectorizeUserTE, CurrentOrder, PointerOps);
7074 if (State == TreeEntry::NeedToGather) {
7075 newTreeEntry(VL, std::nullopt , S, UserTreeIdx,
7076 ReuseShuffleIndices);
7080 auto &BSRef = BlocksSchedules[BB];
7082 BSRef = std::make_unique<BlockScheduling>(BB);
7084 BlockScheduling &BS = *BSRef;
7086 std::optional<ScheduleData *> Bundle =
7087 BS.tryScheduleBundle(UniqueValues,
this, S);
7088#ifdef EXPENSIVE_CHECKS
7093 LLVM_DEBUG(
dbgs() <<
"SLP: We are not able to schedule this bundle!\n");
7094 assert((!BS.getScheduleData(VL0) ||
7095 !BS.getScheduleData(VL0)->isPartOfBundle()) &&
7096 "tryScheduleBundle should cancelScheduling on failure");
7097 newTreeEntry(VL, std::nullopt , S, UserTreeIdx,
7098 ReuseShuffleIndices);
7099 NonScheduledFirst.insert(VL.front());
7102 LLVM_DEBUG(
dbgs() <<
"SLP: We are able to schedule this bundle.\n");
7104 unsigned ShuffleOrOp = S.isAltShuffle() ?
7105 (
unsigned) Instruction::ShuffleVector : S.getOpcode();
7106 switch (ShuffleOrOp) {
7107 case Instruction::PHI: {
7108 auto *PH = cast<PHINode>(VL0);
7111 newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndices);
7115 PHIHandler Handler(*DT, PH, VL);
7116 Handler.buildOperands();
7117 for (
unsigned I : seq<unsigned>(0, PH->getNumOperands()))
7118 TE->setOperand(
I, Handler.getOperands(
I));
7119 for (
unsigned I : seq<unsigned>(0, PH->getNumOperands()))
7120 buildTree_rec(Handler.getOperands(
I),
Depth + 1, {TE, I});
7123 case Instruction::ExtractValue:
7124 case Instruction::ExtractElement: {
7125 if (CurrentOrder.empty()) {
7126 LLVM_DEBUG(
dbgs() <<
"SLP: Reusing or shuffling extract sequence.\n");
7129 dbgs() <<
"SLP: Reusing or shuffling of reordered extract sequence "
7131 for (
unsigned Idx : CurrentOrder)
7139 newTreeEntry(VL, Bundle , S, UserTreeIdx,
7140 ReuseShuffleIndices, CurrentOrder);
7144 Op0.
assign(VL.size(), VL0->getOperand(0));
7145 VectorizableTree.back()->setOperand(0, Op0);
7148 case Instruction::InsertElement: {
7149 assert(ReuseShuffleIndices.
empty() &&
"All inserts should be unique");
7151 auto OrdCompare = [](
const std::pair<int, int> &P1,
7152 const std::pair<int, int> &P2) {
7153 return P1.first > P2.first;
7156 decltype(OrdCompare)>
7157 Indices(OrdCompare);
7158 for (
int I = 0, E = VL.size();
I < E; ++
I) {
7160 Indices.emplace(
Idx,
I);
7162 OrdersType CurrentOrder(VL.size(), VL.size());
7163 bool IsIdentity =
true;
7164 for (
int I = 0, E = VL.size();
I < E; ++
I) {
7165 CurrentOrder[Indices.top().second] =
I;
7166 IsIdentity &= Indices.top().second ==
I;
7170 CurrentOrder.clear();
7171 TreeEntry *
TE = newTreeEntry(VL, Bundle , S, UserTreeIdx,
7172 std::nullopt, CurrentOrder);
7175 TE->setOperandsInOrder();
7176 buildTree_rec(
TE->getOperand(1),
Depth + 1, {TE, 1});
7179 case Instruction::Load: {
7186 TreeEntry *
TE =
nullptr;
7189 case TreeEntry::Vectorize:
7190 TE = newTreeEntry(VL, Bundle , S, UserTreeIdx,
7191 ReuseShuffleIndices, CurrentOrder);
7192 if (CurrentOrder.empty())
7196 TE->setOperandsInOrder();
7198 case TreeEntry::StridedVectorize:
7200 TE = newTreeEntry(VL, TreeEntry::StridedVectorize, Bundle, S,
7201 UserTreeIdx, ReuseShuffleIndices, CurrentOrder);
7202 TE->setOperandsInOrder();
7205 case TreeEntry::ScatterVectorize:
7207 TE = newTreeEntry(VL, TreeEntry::ScatterVectorize, Bundle, S,
7208 UserTreeIdx, ReuseShuffleIndices);
7209 TE->setOperandsInOrder();
7210 buildTree_rec(PointerOps,
Depth + 1, {
TE, 0});
7211 LLVM_DEBUG(
dbgs() <<
"SLP: added a vector of non-consecutive loads.\n");
7213 case TreeEntry::NeedToGather:
7218 case Instruction::ZExt:
7219 case Instruction::SExt:
7220 case Instruction::FPToUI:
7221 case Instruction::FPToSI:
7222 case Instruction::FPExt:
7223 case Instruction::PtrToInt:
7224 case Instruction::IntToPtr:
7225 case Instruction::SIToFP:
7226 case Instruction::UIToFP:
7227 case Instruction::Trunc:
7228 case Instruction::FPTrunc:
7229 case Instruction::BitCast: {
7230 auto [PrevMaxBW, PrevMinBW] = CastMaxMinBWSizes.value_or(
7231 std::make_pair(std::numeric_limits<unsigned>::min(),
7232 std::numeric_limits<unsigned>::max()));
7233 if (ShuffleOrOp == Instruction::ZExt ||
7234 ShuffleOrOp == Instruction::SExt) {
7235 CastMaxMinBWSizes = std::make_pair(
7241 }
else if (ShuffleOrOp == Instruction::Trunc) {
7242 CastMaxMinBWSizes = std::make_pair(
7248 ExtraBitWidthNodes.
insert(VectorizableTree.size() + 1);
7249 }
else if (ShuffleOrOp == Instruction::SIToFP ||
7250 ShuffleOrOp == Instruction::UIToFP) {
7251 unsigned NumSignBits =
7253 if (
auto *OpI = dyn_cast<Instruction>(VL0->getOperand(0))) {
7255 NumSignBits = std::max(NumSignBits,
Mask.countl_zero());
7257 if (NumSignBits * 2 >=
7259 ExtraBitWidthNodes.
insert(VectorizableTree.size() + 1);
7261 TreeEntry *
TE = newTreeEntry(VL, Bundle , S, UserTreeIdx,
7262 ReuseShuffleIndices);
7265 TE->setOperandsInOrder();
7266 for (
unsigned I : seq<unsigned>(0, VL0->getNumOperands()))
7267 buildTree_rec(
TE->getOperand(
I),
Depth + 1, {TE, I});
7270 case Instruction::ICmp:
7271 case Instruction::FCmp: {
7274 TreeEntry *
TE = newTreeEntry(VL, Bundle , S, UserTreeIdx,
7275 ReuseShuffleIndices);
7283 "Commutative Predicate mismatch");
7284 reorderInputsAccordingToOpcode(VL,
Left,
Right, *
this);
7287 for (
Value *V : VL) {
7288 auto *
Cmp = cast<CmpInst>(V);
7291 if (
Cmp->getPredicate() != P0)
7293 Left.push_back(LHS);
7294 Right.push_back(RHS);
7301 if (ShuffleOrOp == Instruction::ICmp) {
7302 unsigned NumSignBits0 =
7304 if (NumSignBits0 * 2 >=
7306 ExtraBitWidthNodes.
insert(getOperandEntry(TE, 0)->
Idx);
7307 unsigned NumSignBits1 =
7309 if (NumSignBits1 * 2 >=
7311 ExtraBitWidthNodes.
insert(getOperandEntry(TE, 1)->
Idx);
7315 case Instruction::Select:
7316 case Instruction::FNeg:
7317 case Instruction::Add:
7318 case Instruction::FAdd:
7319 case Instruction::Sub:
7320 case Instruction::FSub:
7321 case Instruction::Mul:
7322 case Instruction::FMul:
7323 case Instruction::UDiv:
7324 case Instruction::SDiv:
7325 case Instruction::FDiv:
7326 case Instruction::URem:
7327 case Instruction::SRem:
7328 case Instruction::FRem:
7329 case Instruction::Shl:
7330 case Instruction::LShr:
7331 case Instruction::AShr:
7332 case Instruction::And:
7333 case Instruction::Or:
7334 case Instruction::Xor:
7335 case Instruction::Freeze: {
7336 TreeEntry *
TE = newTreeEntry(VL, Bundle , S, UserTreeIdx,
7337 ReuseShuffleIndices);
7344 reorderInputsAccordingToOpcode(VL,
Left,
Right, *
this);
7352 TE->setOperandsInOrder();
7353 for (
unsigned I : seq<unsigned>(0, VL0->getNumOperands()))
7354 buildTree_rec(
TE->getOperand(
I),
Depth + 1, {TE, I});
7357 case Instruction::GetElementPtr: {
7358 TreeEntry *
TE = newTreeEntry(VL, Bundle , S, UserTreeIdx,
7359 ReuseShuffleIndices);
7363 for (
Value *V : VL) {
7364 auto *
GEP = dyn_cast<GetElementPtrInst>(V);
7369 Operands.front().push_back(
GEP->getPointerOperand());
7378 Type *VL0Ty = VL0->getOperand(IndexIdx)->getType();
7380 [VL0Ty, IndexIdx](
Value *V) {
7381 auto *
GEP = dyn_cast<GetElementPtrInst>(V);
7384 return VL0Ty ==
GEP->getOperand(IndexIdx)->getType();
7388 ->getPointerOperandType()
7391 for (
Value *V : VL) {
7392 auto *
I = dyn_cast<GetElementPtrInst>(V);
7395 ConstantInt::get(Ty, 0,
false));
7398 auto *
Op =
I->getOperand(IndexIdx);
7399 auto *CI = dyn_cast<ConstantInt>(
Op);
7404 CI, Ty, CI->getValue().isSignBitSet(), *DL));
7408 for (
unsigned I = 0, Ops =
Operands.size();
I < Ops; ++
I)
7412 case Instruction::Store: {
7413 bool Consecutive = CurrentOrder.empty();
7416 TreeEntry *
TE = newTreeEntry(VL, Bundle , S, UserTreeIdx,
7417 ReuseShuffleIndices, CurrentOrder);
7418 TE->setOperandsInOrder();
7419 buildTree_rec(
TE->getOperand(0),
Depth + 1, {TE, 0});
7423 LLVM_DEBUG(
dbgs() <<
"SLP: added a vector of jumbled stores.\n");
7426 case Instruction::Call: {
7429 CallInst *CI = cast<CallInst>(VL0);
7432 TreeEntry *
TE = newTreeEntry(VL, Bundle , S, UserTreeIdx,
7433 ReuseShuffleIndices);
7438 reorderInputsAccordingToOpcode(VL,
Left,
Right, *
this);
7442 for (
unsigned I : seq<unsigned>(2, CI->
arg_size())) {
7446 for (
Value *V : VL) {
7447 auto *CI2 = cast<CallInst>(V);
7454 for (
unsigned I : seq<unsigned>(2, CI->
arg_size())) {
7461 TE->setOperandsInOrder();
7462 for (
unsigned I : seq<unsigned>(0, CI->
arg_size())) {
7469 for (
Value *V : VL) {
7470 auto *CI2 = cast<CallInst>(V);
7477 case Instruction::ShuffleVector: {
7478 TreeEntry *
TE = newTreeEntry(VL, Bundle , S, UserTreeIdx,
7479 ReuseShuffleIndices);
7483 auto *CI = dyn_cast<CmpInst>(VL0);
7484 if (isa<BinaryOperator>(VL0) || CI) {
7487 return cast<CmpInst>(V)->isCommutative();
7489 reorderInputsAccordingToOpcode(VL,
Left,
Right, *
this);
7491 auto *MainCI = cast<CmpInst>(S.MainOp);
7492 auto *AltCI = cast<CmpInst>(S.AltOp);
7496 "Expected different main/alternate predicates.");
7499 for (
Value *V : VL) {
7500 auto *
Cmp = cast<CmpInst>(V);
7511 Left.push_back(LHS);
7512 Right.push_back(RHS);
7522 TE->setOperandsInOrder();
7523 for (
unsigned I : seq<unsigned>(0, VL0->getNumOperands()))
7524 buildTree_rec(
TE->getOperand(
I),
Depth + 1, {TE, I});
7537 while (isa<StructType, ArrayType, FixedVectorType>(EltTy)) {
7538 if (
auto *ST = dyn_cast<StructType>(EltTy)) {
7540 for (
const auto *Ty : ST->elements())
7541 if (Ty != *ST->element_begin())
7543 N *= ST->getNumElements();
7544 EltTy = *ST->element_begin();
7545 }
else if (
auto *AT = dyn_cast<ArrayType>(EltTy)) {
7546 N *= AT->getNumElements();
7547 EltTy = AT->getElementType();
7549 auto *VT = cast<FixedVectorType>(EltTy);
7550 N *= VT->getNumElements();
7551 EltTy = VT->getElementType();
7558 if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize ||
7566 bool ResizeAllowed)
const {
7567 const auto *It =
find_if(VL, IsaPred<ExtractElementInst, ExtractValueInst>);
7568 assert(It != VL.
end() &&
"Expected at least one extract instruction.");
7569 auto *E0 = cast<Instruction>(*It);
7571 all_of(VL, IsaPred<UndefValue, ExtractElementInst, ExtractValueInst>) &&
7575 Value *Vec = E0->getOperand(0);
7577 CurrentOrder.
clear();
7581 if (E0->getOpcode() == Instruction::ExtractValue) {
7586 LoadInst *LI = dyn_cast<LoadInst>(Vec);
7590 NElts = cast<FixedVectorType>(Vec->
getType())->getNumElements();
7593 unsigned E = VL.
size();
7594 if (!ResizeAllowed && NElts != E)
7597 unsigned MinIdx = NElts, MaxIdx = 0;
7599 auto *Inst = dyn_cast<Instruction>(V);
7602 if (Inst->getOperand(0) != Vec)
7604 if (
auto *EE = dyn_cast<ExtractElementInst>(Inst))
7605 if (isa<UndefValue>(EE->getIndexOperand()))
7610 const unsigned ExtIdx = *
Idx;
7611 if (ExtIdx >= NElts)
7613 Indices[
I] = ExtIdx;
7614 if (MinIdx > ExtIdx)
7616 if (MaxIdx < ExtIdx)
7619 if (MaxIdx - MinIdx + 1 > E)
7621 if (MaxIdx + 1 <= E)
7625 bool ShouldKeepOrder =
true;
7631 CurrentOrder.
assign(E, E);
7632 for (
unsigned I = 0;
I < E; ++
I) {
7635 const unsigned ExtIdx = Indices[
I] - MinIdx;
7636 if (CurrentOrder[ExtIdx] != E) {
7637 CurrentOrder.
clear();
7640 ShouldKeepOrder &= ExtIdx ==
I;
7641 CurrentOrder[ExtIdx] =
I;
7643 if (ShouldKeepOrder)
7644 CurrentOrder.
clear();
7646 return ShouldKeepOrder;
7649bool BoUpSLP::areAllUsersVectorized(
7651 return (
I->hasOneUse() && (!VectorizedVals || VectorizedVals->
contains(
I))) ||
7653 return ScalarToTreeEntry.contains(U) ||
7654 isVectorLikeInstWithConstOps(U) ||
7655 (isa<ExtractElementInst>(U) && MustGather.contains(U));
7659static std::pair<InstructionCost, InstructionCost>
7667 if (
auto *FPCI = dyn_cast<FPMathOperator>(CI))
7668 FMF = FPCI->getFastMathFlags();
7671 dyn_cast<IntrinsicInst>(CI));
7672 auto IntrinsicCost =
7679 auto LibCost = IntrinsicCost;
7686 return {IntrinsicCost, LibCost};
7689void BoUpSLP::TreeEntry::buildAltOpShuffleMask(
7693 unsigned Sz = Scalars.size();
7696 if (!ReorderIndices.empty())
7698 for (
unsigned I = 0;
I < Sz; ++
I) {
7700 if (!ReorderIndices.empty())
7702 auto *OpInst = cast<Instruction>(Scalars[
Idx]);
7703 if (IsAltOp(OpInst)) {
7713 if (!ReuseShuffleIndices.
empty()) {
7716 return Idx != PoisonMaskElem ? Mask[Idx] : PoisonMaskElem;
7726 if (
auto *MainCI = dyn_cast<CmpInst>(MainOp)) {
7727 auto *AltCI = cast<CmpInst>(AltOp);
7730 assert(MainP != AltP &&
"Expected different main/alternate predicates.");
7731 auto *CI = cast<CmpInst>(
I);
7739 assert((MainP ==
P || AltP ==
P || MainP == SwappedP || AltP == SwappedP) &&
7740 "CmpInst expected to match either main or alternate predicate or "
7743 return MainP !=
P && MainP != SwappedP;
7750 const auto *Op0 = Ops.
front();
7756 const bool IsUniform =
all_of(Ops, [=](
Value *V) {
7760 const bool IsPowerOfTwo =
all_of(Ops, [](
Value *V) {
7762 if (
auto *CI = dyn_cast<ConstantInt>(V))
7763 return CI->getValue().isPowerOf2();
7766 const bool IsNegatedPowerOfTwo =
all_of(Ops, [](
Value *V) {
7768 if (
auto *CI = dyn_cast<ConstantInt>(V))
7769 return CI->getValue().isNegatedPowerOf2();
7774 if (IsConstant && IsUniform)
7776 else if (IsConstant)
7790class BaseShuffleAnalysis {
7792 Type *ScalarTy =
nullptr;
7794 BaseShuffleAnalysis(
Type *ScalarTy) : ScalarTy(ScalarTy) {}
7802 unsigned getVF(
Value *V)
const {
7803 assert(V &&
"V cannot be nullptr");
7804 assert(isa<FixedVectorType>(
V->getType()) &&
7805 "V does not have FixedVectorType");
7806 assert(ScalarTy &&
"ScalarTy cannot be nullptr");
7808 unsigned VNumElements =
7809 cast<FixedVectorType>(
V->getType())->getNumElements();
7810 assert(VNumElements > ScalarTyNumElements &&
7811 "the number of elements of V is not large enough");
7812 assert(VNumElements % ScalarTyNumElements == 0 &&
7813 "the number of elements of V is not a vectorized value");
7814 return VNumElements / ScalarTyNumElements;
7822 int Limit =
Mask.size();
7834 if (Limit % VF == 0 &&
all_of(seq<int>(0, Limit / VF), [=](
int Idx) {
7850 unsigned VF =
Mask.size();
7852 for (
int I = 0, Sz = ExtMask.
size();
I < Sz; ++
I) {
7855 int MaskedIdx =
Mask[ExtMask[
I] % VF];
7896 bool SinglePermute) {
7900 while (
auto *SV = dyn_cast<ShuffleVectorInst>(
Op)) {
7902 auto *SVTy = dyn_cast<FixedVectorType>(SV->getType());
7908 if (isIdentityMask(Mask, SVTy,
false)) {
7909 if (!IdentityOp || !SinglePermute ||
7910 (isIdentityMask(Mask, SVTy,
true) &&
7912 IdentityMask.
size()))) {
7917 IdentityMask.
assign(Mask);
7937 if (SV->isZeroEltSplat()) {
7939 IdentityMask.
assign(Mask);
7941 int LocalVF =
Mask.size();
7943 dyn_cast<FixedVectorType>(SV->getOperand(0)->getType()))
7944 LocalVF = SVOpTy->getNumElements();
7948 static_cast<unsigned>(
I) >= SV->getShuffleMask().size())
7950 ExtMask[
Idx] = SV->getMaskValue(
I);
7960 if (!IsOp1Undef && !IsOp2Undef) {
7962 for (
int &
I : Mask) {
7965 if (SV->getMaskValue(
I % SV->getShuffleMask().size()) ==
7972 combineMasks(LocalVF, ShuffleMask, Mask);
7973 Mask.swap(ShuffleMask);
7975 Op = SV->getOperand(0);
7977 Op = SV->getOperand(1);
7979 if (
auto *OpTy = dyn_cast<FixedVectorType>(
Op->getType());
7980 !OpTy || !isIdentityMask(Mask, OpTy, SinglePermute) ||
7985 "Expected masks of same sizes.");
7990 Mask.swap(IdentityMask);
7991 auto *Shuffle = dyn_cast<ShuffleVectorInst>(V);
7992 return SinglePermute &&
7993 (isIdentityMask(Mask, cast<FixedVectorType>(
V->getType()),
7995 (Shuffle &&
Mask.size() == Shuffle->getShuffleMask().size() &&
7996 Shuffle->isZeroEltSplat() &&
8009 template <
typename T,
typename ShuffleBuilderTy>
8011 ShuffleBuilderTy &Builder) {
8012 assert(V1 &&
"Expected at least one vector value.");
8014 Builder.resizeToMatch(V1, V2);
8015 int VF =
Mask.size();
8016 if (
auto *FTy = dyn_cast<FixedVectorType>(V1->
getType()))
8017 VF = FTy->getNumElements();
8024 cast<VectorType>(V1->
getType())->getElementCount().getKnownMinValue();
8027 for (
int I = 0,
E =
Mask.size();
I <
E; ++
I) {
8029 CombinedMask1[
I] =
Mask[
I];
8031 CombinedMask2[
I] =
Mask[
I] - VF;
8038 (void)peekThroughShuffles(Op1, CombinedMask1,
false);
8039 (void)peekThroughShuffles(Op2, CombinedMask2,
false);
8042 if (
auto *SV1 = dyn_cast<ShuffleVectorInst>(Op1))
8043 if (
auto *SV2 = dyn_cast<ShuffleVectorInst>(Op2)) {
8048 ExtMask1[
Idx] = SV1->getMaskValue(
I);
8051 cast<FixedVectorType>(SV1->getOperand(1)->getType())
8053 ExtMask1, UseMask::SecondArg);
8058 ExtMask2[
Idx] = SV2->getMaskValue(
I);
8061 cast<FixedVectorType>(SV2->getOperand(1)->getType())
8063 ExtMask2, UseMask::SecondArg);
8064 if (SV1->getOperand(0)->getType() ==
8065 SV2->getOperand(0)->getType() &&
8066 SV1->getOperand(0)->getType() != SV1->getType() &&
8069 Op1 = SV1->getOperand(0);
8070 Op2 = SV2->getOperand(0);
8072 int LocalVF = ShuffleMask1.size();
8073 if (
auto *FTy = dyn_cast<FixedVectorType>(Op1->
getType()))
8074 LocalVF = FTy->getNumElements();
8075 combineMasks(LocalVF, ShuffleMask1, CombinedMask1);
8076 CombinedMask1.swap(ShuffleMask1);
8078 LocalVF = ShuffleMask2.size();
8079 if (
auto *FTy = dyn_cast<FixedVectorType>(Op2->
getType()))
8080 LocalVF = FTy->getNumElements();
8081 combineMasks(LocalVF, ShuffleMask2, CombinedMask2);
8082 CombinedMask2.swap(ShuffleMask2);
8085 }
while (PrevOp1 != Op1 || PrevOp2 != Op2);
8086 Builder.resizeToMatch(Op1, Op2);
8087 VF = std::max(cast<VectorType>(Op1->
getType())
8089 .getKnownMinValue(),
8090 cast<VectorType>(Op2->
getType())
8092 .getKnownMinValue());
8093 for (
int I = 0,
E =
Mask.size();
I <
E; ++
I) {
8096 "Expected undefined mask element");
8097 CombinedMask1[
I] = CombinedMask2[
I] + (Op1 == Op2 ? 0 : VF);
8103 isa<ShuffleVectorInst>(Op1) &&
8104 cast<ShuffleVectorInst>(Op1)->getShuffleMask() ==
8106 return Builder.createIdentity(Op1);
8107 return Builder.createShuffleVector(
8111 if (isa<PoisonValue>(V1))
8112 return Builder.createPoison(
8113 cast<VectorType>(V1->
getType())->getElementType(),
Mask.size());
8115 bool IsIdentity = peekThroughShuffles(V1, NewMask,
true);
8116 assert(V1 &&
"Expected non-null value after looking through shuffles.");
8119 return Builder.createShuffleVector(V1, NewMask);
8120 return Builder.createIdentity(V1);
8136 int NumSrcElts = Tp->getElementCount().getKnownMinValue();
8139 Mask, NumSrcElts, NumSubElts,
Index)) {
8140 if (
Index + NumSubElts > NumSrcElts &&
8141 Index + NumSrcElts <=
static_cast<int>(Mask.size()))
8151static std::pair<InstructionCost, InstructionCost>
8162 if (Opcode == Instruction::Load || Opcode == Instruction::Store) {
8172 Ptrs, BasePtr, TTI::PointersChainInfo::getUnitStride(), ScalarTy,
8176 for (
Value *V : Ptrs) {
8181 auto *
Ptr = dyn_cast<GetElementPtrInst>(V);
8186 if (!
Ptr || !
Ptr->hasOneUse())
8190 if (PtrsRetainedInVecCode.
size() == Ptrs.size()) {
8196 TTI::PointersChainInfo::getKnownStride(),
8206 [](
const Value *V) {
8207 auto *
Ptr = dyn_cast<GetElementPtrInst>(V);
8208 return Ptr && !
Ptr->hasAllConstantIndices();
8210 ? TTI::PointersChainInfo::getUnknownStride()
8211 : TTI::PointersChainInfo::getKnownStride();
8215 auto *BaseGEP = dyn_cast<GEPOperator>(BasePtr);
8217 auto *It =
find_if(Ptrs, IsaPred<GEPOperator>);
8218 if (It != Ptrs.
end())
8219 BaseGEP = cast<GEPOperator>(*It);
8224 BaseGEP->getPointerOperand(), Indices, VecTy,
8229 return std::make_pair(ScalarCost, VecCost);
8234 for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
8236 switch (E.getOpcode()) {
8237 case Instruction::Load: {
8240 if (E.State != TreeEntry::Vectorize)
8242 Type *ScalarTy = E.getMainOp()->getType();
8244 Align CommonAlignment = computeCommonAlignment<LoadInst>(E.Scalars);
8251 auto *BaseLI = cast<LoadInst>(E.Scalars.back());
8258 Instruction::Load, VecTy, BaseLI->getPointerOperand(),
8259 false, CommonAlignment,
CostKind, BaseLI);
8260 if (StridedCost < OriginalVecCost)
8263 E.State = TreeEntry::StridedVectorize;
8267 case Instruction::Store: {
8269 cast<StoreInst>(E.getMainOp())->getValueOperand()->getType();
8271 Align CommonAlignment = computeCommonAlignment<StoreInst>(E.Scalars);
8278 auto *BaseSI = cast<StoreInst>(E.Scalars.back());
8285 Instruction::Store, VecTy, BaseSI->getPointerOperand(),
8286 false, CommonAlignment,
CostKind, BaseSI);
8287 if (StridedCost < OriginalVecCost)
8290 E.State = TreeEntry::StridedVectorize;
8307 bool IsFinalized =
false;
8320 bool SameNodesEstimated =
true;
8329 if (
auto *VTy = dyn_cast<VectorType>(Ty))
8345 const unsigned Sz = R.DL->getTypeSizeInBits(ScalarTy);
8346 unsigned MinVF = R.getMinVF(2 * Sz);
8347 if (VL.
size() > 2 &&
8348 ((S.getOpcode() == Instruction::Load && !S.isAltShuffle()) ||
8349 (InVectors.
empty() &&
8352 ArrayRef<Value *> SubVL = VL.slice(Idx * MinVF, MinVF);
8353 InstructionsState S = getSameOpcode(SubVL, *R.TLI);
8354 return S.getOpcode() == Instruction::Load &&
8357 !
all_of(Gathers, [&](
Value *V) {
return R.getTreeEntry(V); }) &&
8363 unsigned StartIdx = 0;
8364 unsigned VF = VL.
size() / 2;
8365 for (; VF >= MinVF; VF /= 2) {
8366 for (
unsigned Cnt = StartIdx,
End = VL.
size(); Cnt + VF <=
End;
8369 if (S.getOpcode() != Instruction::Load || S.isAltShuffle()) {
8371 if (SliceS.getOpcode() != Instruction::Load ||
8372 SliceS.isAltShuffle())
8380 CurrentOrder, PointerOps);
8390 CurrentOrder.
empty()) ||
8399 if (Cnt == StartIdx)
8408 if (StartIdx >= VL.
size())
8411 if (!VectorizedLoads.
empty())
8414 if (!VectorizedLoads.
empty()) {
8416 bool NeedInsertSubvectorAnalysis =
8417 !NumParts || (VL.
size() / VF) > NumParts;
8423 getBuildVectorCost(VL.
slice(
I, std::min(
End -
I, VF)), Root);
8430 for (
Value *V : VectorizedLoads) {
8431 auto *LI = cast<LoadInst>(V);
8438 for (
const std::pair<unsigned, LoadsState> &
P : VectorizedStarts) {
8439 auto *LI = cast<LoadInst>(VL[
P.first]);
8448 false, Alignment, CostKind, LI);
8451 P.first, std::min<unsigned>(VL.
size() -
P.first, VF))))
8452 if (!R.areAllUsersVectorized(cast<Instruction>(V)))
8454 LoadTy, CostKind,
Idx);
8458 PointerOps[
I] = cast<LoadInst>(V)->getPointerOperand();
8459 auto [ScalarGEPCost, VectorGEPCost] =
8461 Instruction::Load, CostKind, LI->
getType(), LoadTy);
8462 GatherCost += VectorGEPCost - ScalarGEPCost;
8464 for (
unsigned P : ScatterVectorized) {
8465 auto *LI0 = cast<LoadInst>(VL[
P]);
8467 Align CommonAlignment = computeCommonAlignment<LoadInst>(Slice);
8469 Instruction::Load, LoadTy, LI0->getPointerOperand(),
8470 false, CommonAlignment, CostKind, LI0);
8474 PointerOps[
I] = cast<LoadInst>(V)->getPointerOperand();
8482 auto [ScalarGEPCost, VectorGEPCost] =
8484 CostKind, ScalarTy, VecTy);
8485 GatherCost += VectorGEPCost - ScalarGEPCost;
8486 if (!Order.
empty()) {
8490 VecTy, Mask, CostKind);
8493 GatherCost += R.getGatherCost(PointerOps,
true,
8494 PointerOps.
front()->getType());
8497 if (NeedInsertSubvectorAnalysis) {
8500 for (
unsigned I = VF, E = VL.
size();
I < E;
I += VF) {
8501 for (
unsigned Idx : seq<unsigned>(0, E))
8504 ShuffleMask, CostKind,
I, LoadTy);
8507 GatherCost -= ScalarsCost;
8509 GatherCost = std::min(BaseCost, GatherCost);
8510 }
else if (!Root &&
isSplat(VL)) {
8513 const auto *It =
find_if_not(VL, IsaPred<UndefValue>);
8514 assert(It != VL.
end() &&
"Expected at least one non-undef value.");
8517 count(VL, *It) > 1 &&
8521 CostKind, std::distance(VL.
begin(), It),
8526 return isa<PoisonValue>(V) ? PoisonMaskElem : 0;
8532 VecTy, ShuffleMask, CostKind,
8537 (
all_of(Gathers, IsaPred<UndefValue>)
8539 : R.getGatherCost(Gathers, !Root && VL.
equals(Gathers),
8547 ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
8548 unsigned NumParts) {
8549 assert(VL.
size() > NumParts &&
"Unexpected scalarized shuffle.");
8551 std::accumulate(VL.
begin(), VL.
end(), 0, [](
unsigned Sz,
Value *V) {
8552 auto *EE = dyn_cast<ExtractElementInst>(V);
8555 auto *VecTy = dyn_cast<FixedVectorType>(EE->getVectorOperandType());
8558 return std::max(Sz, VecTy->getNumElements());
8564 -> std::optional<TTI::ShuffleKind> {
8565 if (NumElts <= EltsPerVector)
8566 return std::nullopt;
8568 alignDown(std::accumulate(Mask.begin(), Mask.end(), INT_MAX,
8570 if (I == PoisonMaskElem)
8572 return std::min(S, I);
8575 int OffsetReg1 = OffsetReg0;
8579 int FirstRegId = -1;
8580 Indices.assign(1, OffsetReg0);
8584 int Idx =
I - OffsetReg0;
8586 (
Idx / NumElts) * NumParts + (
Idx % NumElts) / EltsPerVector;
8589 RegIndices.
insert(RegId);
8590 if (RegIndices.
size() > 2)
8591 return std::nullopt;
8592 if (RegIndices.
size() == 2) {
8594 if (Indices.size() == 1) {
8597 std::next(Mask.begin(), Pos), Mask.end(), INT_MAX,
8599 if (I == PoisonMaskElem)
8601 int RegId = ((I - OffsetReg0) / NumElts) * NumParts +
8602 ((I - OffsetReg0) % NumElts) / EltsPerVector;
8603 if (RegId == FirstRegId)
8605 return std::min(S, I);
8608 Indices.push_back(OffsetReg1 % NumElts);
8610 Idx =
I - OffsetReg1;
8612 I = (
Idx % NumElts) % EltsPerVector +
8613 (RegId == FirstRegId ? 0 : EltsPerVector);
8622 for (
unsigned Part : seq<unsigned>(NumParts)) {
8623 if (!ShuffleKinds[Part])
8626 Part * EltsPerVector,
getNumElems(Mask.size(), EltsPerVector, Part));
8630 std::optional<TTI::ShuffleKind> RegShuffleKind =
8631 CheckPerRegistersShuffle(SubMask, Indices);
8632 if (!RegShuffleKind) {
8635 MaskSlice, std::max<unsigned>(NumElts, MaskSlice.
size())))
8647 for (
unsigned Idx : Indices) {
8649 "SK_ExtractSubvector index out of range");
8653 std::nullopt, CostKind,
Idx,
8662 if (OriginalCost <
Cost)
8663 Cost = OriginalCost;
8671 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
8678 void estimateNodesPermuteCost(
const TreeEntry &E1,
const TreeEntry *E2,
8680 unsigned SliceSize) {
8681 if (SameNodesEstimated) {
8687 if ((InVectors.
size() == 2 &&
8688 InVectors.
front().get<
const TreeEntry *>() == &E1 &&
8689 InVectors.
back().get<
const TreeEntry *>() == E2) ||
8690 (!E2 && InVectors.
front().get<
const TreeEntry *>() == &E1)) {
8691 unsigned Limit =
getNumElems(Mask.size(), SliceSize, Part);
8694 "Expected all poisoned elements.");
8696 copy(SubMask, std::next(CommonMask.
begin(), SliceSize * Part));
8701 Cost += createShuffle(InVectors.
front(),
8702 InVectors.
size() == 1 ?
nullptr : InVectors.
back(),
8704 transformMaskAfterShuffle(CommonMask, CommonMask);
8706 SameNodesEstimated =
false;
8707 if (!E2 && InVectors.
size() == 1) {
8708 unsigned VF = E1.getVectorFactor();
8711 cast<FixedVectorType>(V1->
getType())->getNumElements());
8713 const auto *E = InVectors.
front().get<
const TreeEntry *>();
8714 VF = std::max(VF, E->getVectorFactor());
8716 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
8718 CommonMask[
Idx] = Mask[
Idx] + VF;
8719 Cost += createShuffle(InVectors.
front(), &E1, CommonMask);
8720 transformMaskAfterShuffle(CommonMask, CommonMask);
8722 Cost += createShuffle(&E1, E2, Mask);
8723 transformMaskAfterShuffle(CommonMask, Mask);
8727 class ShuffleCostBuilder {
8730 static bool isEmptyOrIdentity(
ArrayRef<int> Mask,
unsigned VF) {
8732 return Mask.empty() ||
8733 (VF == Mask.size() &&
8741 ~ShuffleCostBuilder() =
default;
8746 cast<VectorType>(V1->
getType())->getElementCount().getKnownMinValue();
8747 if (isEmptyOrIdentity(Mask, VF))
8750 cast<VectorType>(V1->
getType()), Mask);
8755 cast<VectorType>(V1->
getType())->getElementCount().getKnownMinValue();
8756 if (isEmptyOrIdentity(Mask, VF))
8759 cast<VectorType>(V1->
getType()), Mask);
8765 void resizeToMatch(
Value *&,
Value *&)
const {}
8775 ShuffleCostBuilder Builder(
TTI);
8778 unsigned CommonVF = Mask.size();
8780 auto GetNodeMinBWAffectedCost = [&](
const TreeEntry &E,
8784 Type *EScalarTy = E.Scalars.front()->getType();
8785 bool IsSigned =
true;
8786 if (
auto It = R.MinBWs.find(&E); It != R.MinBWs.end()) {
8788 IsSigned = It->second.second;
8790 if (EScalarTy != ScalarTy) {
8791 unsigned CastOpcode = Instruction::Trunc;
8792 unsigned DstSz = R.DL->getTypeSizeInBits(ScalarTy);
8793 unsigned SrcSz = R.DL->getTypeSizeInBits(EScalarTy);
8795 CastOpcode = IsSigned ? Instruction::SExt : Instruction::ZExt;
8803 if (isa<Constant>(V))
8805 auto *VecTy = cast<VectorType>(V->getType());
8807 if (EScalarTy != ScalarTy) {
8809 unsigned CastOpcode = Instruction::Trunc;
8810 unsigned DstSz = R.DL->getTypeSizeInBits(ScalarTy);
8811 unsigned SrcSz = R.DL->getTypeSizeInBits(EScalarTy);
8813 CastOpcode = IsSigned ? Instruction::SExt : Instruction::ZExt;
8820 if (!V1 && !V2 && !P2.
isNull()) {
8822 const TreeEntry *E = P1.
get<
const TreeEntry *>();
8823 unsigned VF = E->getVectorFactor();
8824 const TreeEntry *E2 = P2.
get<
const TreeEntry *>();
8825 CommonVF = std::max(VF, E2->getVectorFactor());
8828 return Idx < 2 * static_cast<int>(CommonVF);
8830 "All elements in mask must be less than 2 * CommonVF.");
8831 if (E->Scalars.size() == E2->Scalars.size()) {
8835 for (
int &
Idx : CommonMask) {
8838 if (
Idx <
static_cast<int>(CommonVF) && !EMask.
empty())
8840 else if (
Idx >=
static_cast<int>(CommonVF))
8841 Idx = (E2Mask.
empty() ?
Idx - CommonVF : E2Mask[
Idx - CommonVF]) +
8845 CommonVF = E->Scalars.size();
8846 ExtraCost += GetNodeMinBWAffectedCost(*E, CommonVF) +
8847 GetNodeMinBWAffectedCost(*E2, CommonVF);
8849 ExtraCost += GetNodeMinBWAffectedCost(*E, E->getVectorFactor()) +
8850 GetNodeMinBWAffectedCost(*E2, E2->getVectorFactor());
8854 }
else if (!V1 && P2.
isNull()) {
8856 const TreeEntry *E = P1.
get<
const TreeEntry *>();
8857 unsigned VF = E->getVectorFactor();
8861 [=](
int Idx) {
return Idx < static_cast<int>(CommonVF); }) &&
8862 "All elements in mask must be less than CommonVF.");
8863 if (E->Scalars.size() == Mask.size() && VF != Mask.size()) {
8865 assert(!EMask.
empty() &&
"Expected non-empty common mask.");
8866 for (
int &
Idx : CommonMask) {
8870 CommonVF = E->Scalars.size();
8872 ExtraCost += GetNodeMinBWAffectedCost(*E, CommonVF);
8875 if (!E->ReorderIndices.empty() && CommonVF == E->ReorderIndices.size() &&
8876 CommonVF == CommonMask.
size() &&
8878 [](
const auto &&
P) {
8880 static_cast<unsigned>(
P.value()) !=
P.index();
8888 }
else if (V1 && P2.
isNull()) {
8890 ExtraCost += GetValueMinBWAffectedCost(V1);
8891 CommonVF = getVF(V1);
8894 [=](
int Idx) {
return Idx < static_cast<int>(CommonVF); }) &&
8895 "All elements in mask must be less than CommonVF.");
8896 }
else if (V1 && !V2) {
8898 unsigned VF = getVF(V1);
8899 const TreeEntry *E2 = P2.
get<
const TreeEntry *>();
8900 CommonVF = std::max(VF, E2->getVectorFactor());
8903 return Idx < 2 * static_cast<int>(CommonVF);
8905 "All elements in mask must be less than 2 * CommonVF.");
8906 if (E2->Scalars.size() == VF && VF != CommonVF) {
8908 assert(!E2Mask.
empty() &&
"Expected non-empty common mask.");
8909 for (
int &
Idx : CommonMask) {
8912 if (
Idx >=
static_cast<int>(CommonVF))
8913 Idx = E2Mask[
Idx - CommonVF] + VF;
8917 ExtraCost += GetValueMinBWAffectedCost(V1);
8919 ExtraCost += GetNodeMinBWAffectedCost(
8920 *E2, std::min(CommonVF, E2->getVectorFactor()));
8922 }
else if (!V1 && V2) {
8924 unsigned VF = getVF(V2);
8925 const TreeEntry *E1 = P1.
get<
const TreeEntry *>();
8926 CommonVF = std::max(VF, E1->getVectorFactor());
8929 return Idx < 2 * static_cast<int>(CommonVF);
8931 "All elements in mask must be less than 2 * CommonVF.");
8932 if (E1->Scalars.size() == VF && VF != CommonVF) {
8934 assert(!E1Mask.
empty() &&
"Expected non-empty common mask.");
8935 for (
int &
Idx : CommonMask) {
8938 if (
Idx >=
static_cast<int>(CommonVF))
8939 Idx = E1Mask[
Idx - CommonVF] + VF;
8945 ExtraCost += GetNodeMinBWAffectedCost(
8946 *E1, std::min(CommonVF, E1->getVectorFactor()));
8948 ExtraCost += GetValueMinBWAffectedCost(V2);
8951 assert(V1 && V2 &&
"Expected both vectors.");
8952 unsigned VF = getVF(V1);
8953 CommonVF = std::max(VF, getVF(V2));
8956 return Idx < 2 * static_cast<int>(CommonVF);
8958 "All elements in mask must be less than 2 * CommonVF.");
8960 GetValueMinBWAffectedCost(V1) + GetValueMinBWAffectedCost(V2);
8961 if (V1->
getType() != V2->getType()) {
8965 if (cast<VectorType>(V1->
getType())->getElementType() != ScalarTy)
8967 if (cast<VectorType>(V2->getType())->getElementType() != ScalarTy)
8971 if (
auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
8978 if (InVectors.
size() == 2)
8980 return ExtraCost + BaseShuffleAnalysis::createShuffle<InstructionCost>(
8981 V1, V2, CommonMask, Builder);
8988 : BaseShuffleAnalysis(ScalarTy),
TTI(
TTI),
8989 VectorizedVals(VectorizedVals.begin(), VectorizedVals.end()), R(R),
8990 CheckedExtracts(CheckedExtracts) {}
8992 ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
8993 unsigned NumParts,
bool &UseVecBaseAsInput) {
8994 UseVecBaseAsInput =
false;
8997 Value *VecBase =
nullptr;
9000 if (NumParts == VL.
size())
9004 bool PrevNodeFound =
any_of(
9006 [&](
const std::unique_ptr<TreeEntry> &TE) {
9007 return ((!TE->isAltShuffle() &&
9008 TE->getOpcode() == Instruction::ExtractElement) ||
9010 all_of(enumerate(TE->Scalars), [&](auto &&Data) {
9011 return VL.size() > Data.index() &&
9012 (Mask[Data.index()] == PoisonMaskElem ||
9013 isa<UndefValue>(VL[Data.index()]) ||
9014 Data.value() == VL[Data.index()]);
9019 for (
unsigned Part : seq<unsigned>(NumParts)) {
9021 ArrayRef<int> SubMask = Mask.slice(Part * SliceSize, Limit);
9024 if (isa<UndefValue>(V) ||
9033 auto *EE = cast<ExtractElementInst>(V);
9034 VecBase = EE->getVectorOperand();
9035 UniqueBases.
insert(VecBase);
9036 const TreeEntry *VE = R.getTreeEntry(V);
9037 if (!CheckedExtracts.
insert(V).second ||
9038 !R.areAllUsersVectorized(cast<Instruction>(V), &VectorizedVals) ||
9041 return isa<GetElementPtrInst>(U) &&
9042 !R.areAllUsersVectorized(cast<Instruction>(U),
9050 unsigned Idx = *EEIdx;
9052 if (EE->hasOneUse() || !PrevNodeFound) {
9054 if (isa<SExtInst, ZExtInst>(Ext) &&
9055 all_of(Ext->users(), IsaPred<GetElementPtrInst>)) {
9060 EE->getVectorOperandType(),
Idx);
9063 Ext->getOpcode(), Ext->getType(), EE->getType(),
9079 Cost += computeExtractCost(VL, Mask, ShuffleKinds, NumParts);
9082 transformMaskAfterShuffle(CommonMask, CommonMask);
9083 SameNodesEstimated =
false;
9084 if (NumParts != 1 && UniqueBases.
size() != 1) {
9085 UseVecBaseAsInput =
true;
9093 std::optional<InstructionCost>
9097 return std::nullopt;
9103 return Idx < static_cast<int>(E1.getVectorFactor());
9105 "Expected single vector shuffle mask.");
9109 if (InVectors.
empty()) {
9110 CommonMask.
assign(Mask.begin(), Mask.end());
9111 InVectors.
assign({&E1, &E2});
9114 assert(!CommonMask.
empty() &&
"Expected non-empty common mask.");
9117 if (NumParts == 0 || NumParts >= Mask.size())
9122 unsigned Part = std::distance(Mask.begin(), It) / SliceSize;
9123 estimateNodesPermuteCost(E1, &E2, Mask, Part, SliceSize);
9126 if (InVectors.
empty()) {
9127 CommonMask.
assign(Mask.begin(), Mask.end());
9128 InVectors.
assign(1, &E1);
9131 assert(!CommonMask.
empty() &&
"Expected non-empty common mask.");
9134 if (NumParts == 0 || NumParts >= Mask.size())
9139 unsigned Part = std::distance(Mask.begin(), It) / SliceSize;
9140 estimateNodesPermuteCost(E1,
nullptr, Mask, Part, SliceSize);
9141 if (!SameNodesEstimated && InVectors.
size() == 1)
9154 cast<ExtractElementInst>(InVectors.
front()
9155 .get<
const TreeEntry *>()
9156 ->Scalars[
P.index()]);
9157 return EI->getVectorOperand() == V1 ||
9158 EI->getVectorOperand() == V2;
9160 "Expected extractelement vectors.");
9164 if (InVectors.
empty()) {
9166 "Expected empty input mask/vectors.");
9167 CommonMask.
assign(Mask.begin(), Mask.end());
9174 InVectors.
front().is<
const TreeEntry *>() && !CommonMask.
empty() &&
9178 .get<const TreeEntry *>()
9179 ->Scalars[
P.index()];
9181 return P.value() == Mask[
P.index()] ||
9182 isa<UndefValue>(Scalar);
9183 if (isa<Constant>(V1))
9185 auto *EI = cast<ExtractElementInst>(Scalar);
9186 return EI->getVectorOperand() == V1;
9188 "Expected only tree entry for extractelement vectors.");
9192 "Expected only tree entries from extracts/reused buildvectors.");
9193 unsigned VF = getVF(V1);
9194 if (InVectors.
size() == 2) {
9195 Cost += createShuffle(InVectors.
front(), InVectors.
back(), CommonMask);
9196 transformMaskAfterShuffle(CommonMask, CommonMask);
9197 VF = std::max<unsigned>(VF, CommonMask.
size());
9198 }
else if (
const auto *InTE =
9199 InVectors.
front().dyn_cast<
const TreeEntry *>()) {
9200 VF = std::max(VF, InTE->getVectorFactor());
9204 ->getNumElements());
9207 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
9209 CommonMask[
Idx] = Mask[
Idx] + VF;
9212 Value *Root =
nullptr) {
9213 Cost += getBuildVectorCost(VL, Root);
9217 unsigned VF = VL.
size();
9219 VF = std::min(VF, MaskVF);
9221 if (isa<UndefValue>(V)) {
9227 if (
auto *VecTy = dyn_cast<FixedVectorType>(Vals.
front()->getType())) {
9234 Type *ScalarTy = V->getType()->getScalarType();
9236 if (isa<PoisonValue>(V))
9238 else if (isa<UndefValue>(V))
9242 std::fill_n(NewVals.
begin() +
I * VecTyNumElements, VecTyNumElements,
9251 cast<FixedVectorType>(Root->
getType())->getNumElements()),
9262 if (InVectors.
size() == 2)
9263 Cost += createShuffle(Vec, InVectors.
back(), CommonMask);
9265 Cost += createShuffle(Vec,
nullptr, CommonMask);
9266 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
9270 "Expected vector length for the final value before action.");
9272 Action(V, CommonMask);
9273 InVectors.
front() = V;
9276 if (CommonMask.
empty()) {
9277 assert(InVectors.
size() == 1 &&
"Expected only one vector with no mask");
9281 createShuffle(InVectors.
front(),
9282 InVectors.
size() == 2 ? InVectors.
back() :
nullptr,
9288 "Shuffle construction must be finalized.");
9292const BoUpSLP::TreeEntry *BoUpSLP::getOperandEntry(
const TreeEntry *E,
9293 unsigned Idx)
const {
9295 if (
const TreeEntry *TE = getTreeEntry(
Op)) {
9296 if (
find_if(TE->UserTreeIndices, [&](
const EdgeInfo &EI) {
9297 return EI.EdgeIdx == Idx && EI.UserTE == E;
9298 }) != TE->UserTreeIndices.end())
9300 auto MIt = MultiNodeScalars.
find(
Op);
9301 if (MIt != MultiNodeScalars.
end()) {
9302 for (
const TreeEntry *TE : MIt->second) {
9303 if (
find_if(TE->UserTreeIndices, [&](
const EdgeInfo &EI) {
9304 return EI.EdgeIdx == Idx && EI.UserTE == E;
9305 }) != TE->UserTreeIndices.end())
9311 find_if(VectorizableTree, [&](
const std::unique_ptr<TreeEntry> &TE) {
9312 return TE->isGather() &&
9313 find_if(
TE->UserTreeIndices, [&](
const EdgeInfo &EI) {
9314 return EI.EdgeIdx == Idx && EI.UserTE == E;
9315 }) !=
TE->UserTreeIndices.end();
9317 assert(It != VectorizableTree.end() &&
"Expected vectorizable entry.");
9322 if (
TE.State == TreeEntry::ScatterVectorize ||
9323 TE.State == TreeEntry::StridedVectorize)
9325 if (
TE.State == TreeEntry::Vectorize &&
TE.getOpcode() == Instruction::Load &&
9326 !
TE.isAltShuffle()) {
9327 if (
TE.ReorderIndices.empty())
9366 Type *ScalarTy = VL[0]->getType();
9367 if (!E->isGather()) {
9368 if (
auto *SI = dyn_cast<StoreInst>(VL[0]))
9369 ScalarTy =
SI->getValueOperand()->getType();
9370 else if (
auto *CI = dyn_cast<CmpInst>(VL[0]))
9372 else if (
auto *IE = dyn_cast<InsertElementInst>(VL[0]))
9373 ScalarTy =
IE->getOperand(1)->getType();
9381 auto It = MinBWs.
find(E);
9382 Type *OrigScalarTy = ScalarTy;
9383 if (It != MinBWs.
end())
9386 unsigned EntryVF = E->getVectorFactor();
9389 bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
9390 if (E->isGather()) {
9393 if (isa<InsertElementInst>(VL[0]))
9395 return processBuildVector<ShuffleCostEstimator, InstructionCost>(
9396 E, ScalarTy, *
TTI, VectorizedVals, *
this, CheckedExtracts);
9401 if (!E->ReorderIndices.empty() &&
9402 (E->State != TreeEntry::StridedVectorize || !IsReverseOrder)) {
9404 if (E->getOpcode() == Instruction::Store) {
9406 NewMask.
resize(E->ReorderIndices.size());
9407 copy(E->ReorderIndices, NewMask.
begin());
9413 if (NeedToShuffleReuses)
9414 ::addMask(Mask, E->ReuseShuffleIndices);
9418 assert((E->State == TreeEntry::Vectorize ||
9419 E->State == TreeEntry::ScatterVectorize ||
9420 E->State == TreeEntry::StridedVectorize) &&
9424 (E->getOpcode() == Instruction::GetElementPtr &&
9425 E->getMainOp()->getType()->isPointerTy())) &&
9428 unsigned ShuffleOrOp =
9429 E->isAltShuffle() ? (
unsigned)Instruction::ShuffleVector : E->getOpcode();
9431 const unsigned Sz = UniqueValues.
size();
9433 for (
unsigned I = 0;
I < Sz; ++
I) {
9434 if (getTreeEntry(UniqueValues[
I]) == E)
9438 auto GetCastContextHint = [&](
Value *
V) {
9439 if (
const TreeEntry *OpTE = getTreeEntry(V))
9440 return getCastContextHint(*OpTE);
9441 InstructionsState SrcState =
getSameOpcode(E->getOperand(0), *TLI);
9442 if (SrcState.getOpcode() == Instruction::Load && !SrcState.isAltShuffle())
9451 if (isa<CastInst, CallInst>(VL0)) {
9455 ScalarCost = (Sz - UsedScalars.count()) * ScalarEltCost(0);
9457 for (
unsigned I = 0;
I < Sz; ++
I) {
9458 if (UsedScalars.test(
I))
9460 ScalarCost += ScalarEltCost(
I);
9468 const EdgeInfo &EI = E->UserTreeIndices.front();
9469 if ((EI.UserTE->getOpcode() != Instruction::Select ||
9471 It != MinBWs.
end()) {
9472 auto UserBWIt = MinBWs.
find(EI.UserTE);
9473 Type *UserScalarTy =
9474 EI.UserTE->getOperand(EI.EdgeIdx).front()->getType();
9475 if (UserBWIt != MinBWs.
end())
9477 UserBWIt->second.first);
9478 if (ScalarTy != UserScalarTy) {
9479 unsigned BWSz =
DL->getTypeSizeInBits(ScalarTy);
9480 unsigned SrcBWSz =
DL->getTypeSizeInBits(UserScalarTy);
9485 VecOpcode = Instruction::Trunc;
9488 It->second.second ? Instruction::SExt : Instruction::ZExt;
9495 LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecCost - CommonCost,
9496 ScalarCost,
"Calculated costs for Tree"));
9497 return VecCost - ScalarCost;
9502 assert((E->State == TreeEntry::Vectorize ||
9503 E->State == TreeEntry::StridedVectorize) &&
9504 "Entry state expected to be Vectorize or StridedVectorize here.");
9508 *
TTI, Ptrs, BasePtr, E->getOpcode(),
CostKind, OrigScalarTy, VecTy);
9509 LLVM_DEBUG(dumpTreeCosts(E, 0, VecCost, ScalarCost,
9510 "Calculated GEPs cost for Tree"));
9512 return VecCost - ScalarCost;
9515 switch (ShuffleOrOp) {
9516 case Instruction::PHI: {
9520 for (
Value *V : UniqueValues) {
9521 auto *
PHI = dyn_cast<PHINode>(V);
9526 for (
unsigned I = 0,
N =
PHI->getNumIncomingValues();
I <
N; ++
I) {
9530 if (
const TreeEntry *OpTE = getTreeEntry(
Operands.front()))
9532 if (!OpTE->ReuseShuffleIndices.empty())
9533 ScalarCost +=
TTI::TCC_Basic * (OpTE->ReuseShuffleIndices.size() -
9534 OpTE->Scalars.size());
9537 return CommonCost - ScalarCost;
9539 case Instruction::ExtractValue:
9540 case Instruction::ExtractElement: {
9541 auto GetScalarCost = [&](
unsigned Idx) {
9542 auto *
I = cast<Instruction>(UniqueValues[
Idx]);
9544 if (ShuffleOrOp == Instruction::ExtractElement) {
9545 auto *EE = cast<ExtractElementInst>(
I);
9546 SrcVecTy = EE->getVectorOperandType();
9548 auto *EV = cast<ExtractValueInst>(
I);
9549 Type *AggregateTy = EV->getAggregateOperand()->getType();
9551 if (
auto *ATy = dyn_cast<ArrayType>(AggregateTy))
9552 NumElts = ATy->getNumElements();
9557 if (
I->hasOneUse()) {
9559 if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
9560 all_of(
Ext->users(), IsaPred<GetElementPtrInst>)) {
9567 Ext->getOpcode(),
Ext->getType(),
I->getType(),
9575 auto GetVectorCost = [](
InstructionCost CommonCost) {
return CommonCost; };
9576 return GetCostDiff(GetScalarCost, GetVectorCost);
9578 case Instruction::InsertElement: {
9579 assert(E->ReuseShuffleIndices.empty() &&
9580 "Unique insertelements only are expected.");
9581 auto *SrcVecTy = cast<FixedVectorType>(VL0->
getType());
9582 unsigned const NumElts = SrcVecTy->getNumElements();
9583 unsigned const NumScalars = VL.
size();
9589 unsigned OffsetEnd = OffsetBeg;
9590 InsertMask[OffsetBeg] = 0;
9593 if (OffsetBeg >
Idx)
9595 else if (OffsetEnd <
Idx)
9597 InsertMask[
Idx] =
I + 1;
9601 VecScalarsSz =
PowerOf2Ceil((NumElts + NumOfParts - 1) / NumOfParts);
9602 unsigned VecSz = (1 + OffsetEnd / VecScalarsSz - OffsetBeg / VecScalarsSz) *
9604 unsigned Offset = VecScalarsSz * (OffsetBeg / VecScalarsSz);
9605 unsigned InsertVecSz = std::min<unsigned>(
9607 ((OffsetEnd - OffsetBeg + VecScalarsSz) / VecScalarsSz) * VecScalarsSz);
9608 bool IsWholeSubvector =
9609 OffsetBeg ==
Offset && ((OffsetEnd + 1) % VecScalarsSz == 0);
9613 if (OffsetBeg + InsertVecSz > VecSz) {
9616 InsertVecSz = VecSz;
9622 if (!E->ReorderIndices.empty()) {
9627 std::iota(
Mask.begin(), std::next(
Mask.begin(), InsertVecSz), 0);
9629 bool IsIdentity =
true;
9631 Mask.swap(PrevMask);
9632 for (
unsigned I = 0;
I < NumScalars; ++
I) {
9634 DemandedElts.
setBit(InsertIdx);
9635 IsIdentity &= InsertIdx - OffsetBeg ==
I;
9636 Mask[InsertIdx - OffsetBeg] =
I;
9638 assert(
Offset < NumElts &&
"Failed to find vector index offset");
9653 auto *FirstInsert = cast<Instruction>(*
find_if(E->Scalars, [E](
Value *V) {
9654 return !is_contained(E->Scalars, cast<Instruction>(V)->getOperand(0));
9662 buildUseMask(NumElts, InsertMask, UseMask::UndefsAsMask));
9663 if (!InMask.
all() && NumScalars != NumElts && !IsWholeSubvector) {
9664 if (InsertVecSz != VecSz) {
9676 for (
unsigned I = OffsetEnd + 1 -
Offset;
I < VecSz; ++
I)
9685 case Instruction::ZExt:
9686 case Instruction::SExt:
9687 case Instruction::FPToUI:
9688 case Instruction::FPToSI:
9689 case Instruction::FPExt:
9690 case Instruction::PtrToInt:
9691 case Instruction::IntToPtr:
9692 case Instruction::SIToFP:
9693 case Instruction::UIToFP:
9694 case Instruction::Trunc:
9695 case Instruction::FPTrunc:
9696 case Instruction::BitCast: {
9697 auto SrcIt = MinBWs.
find(getOperandEntry(E, 0));
9700 unsigned Opcode = ShuffleOrOp;
9701 unsigned VecOpcode = Opcode;
9703 (SrcIt != MinBWs.
end() || It != MinBWs.
end())) {
9705 unsigned SrcBWSz =
DL->getTypeSizeInBits(SrcScalarTy);
9706 if (SrcIt != MinBWs.
end()) {
9707 SrcBWSz = SrcIt->second.first;
9711 unsigned BWSz =
DL->getTypeSizeInBits(ScalarTy);
9712 if (BWSz == SrcBWSz) {
9713 VecOpcode = Instruction::BitCast;
9714 }
else if (BWSz < SrcBWSz) {
9715 VecOpcode = Instruction::Trunc;
9716 }
else if (It != MinBWs.
end()) {
9717 assert(BWSz > SrcBWSz &&
"Invalid cast!");
9718 VecOpcode = It->second.second ? Instruction::SExt : Instruction::ZExt;
9719 }
else if (SrcIt != MinBWs.
end()) {
9720 assert(BWSz > SrcBWSz &&
"Invalid cast!");
9722 SrcIt->second.second ? Instruction::SExt : Instruction::ZExt;
9724 }
else if (VecOpcode == Instruction::SIToFP && SrcIt != MinBWs.
end() &&
9725 !SrcIt->second.second) {
9726 VecOpcode = Instruction::UIToFP;
9729 auto *
VI = cast<Instruction>(UniqueValues[
Idx]);
9737 if (VecOpcode != Opcode && VecOpcode == Instruction::BitCast)
9739 auto *
VI = VL0->
getOpcode() == Opcode ? VL0 :
nullptr;
9743 VecOpcode == Opcode ? VI :
nullptr);
9745 return GetCostDiff(GetScalarCost, GetVectorCost);
9747 case Instruction::FCmp:
9748 case Instruction::ICmp:
9749 case Instruction::Select: {
9753 match(VL0, MatchCmp))
9759 auto GetScalarCost = [&](
unsigned Idx) {
9760 auto *
VI = cast<Instruction>(UniqueValues[
Idx]);
9766 !
match(VI, MatchCmp)) ||
9767 (CurrentPred != VecPred && CurrentPred != SwappedVecPred))
9773 E->getOpcode(), OrigScalarTy, Builder.getInt1Ty(), CurrentPred,
9777 Type *CanonicalType = OrigScalarTy;
9784 {CanonicalType, CanonicalType});
9790 auto *CI = cast<CmpInst>(
VI->getOperand(0));
9792 CI->
getOpcode(), OrigScalarTy, Builder.getInt1Ty(),
9795 ScalarCost = std::min(ScalarCost, IntrinsicCost);
9804 E->getOpcode(), VecTy, MaskTy, VecPred,
CostKind, VL0);
9810 Type *CanonicalType = VecTy;
9816 {CanonicalType, CanonicalType});
9823 cast<CmpInst>(cast<Instruction>(VL.
front())->getOperand(0));
9827 VecCost = std::min(VecCost, IntrinsicCost);
9829 if (
auto *SI = dyn_cast<SelectInst>(VL0)) {
9832 unsigned CondNumElements = CondType->getNumElements();
9834 assert(VecTyNumElements >= CondNumElements &&
9835 VecTyNumElements % CondNumElements == 0 &&
9836 "Cannot vectorize Instruction::Select");
9837 if (CondNumElements != VecTyNumElements) {
9846 return VecCost + CommonCost;
9848 return GetCostDiff(GetScalarCost, GetVectorCost);
9850 case Instruction::FNeg:
9851 case Instruction::Add:
9852 case Instruction::FAdd:
9853 case Instruction::Sub:
9854 case Instruction::FSub:
9855 case Instruction::Mul:
9856 case Instruction::FMul:
9857 case Instruction::UDiv:
9858 case Instruction::SDiv:
9859 case Instruction::FDiv:
9860 case Instruction::URem:
9861 case Instruction::SRem:
9862 case Instruction::FRem:
9863 case Instruction::Shl:
9864 case Instruction::LShr:
9865 case Instruction::AShr:
9866 case Instruction::And:
9867 case Instruction::Or:
9868 case Instruction::Xor: {
9869 auto GetScalarCost = [&](
unsigned Idx) {
9870 auto *
VI = cast<Instruction>(UniqueValues[
Idx]);
9871 unsigned OpIdx = isa<UnaryOperator>(VI) ? 0 : 1;
9880 if (ShuffleOrOp == Instruction::And && It != MinBWs.
end()) {
9881 for (
unsigned I : seq<unsigned>(0, E->getNumOperands())) {
9884 auto *CI = dyn_cast<ConstantInt>(
Op);
9885 return CI && CI->getValue().countr_one() >= It->second.first;
9890 unsigned OpIdx = isa<UnaryOperator>(VL0) ? 0 : 1;
9894 Op2Info, std::nullopt,
nullptr, TLI) +
9897 return GetCostDiff(GetScalarCost, GetVectorCost);
9899 case Instruction::GetElementPtr: {
9900 return CommonCost + GetGEPCostDiff(VL, VL0);
9902 case Instruction::Load: {
9903 auto GetScalarCost = [&](
unsigned Idx) {
9904 auto *
VI = cast<LoadInst>(UniqueValues[
Idx]);
9906 VI->getAlign(),
VI->getPointerAddressSpace(),
9909 auto *LI0 = cast<LoadInst>(VL0);
9912 if (E->State == TreeEntry::Vectorize) {
9914 Instruction::Load, VecTy, LI0->getAlign(),
9916 }
else if (E->State == TreeEntry::StridedVectorize) {
9917 Align CommonAlignment =
9918 computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
9920 Instruction::Load, VecTy, LI0->getPointerOperand(),
9923 assert(E->State == TreeEntry::ScatterVectorize &&
"Unknown EntryState");
9924 Align CommonAlignment =
9925 computeCommonAlignment<LoadInst>(UniqueValues.getArrayRef());
9927 Instruction::Load, VecTy, LI0->getPointerOperand(),
9930 return VecLdCost + CommonCost;
9936 if (E->State == TreeEntry::ScatterVectorize)
9942 PointerOps[
I] = cast<LoadInst>(V)->getPointerOperand();
9943 return Cost + GetGEPCostDiff(PointerOps, LI0->getPointerOperand());
9945 case Instruction::Store: {
9946 bool IsReorder = !E->ReorderIndices.empty();
9947 auto GetScalarCost = [=](
unsigned Idx) {
9948 auto *
VI = cast<StoreInst>(VL[
Idx]);
9951 VI->getAlign(),
VI->getPointerAddressSpace(),
9955 cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
9959 if (E->State == TreeEntry::StridedVectorize) {
9960 Align CommonAlignment =
9961 computeCommonAlignment<StoreInst>(UniqueValues.getArrayRef());
9963 Instruction::Store, VecTy, BaseSI->getPointerOperand(),
9966 assert(E->State == TreeEntry::Vectorize &&
9967 "Expected either strided or consecutive stores.");
9970 Instruction::Store, VecTy, BaseSI->getAlign(),
9971 BaseSI->getPointerAddressSpace(),
CostKind, OpInfo);
9973 return VecStCost + CommonCost;
9977 unsigned Idx = IsReorder ? E->ReorderIndices[
I] :
I;
9978 PointerOps[
Idx] = cast<StoreInst>(V)->getPointerOperand();
9981 return GetCostDiff(GetScalarCost, GetVectorCost) +
9982 GetGEPCostDiff(PointerOps, BaseSI->getPointerOperand());
9984 case Instruction::Call: {
9985 auto GetScalarCost = [&](
unsigned Idx) {
9986 auto *CI = cast<CallInst>(UniqueValues[
Idx]);
9997 auto *CI = cast<CallInst>(VL0);
10001 It != MinBWs.
end() ? It->second.first : 0);
10003 return std::min(VecCallCosts.first, VecCallCosts.second) + CommonCost;
10005 return GetCostDiff(GetScalarCost, GetVectorCost);
10007 case Instruction::ShuffleVector: {
10008 assert(E->isAltShuffle() &&
10013 (isa<CmpInst>(VL0) && isa<CmpInst>(E->getAltOp()))) &&
10014 "Invalid Shuffle Vector Operand");
10017 auto TryFindNodeWithEqualOperands = [=]() {
10018 for (
const std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
10021 if (
TE->isAltShuffle() &&
10022 ((
TE->getOpcode() == E->getOpcode() &&
10023 TE->getAltOpcode() == E->getAltOpcode()) ||
10024 (
TE->getOpcode() == E->getAltOpcode() &&
10025 TE->getAltOpcode() == E->getOpcode())) &&
10026 TE->hasEqualOperands(*E))
10031 auto GetScalarCost = [&](
unsigned Idx) {
10032 auto *
VI = cast<Instruction>(UniqueValues[
Idx]);
10033 assert(E->isOpcodeOrAlt(VI) &&
"Unexpected main/alternate opcode");
10043 if (TryFindNodeWithEqualOperands()) {
10045 dbgs() <<
"SLP: diamond match for alternate node found.\n";
10052 TTIRef.getArithmeticInstrCost(E->getOpcode(), VecTy,
CostKind);
10054 TTIRef.getArithmeticInstrCost(E->getAltOpcode(), VecTy,
CostKind);
10055 }
else if (
auto *CI0 = dyn_cast<CmpInst>(VL0)) {
10057 VecCost = TTIRef.getCmpSelInstrCost(E->getOpcode(), VecTy, MaskTy,
10058 CI0->getPredicate(),
CostKind, VL0);
10059 VecCost += TTIRef.getCmpSelInstrCost(
10060 E->getOpcode(), VecTy, MaskTy,
10061 cast<CmpInst>(E->getAltOp())->getPredicate(),
CostKind,
10064 Type *SrcSclTy = E->getMainOp()->getOperand(0)->getType();
10067 auto SrcIt = MinBWs.
find(getOperandEntry(E, 0));
10068 unsigned BWSz =
DL->getTypeSizeInBits(ScalarTy);
10070 DL->getTypeSizeInBits(E->getMainOp()->getOperand(0)->getType());
10071 if (SrcIt != MinBWs.
end()) {
10072 SrcBWSz = SrcIt->second.first;
10076 if (BWSz <= SrcBWSz) {
10077 if (BWSz < SrcBWSz)
10079 TTIRef.getCastInstrCost(Instruction::Trunc, VecTy, SrcTy,
10083 <<
"SLP: alternate extension, which should be truncated.\n";
10089 VecCost = TTIRef.getCastInstrCost(E->getOpcode(), VecTy, SrcTy,
10092 TTIRef.getCastInstrCost(E->getAltOpcode(), VecTy, SrcTy,
10096 E->buildAltOpShuffleMask(
10098 assert(E->isOpcodeOrAlt(
I) &&
"Unexpected main/alternate opcode");
10099 return I->getOpcode() == E->getAltOpcode();
10108 unsigned Opcode0 = E->getOpcode();
10109 unsigned Opcode1 = E->getAltOpcode();
10113 if (TTIRef.isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask)) {
10115 VecTy, Opcode0, Opcode1, OpcodeMask,
CostKind);
10116 return AltVecCost < VecCost ? AltVecCost : VecCost;
10121 return GetCostDiff(GetScalarCost, GetVectorCost);
10123 case Instruction::Freeze:
10130bool BoUpSLP::isFullyVectorizableTinyTree(
bool ForReduction)
const {
10132 << VectorizableTree.size() <<
" is fully vectorizable .\n");
10134 auto &&AreVectorizableGathers = [
this](
const TreeEntry *
TE,
unsigned Limit) {
10136 return TE->isGather() &&
10138 [
this](
Value *V) { return EphValues.contains(V); }) &&
10140 TE->Scalars.size() < Limit ||
10141 ((
TE->getOpcode() == Instruction::ExtractElement ||
10142 all_of(
TE->Scalars, IsaPred<ExtractElementInst, UndefValue>)) &&
10144 (
TE->isGather() &&
TE->getOpcode() == Instruction::Load &&
10145 !
TE->isAltShuffle()));
10149 if (VectorizableTree.size() == 1 &&
10150 (VectorizableTree[0]->State == TreeEntry::Vectorize ||
10151 VectorizableTree[0]->State == TreeEntry::StridedVectorize ||
10153 AreVectorizableGathers(VectorizableTree[0].
get(),
10154 VectorizableTree[0]->Scalars.size()) &&
10155 VectorizableTree[0]->getVectorFactor() > 2)))
10158 if (VectorizableTree.size() != 2)
10166 if (VectorizableTree[0]->State == TreeEntry::Vectorize &&
10167 AreVectorizableGathers(VectorizableTree[1].
get(),
10168 VectorizableTree[0]->Scalars.size()))
10172 if (VectorizableTree[0]->
isGather() ||
10173 (VectorizableTree[1]->isGather() &&
10174 VectorizableTree[0]->State != TreeEntry::ScatterVectorize &&
10175 VectorizableTree[0]->State != TreeEntry::StridedVectorize))
10183 bool MustMatchOrInst) {
10187 Value *ZextLoad = Root;
10188 const APInt *ShAmtC;
10189 bool FoundOr =
false;
10190 while (!isa<ConstantExpr>(ZextLoad) &&
10193 ShAmtC->
urem(8) == 0))) {
10194 auto *BinOp = cast<BinaryOperator>(ZextLoad);
10195 ZextLoad = BinOp->getOperand(0);
10196 if (BinOp->getOpcode() == Instruction::Or)
10201 if ((MustMatchOrInst && !FoundOr) || ZextLoad == Root ||
10208 Type *SrcTy = Load->getType();
10215 LLVM_DEBUG(
dbgs() <<
"SLP: Assume load combining for tree starting at "
10216 << *(cast<Instruction>(Root)) <<
"\n");
10225 unsigned NumElts = VectorizableTree[0]->Scalars.size();
10226 Value *FirstReduced = VectorizableTree[0]->Scalars[0];
10234 unsigned NumElts = Stores.
size();
10235 for (
Value *Scalar : Stores) {
10246 if (VectorizableTree.size() == 2 &&
10247 isa<InsertElementInst>(VectorizableTree[0]->Scalars[0]) &&
10248 VectorizableTree[1]->isGather() &&
10249 (VectorizableTree[1]->getVectorFactor() <= 2 ||
10250 !(
isSplat(VectorizableTree[1]->Scalars) ||
10258 constexpr int Limit = 4;
10260 !VectorizableTree.empty() &&
10261 all_of(VectorizableTree, [&](
const std::unique_ptr<TreeEntry> &TE) {
10262 return (TE->isGather() &&
10263 TE->getOpcode() != Instruction::ExtractElement &&
10264 count_if(TE->Scalars, IsaPred<ExtractElementInst>) <= Limit) ||
10265 TE->getOpcode() == Instruction::PHI;
10276 if (isFullyVectorizableTinyTree(ForReduction))
10281 bool IsAllowedSingleBVNode =
10282 VectorizableTree.size() > 1 ||
10283 (VectorizableTree.size() == 1 && VectorizableTree.front()->getOpcode() &&
10284 !VectorizableTree.front()->isAltShuffle() &&
10285 VectorizableTree.front()->getOpcode() != Instruction::PHI &&
10286 VectorizableTree.front()->getOpcode() != Instruction::GetElementPtr &&
10288 if (
any_of(VectorizableTree, [&](
const std::unique_ptr<TreeEntry> &TE) {
10289 return TE->isGather() &&
all_of(TE->Scalars, [&](
Value *V) {
10290 return isa<ExtractElementInst, UndefValue>(V) ||
10291 (IsAllowedSingleBVNode &&
10292 !V->hasNUsesOrMore(UsesLimit) &&
10293 any_of(V->users(), IsaPred<InsertElementInst>));
10298 assert(VectorizableTree.empty()
10299 ? ExternalUses.empty()
10300 :
true &&
"We shouldn't have any external users");
10312 unsigned BundleWidth = VectorizableTree.front()->Scalars.size();
10325 for (
const auto &TEPtr : VectorizableTree) {
10326 if (TEPtr->State != TreeEntry::Vectorize)
10328 Instruction *Inst = dyn_cast<Instruction>(TEPtr->Scalars[0]);
10334 auto *NodeA = DT->
getNode(
A->getParent());
10335 auto *NodeB = DT->
getNode(
B->getParent());
10336 assert(NodeA &&
"Should only process reachable instructions");
10337 assert(NodeB &&
"Should only process reachable instructions");
10338 assert((NodeA == NodeB) == (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
10339 "Different nodes should have different DFS numbers");
10340 if (NodeA != NodeB)
10341 return NodeA->getDFSNumIn() > NodeB->getDFSNumIn();
10342 return B->comesBefore(
A);
10352 LiveValues.
erase(PrevInst);
10353 for (
auto &J : PrevInst->
operands()) {
10354 if (isa<Instruction>(&*J) && getTreeEntry(&*J))
10355 LiveValues.
insert(cast<Instruction>(&*J));
10359 dbgs() <<
"SLP: #LV: " << LiveValues.
size();
10360 for (
auto *
X : LiveValues)
10361 dbgs() <<
" " <<
X->getName();
10362 dbgs() <<
", Looking at ";
10367 unsigned NumCalls = 0;
10371 while (InstIt != PrevInstIt) {
10372 if (PrevInstIt == PrevInst->
getParent()->rend()) {
10373 PrevInstIt = Inst->getParent()->rbegin();
10378 if (
auto *
II = dyn_cast<IntrinsicInst>(
I)) {
10379 if (
II->isAssumeLikeIntrinsic())
10383 for (
auto &ArgOp :
II->args())
10385 if (
auto *FPMO = dyn_cast<FPMathOperator>(
II))
10386 FMF = FPMO->getFastMathFlags();
10393 if (IntrCost < CallCost)
10400 if (isa<CallBase>(&*PrevInstIt) && !NoCallIntrinsic(&*PrevInstIt) &&
10401 &*PrevInstIt != PrevInst)
10409 for (
auto *
II : LiveValues) {
10410 auto *ScalarTy =
II->getType();
10411 if (
auto *VectorTy = dyn_cast<FixedVectorType>(ScalarTy))
10412 ScalarTy = VectorTy->getElementType();
10430 const auto *I1 = IE1;
10431 const auto *I2 = IE2;
10443 if (I1 && (I1 == IE1 || I1->hasOneUse()) &&
10445 I1 = dyn_cast<InsertElementInst>(I1->getOperand(0));
10446 if (I2 && ((I2 == IE2 || I2->
hasOneUse())) &&
10448 I2 = dyn_cast<InsertElementInst>(I2->getOperand(0));
10449 }
while ((I1 && PrevI1 != I1) || (I2 && PrevI2 != I2));
10456struct ValueSelect {
10457 template <
typename U>
10458 static std::enable_if_t<std::is_same_v<Value *, U>,
Value *>
get(
Value *V) {
10461 template <
typename U>
10462 static std::enable_if_t<!std::is_same_v<Value *, U>,
U>
get(
Value *) {
10480template <
typename T>
10486 assert(!ShuffleMask.empty() &&
"Empty list of shuffles for inserts.");
10488 auto VMIt = std::next(ShuffleMask.begin());
10491 buildUseMask(Mask.size(), Mask, UseMask::UndefsAsMask);
10493 if (!IsBaseUndef.
all()) {
10495 std::pair<T *, bool> Res =
10496 ResizeAction(ShuffleMask.begin()->first, Mask,
false);
10498 for (
unsigned Idx = 0, VF = Mask.size();
Idx < VF; ++
Idx) {
10502 Mask[
Idx] = (Res.second ?
Idx : Mask[
Idx]) + VF;
10504 auto *V = ValueSelect::get<T *>(
Base);
10506 assert((!V || GetVF(V) == Mask.size()) &&
10507 "Expected base vector of VF number of elements.");
10508 Prev = Action(Mask, {
nullptr, Res.first});
10509 }
else if (ShuffleMask.size() == 1) {
10512 std::pair<T *, bool> Res = ResizeAction(ShuffleMask.begin()->first, Mask,
10518 Prev = Action(Mask, {ShuffleMask.begin()->first});
10522 unsigned Vec1VF = GetVF(ShuffleMask.begin()->first);
10523 unsigned Vec2VF = GetVF(VMIt->first);
10524 if (Vec1VF == Vec2VF) {
10528 for (
unsigned I = 0, VF = Mask.size();
I < VF; ++
I) {
10531 Mask[
I] = SecMask[
I] + Vec1VF;
10534 Prev = Action(Mask, {ShuffleMask.begin()->first, VMIt->first});
10537 std::pair<T *, bool> Res1 = ResizeAction(ShuffleMask.begin()->first, Mask,
10539 std::pair<T *, bool> Res2 =
10540 ResizeAction(VMIt->first, VMIt->second,
false);
10542 for (
unsigned I = 0, VF = Mask.size();
I < VF; ++
I) {
10549 Mask[
I] = (Res2.second ?
I : SecMask[
I]) + VF;
10552 Prev = Action(Mask, {Res1.first, Res2.first});
10554 VMIt = std::next(VMIt);
10556 bool IsBaseNotUndef = !IsBaseUndef.
all();
10557 (void)IsBaseNotUndef;
10559 for (
auto E = ShuffleMask.end(); VMIt != E; ++VMIt) {
10561 std::pair<T *, bool> Res =
10562 ResizeAction(VMIt->first, VMIt->second,
false);
10564 for (
unsigned I = 0, VF = Mask.size();
I < VF; ++
I) {
10567 "Multiple uses of scalars.");
10568 Mask[
I] = (Res.second ?
I : SecMask[
I]) + VF;
10573 Prev = Action(Mask, {Prev, Res.first});
10581 << VectorizableTree.size() <<
".\n");
10583 unsigned BundleWidth = VectorizableTree[0]->Scalars.size();
10586 for (
unsigned I = 0, E = VectorizableTree.size();
I < E; ++
I) {
10587 TreeEntry &TE = *VectorizableTree[
I];
10588 if (TE.isGather()) {
10589 if (
const TreeEntry *E = getTreeEntry(TE.getMainOp());
10590 E && E->getVectorFactor() == TE.getVectorFactor() &&
10591 E->isSame(TE.Scalars)) {
10596 <<
"SLP: Current total cost = " <<
Cost <<
"\n");
10605 <<
"SLP: Current total cost = " <<
Cost <<
"\n");
10615 std::optional<DenseMap<Value *, unsigned>> ValueToExtUses;
10616 for (ExternalUser &EU : ExternalUses) {
10618 if (!isa_and_nonnull<InsertElementInst>(EU.User) &&
10619 !ExtractCostCalculated.
insert(EU.Scalar).second)
10625 if (EphValues.
count(EU.User))
10629 if (isa<FixedVectorType>(EU.Scalar->getType()))
10634 if (
auto *VU = dyn_cast_or_null<InsertElementInst>(EU.User);
10636 if (
auto *FTy = dyn_cast<FixedVectorType>(VU->
getType())) {
10637 if (!UsedInserts.
insert(VU).second)
10641 const TreeEntry *ScalarTE = getTreeEntry(EU.Scalar);
10644 [
this, VU](
const std::pair<Value *, const TreeEntry *> &Pair) {
10646 VU, cast<InsertElementInst>(Pair.first),
10648 Value *Op0 = II->getOperand(0);
10649 if (getTreeEntry(II) && !getTreeEntry(Op0))
10655 if (It == FirstUsers.
end()) {
10662 while (
auto *IEBase = dyn_cast<InsertElementInst>(
Base)) {
10663 if (IEBase != EU.User &&
10664 (!IEBase->hasOneUse() ||
10668 if (
const TreeEntry *E = getTreeEntry(IEBase)) {
10671 IEBase = cast<InsertElementInst>(
Base);
10674 "InsertElementInstruction used already.");
10676 Base = IEBase->getOperand(0);
10677 }
while (E == getTreeEntry(
Base));
10680 Base = cast<InsertElementInst>(
Base)->getOperand(0);
10684 VecId = FirstUsers.
size() - 1;
10685 auto It = MinBWs.
find(ScalarTE);
10686 if (It != MinBWs.
end() &&
10688 .
insert(std::make_pair(ScalarTE, FTy->getElementType()))
10690 unsigned BWSz = It->second.first;
10691 unsigned DstBWSz =
DL->getTypeSizeInBits(FTy->getElementType());
10692 unsigned VecOpcode;
10693 if (DstBWSz < BWSz)
10694 VecOpcode = Instruction::Trunc;
10697 It->second.second ? Instruction::SExt : Instruction::ZExt;
10702 FTy->getNumElements()),
10705 <<
" for extending externally used vector with "
10706 "non-equal minimum bitwidth.\n");
10712 VecId = std::distance(FirstUsers.
begin(), It);
10714 int InIdx = *InsertIdx;
10718 Mask[InIdx] = EU.Lane;
10719 DemandedElts[VecId].setBit(InIdx);
10727 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(EU.Scalar)) {
10728 if (!ValueToExtUses) {
10729 ValueToExtUses.emplace();
10731 ValueToExtUses->try_emplace(
P.value().Scalar,
P.index());
10737 if (!getTreeEntry(V))
10739 auto It = ValueToExtUses->find(V);
10740 if (It != ValueToExtUses->end()) {
10742 ExternalUses[It->second].User = nullptr;
10747 if (CanBeUsedAsGEP) {
10749 ExternalUsesAsGEPs.
insert(EU.Scalar);
10757 auto *VecTy =
getWidenedType(EU.Scalar->getType(), BundleWidth);
10758 auto It = MinBWs.
find(getTreeEntry(EU.Scalar));
10759 if (It != MinBWs.
end()) {
10762 It->second.second ? Instruction::SExt : Instruction::ZExt;
10772 if (!VectorizedVals.
empty()) {
10773 const TreeEntry &Root = *VectorizableTree.front();
10774 auto BWIt = MinBWs.find(&Root);
10775 if (BWIt != MinBWs.end()) {
10776 Type *DstTy = Root.Scalars.front()->getType();
10777 unsigned OriginalSz =
DL->getTypeSizeInBits(DstTy);
10779 ReductionBitWidth == 0 ? BWIt->second.first : ReductionBitWidth;
10780 if (OriginalSz != SrcSz) {
10781 unsigned Opcode = Instruction::Trunc;
10782 if (OriginalSz > SrcSz)
10783 Opcode = BWIt->second.second ? Instruction::SExt : Instruction::ZExt;
10793 Cost += SpillCost + ExtractCost;
10797 unsigned VF =
Mask.size();
10798 unsigned VecVF =
TE->getVectorFactor();
10800 (
any_of(Mask, [VF](
int Idx) {
return Idx >=
static_cast<int>(VF); }) ||
10803 std::copy(
Mask.begin(), std::next(
Mask.begin(), std::min(VF, VecVF)),
10809 dbgs() <<
"SLP: Adding cost " <<
C
10810 <<
" for final shuffle of insertelement external users.\n";
10811 TE->dump();
dbgs() <<
"SLP: Current total cost = " <<
Cost <<
"\n");
10813 return std::make_pair(TE,
true);
10815 return std::make_pair(TE,
false);
10818 for (
int I = 0, E = FirstUsers.size();
I < E; ++
I) {
10819 Value *
Base = cast<Instruction>(FirstUsers[
I].first)->getOperand(0);
10820 auto Vector = ShuffleMasks[
I].takeVector();
10824 assert((TEs.size() == 1 || TEs.size() == 2) &&
10825 "Expected exactly 1 or 2 tree entries.");
10826 if (TEs.size() == 1) {
10828 VF = TEs.front()->getVectorFactor();
10829 auto *FTy =
getWidenedType(TEs.back()->Scalars.front()->getType(), VF);
10833 (
Data.index() < VF &&
10834 static_cast<int>(
Data.index()) ==
Data.value());
10839 <<
" for final shuffle of insertelement "
10840 "external users.\n";
10841 TEs.front()->
dump();
10842 dbgs() <<
"SLP: Current total cost = " <<
Cost <<
"\n");
10848 TEs.front()->getVectorFactor() == TEs.back()->getVectorFactor())
10849 VF = TEs.front()->getVectorFactor();
10853 auto *FTy =
getWidenedType(TEs.back()->Scalars.front()->getType(), VF);
10857 <<
" for final shuffle of vector node and external "
10858 "insertelement users.\n";
10859 if (TEs.front()) { TEs.front()->dump(); } TEs.back()->dump();
10860 dbgs() <<
"SLP: Current total cost = " <<
Cost <<
"\n");
10866 (void)performExtractsShuffleAction<const TreeEntry>(
10868 [](
const TreeEntry *E) {
return E->getVectorFactor(); }, ResizeToVF,
10869 EstimateShufflesCost);
10871 cast<FixedVectorType>(FirstUsers[
I].first->getType()), DemandedElts[
I],
10873 Cost -= InsertCost;
10877 if (ReductionBitWidth != 0) {
10878 assert(UserIgnoreList &&
"Expected reduction tree.");
10879 const TreeEntry &E = *VectorizableTree.front();
10880 auto It = MinBWs.find(&E);
10881 if (It != MinBWs.end() && It->second.first != ReductionBitWidth) {
10882 unsigned SrcSize = It->second.first;
10883 unsigned DstSize = ReductionBitWidth;
10884 unsigned Opcode = Instruction::Trunc;
10885 if (SrcSize < DstSize)
10886 Opcode = It->second.second ? Instruction::SExt : Instruction::ZExt;
10888 getWidenedType(Builder.getIntNTy(SrcSize), E.getVectorFactor());
10890 getWidenedType(Builder.getIntNTy(DstSize), E.getVectorFactor());
10893 switch (E.getOpcode()) {
10894 case Instruction::SExt:
10895 case Instruction::ZExt:
10896 case Instruction::Trunc: {
10897 const TreeEntry *OpTE = getOperandEntry(&E, 0);
10898 CCH = getCastContextHint(*OpTE);
10908 <<
" for final resize for reduction from " << SrcVecTy
10909 <<
" to " << DstVecTy <<
"\n";
10910 dbgs() <<
"SLP: Current total cost = " <<
Cost <<
"\n");
10918 OS <<
"SLP: Spill Cost = " << SpillCost <<
".\n"
10919 <<
"SLP: Extract Cost = " << ExtractCost <<
".\n"
10920 <<
"SLP: Total Cost = " <<
Cost <<
".\n";
10924 ViewGraph(
this,
"SLP" +
F->getName(),
false, Str);
10935std::optional<TTI::ShuffleKind>
10936BoUpSLP::tryToGatherSingleRegisterExtractElements(
10942 for (
int I = 0, E = VL.
size();
I < E; ++
I) {
10943 auto *EI = dyn_cast<ExtractElementInst>(VL[
I]);
10945 if (isa<UndefValue>(VL[
I]))
10949 auto *VecTy = dyn_cast<FixedVectorType>(EI->getVectorOperandType());
10950 if (!VecTy || !isa<ConstantInt, UndefValue>(EI->getIndexOperand()))
10959 ExtractMask.reset(*
Idx);
10964 VectorOpToIdx[EI->getVectorOperand()].push_back(
I);
10969 stable_sort(Vectors, [](
const auto &P1,
const auto &P2) {
10970 return P1.second.size() > P2.second.size();
10973 const int UndefSz = UndefVectorExtracts.
size();
10974 unsigned SingleMax = 0;
10975 unsigned PairMax = 0;
10976 if (!Vectors.
empty()) {
10977 SingleMax = Vectors.
front().second.size() + UndefSz;
10978 if (Vectors.
size() > 1) {
10979 auto *ItNext = std::next(Vectors.
begin());
10980 PairMax = SingleMax + ItNext->second.size();
10983 if (SingleMax == 0 && PairMax == 0 && UndefSz == 0)
10984 return std::nullopt;
10990 if (SingleMax >= PairMax && SingleMax) {
10991 for (
int Idx : Vectors.
front().second)
10993 }
else if (!Vectors.
empty()) {
10994 for (
unsigned Idx : {0, 1})
10995 for (
int Idx : Vectors[
Idx].second)
10999 for (
int Idx : UndefVectorExtracts)
11003 std::optional<TTI::ShuffleKind> Res =
11009 return std::nullopt;
11013 for (
int I = 0, E = GatheredExtracts.size();
I < E; ++
I) {
11014 if (Mask[
I] ==
PoisonMaskElem && !isa<PoisonValue>(GatheredExtracts[
I]) &&
11015 isa<UndefValue>(GatheredExtracts[
I])) {
11019 auto *EI = dyn_cast<ExtractElementInst>(VL[
I]);
11020 if (!EI || !isa<FixedVectorType>(EI->getVectorOperandType()) ||
11021 !isa<ConstantInt, UndefValue>(EI->getIndexOperand()) ||
11036 unsigned NumParts)
const {
11037 assert(NumParts > 0 &&
"NumParts expected be greater than or equal to 1.");
11041 for (
unsigned Part : seq<unsigned>(NumParts)) {
11047 std::optional<TTI::ShuffleKind> Res =
11048 tryToGatherSingleRegisterExtractElements(SubVL, SubMask);
11049 ShufflesRes[Part] = Res;
11050 copy(SubMask, std::next(
Mask.begin(), Part * SliceSize));
11052 if (
none_of(ShufflesRes, [](
const std::optional<TTI::ShuffleKind> &Res) {
11053 return Res.has_value();
11055 ShufflesRes.clear();
11056 return ShufflesRes;
11059std::optional<TargetTransformInfo::ShuffleKind>
11060BoUpSLP::isGatherShuffledSingleRegisterEntry(
11066 const EdgeInfo &TEUseEI =
TE->UserTreeIndices.front();
11067 const Instruction *TEInsertPt = &getLastInstructionInBundle(TEUseEI.UserTE);
11071 if (
auto *
PHI = dyn_cast<PHINode>(TEUseEI.UserTE->getMainOp())) {
11072 TEInsertBlock =
PHI->getIncomingBlock(TEUseEI.EdgeIdx);
11075 TEInsertBlock = TEInsertPt->
getParent();
11078 return std::nullopt;
11079 auto *NodeUI = DT->
getNode(TEInsertBlock);
11080 assert(NodeUI &&
"Should only process reachable instructions");
11082 auto CheckOrdering = [&](
const Instruction *InsertPt) {
11096 auto *NodeEUI = DT->
getNode(InsertBlock);
11099 assert((NodeUI == NodeEUI) ==
11100 (NodeUI->getDFSNumIn() == NodeEUI->getDFSNumIn()) &&
11101 "Different nodes should have different DFS numbers");
11103 if (TEInsertPt->
getParent() != InsertBlock &&
11106 if (TEInsertPt->
getParent() == InsertBlock &&
11120 for (
Value *V : VL) {
11125 for (
const TreeEntry *TEPtr : ValueToGatherNodes.find(V)->second) {
11129 [&](
Value *V) { return GatheredScalars.contains(V); }) &&
11130 "Must contain at least single gathered value.");
11131 assert(TEPtr->UserTreeIndices.size() == 1 &&
11132 "Expected only single user of a gather node.");
11133 const EdgeInfo &UseEI = TEPtr->UserTreeIndices.front();
11135 PHINode *UserPHI = dyn_cast<PHINode>(UseEI.UserTE->getMainOp());
11138 : &getLastInstructionInBundle(UseEI.UserTE);
11139 if (TEInsertPt == InsertPt) {
11143 if (TEUseEI.UserTE == UseEI.UserTE && TEUseEI.EdgeIdx < UseEI.EdgeIdx)
11147 if (TEUseEI.UserTE != UseEI.UserTE &&
11148 TEUseEI.UserTE->Idx < UseEI.UserTE->Idx)
11154 if ((TEInsertBlock != InsertPt->
getParent() ||
11155 TEUseEI.EdgeIdx < UseEI.EdgeIdx || TEUseEI.UserTE != UseEI.UserTE) &&
11156 !CheckOrdering(InsertPt))
11160 if (
const TreeEntry *VTE = getTreeEntry(V)) {
11162 if (VTE->State != TreeEntry::Vectorize) {
11163 auto It = MultiNodeScalars.
find(V);
11164 if (It == MultiNodeScalars.
end())
11166 VTE = *It->getSecond().begin();
11168 auto *MIt =
find_if(It->getSecond(), [](
const TreeEntry *MTE) {
11169 return MTE->State == TreeEntry::Vectorize;
11171 if (MIt == It->getSecond().end())
11176 Instruction &LastBundleInst = getLastInstructionInBundle(VTE);
11177 if (&LastBundleInst == TEInsertPt || !CheckOrdering(&LastBundleInst))
11181 if (VToTEs.
empty())
11183 if (UsedTEs.
empty()) {
11197 if (!VToTEs.
empty()) {
11203 VToTEs = SavedVToTEs;
11212 if (UsedTEs.
size() == 2)
11214 UsedTEs.push_back(SavedVToTEs);
11221 if (UsedTEs.
empty()) {
11223 return std::nullopt;
11227 if (UsedTEs.
size() == 1) {
11230 UsedTEs.front().
end());
11231 sort(FirstEntries, [](
const TreeEntry *TE1,
const TreeEntry *TE2) {
11232 return TE1->Idx < TE2->Idx;
11235 auto *It =
find_if(FirstEntries, [=](
const TreeEntry *EntryPtr) {
11236 return EntryPtr->isSame(VL) || EntryPtr->isSame(
TE->Scalars);
11238 if (It != FirstEntries.end() &&
11239 ((*It)->getVectorFactor() == VL.size() ||
11240 ((*It)->getVectorFactor() ==
TE->Scalars.size() &&
11241 TE->ReuseShuffleIndices.size() == VL.size() &&
11242 (*It)->isSame(
TE->Scalars)))) {
11243 Entries.push_back(*It);
11244 if ((*It)->getVectorFactor() == VL.size()) {
11245 std::iota(std::next(
Mask.begin(), Part * VL.size()),
11246 std::next(
Mask.begin(), (Part + 1) * VL.size()), 0);
11252 for (
int I = 0, Sz = VL.size();
I < Sz; ++
I)
11253 if (isa<PoisonValue>(VL[
I]))
11259 Entries.push_back(FirstEntries.front());
11262 assert(UsedTEs.
size() == 2 &&
"Expected at max 2 permuted entries.");
11265 for (
const TreeEntry *TE : UsedTEs.front()) {
11266 unsigned VF =
TE->getVectorFactor();
11267 auto It = VFToTE.
find(VF);
11268 if (It != VFToTE.
end()) {
11269 if (It->second->Idx >
TE->Idx)
11270 It->getSecond() =
TE;
11277 UsedTEs.back().
end());
11278 sort(SecondEntries, [](
const TreeEntry *TE1,
const TreeEntry *TE2) {
11279 return TE1->Idx < TE2->Idx;
11281 for (
const TreeEntry *TE : SecondEntries) {
11282 auto It = VFToTE.
find(
TE->getVectorFactor());
11283 if (It != VFToTE.
end()) {
11285 Entries.push_back(It->second);
11286 Entries.push_back(TE);
11292 if (Entries.empty()) {
11294 UsedTEs.front(), [](
const TreeEntry *TE1,
const TreeEntry *TE2) {
11295 return TE1->Idx < TE2->Idx;
11297 Entries.push_back(SecondEntries.front());
11298 VF = std::max(Entries.front()->getVectorFactor(),
11299 Entries.back()->getVectorFactor());
11303 bool IsSplatOrUndefs =
isSplat(VL) ||
all_of(VL, IsaPred<UndefValue>);
11306 auto AreCompatiblePHIs = [&](
Value *
V,
Value *V1) {
11307 auto *
PHI = cast<PHINode>(V);
11308 auto *PHI1 = cast<PHINode>(V1);
11313 for (
int I = 0, E =
PHI->getNumIncomingValues();
I < E; ++
I) {
11315 Value *In1 = PHI1->getIncomingValue(
I);
11320 if (cast<Instruction>(In)->
getParent() !=
11330 auto MightBeIgnored = [=](
Value *
V) {
11331 auto *
I = dyn_cast<Instruction>(V);
11332 return I && !IsSplatOrUndefs && !ScalarToTreeEntry.
count(
I) &&
11334 !areAllUsersVectorized(
I, UserIgnoreList) &&
isSimple(
I);
11339 auto NeighborMightBeIgnored = [&](
Value *
V,
int Idx) {
11341 bool UsedInSameVTE =
false;
11342 auto It = UsedValuesEntry.
find(V1);
11343 if (It != UsedValuesEntry.
end())
11344 UsedInSameVTE = It->second == UsedValuesEntry.
find(V)->second;
11345 return V != V1 && MightBeIgnored(V1) && !UsedInSameVTE &&
11347 cast<Instruction>(V)->getParent() ==
11348 cast<Instruction>(V1)->getParent() &&
11349 (!isa<PHINode>(V1) || AreCompatiblePHIs(V, V1));
11354 for (
int I = 0, E = VL.size();
I < E; ++
I) {
11356 auto It = UsedValuesEntry.
find(V);
11357 if (It == UsedValuesEntry.
end())
11363 ((
I > 0 && NeighborMightBeIgnored(V,
I - 1)) ||
11364 (
I != E - 1 && NeighborMightBeIgnored(V,
I + 1)))))
11366 unsigned Idx = It->second;
11373 for (
unsigned I = 0, Sz = Entries.size();
I < Sz; ++
I) {
11374 if (!UsedIdxs.test(
I))
11380 for (std::pair<unsigned, int> &Pair : EntryLanes)
11381 if (Pair.first ==
I)
11382 Pair.first = TempEntries.
size();
11385 Entries.swap(TempEntries);
11386 if (EntryLanes.size() == Entries.size() &&
11388 .
slice(Part * VL.size(),
11389 std::min<int>(VL.size(),
TE->Scalars.size())))) {
11395 return std::nullopt;
11398 bool IsIdentity = Entries.size() == 1;
11401 for (
const std::pair<unsigned, int> &Pair : EntryLanes) {
11402 unsigned Idx = Part * VL.size() + Pair.second;
11405 (ForOrder ? std::distance(
11406 Entries[Pair.first]->Scalars.begin(),
11407 find(Entries[Pair.first]->Scalars, VL[Pair.second]))
11408 : Entries[Pair.first]->findLaneForValue(VL[Pair.second]));
11409 IsIdentity &=
Mask[
Idx] == Pair.second;
11411 switch (Entries.size()) {
11413 if (IsIdentity || EntryLanes.size() > 1 || VL.size() <= 2)
11417 if (EntryLanes.size() > 2 || VL.size() <= 2)
11425 std::fill(std::next(
Mask.begin(), Part * VL.size()),
11427 return std::nullopt;
11431BoUpSLP::isGatherShuffledEntry(
11435 assert(NumParts > 0 && NumParts < VL.
size() &&
11436 "Expected positive number of registers.");
11439 if (TE == VectorizableTree.front().get())
11442 if (
TE->isNonPowOf2Vec())
11445 assert(
TE->UserTreeIndices.size() == 1 &&
11446 "Expected only single user of the gather node.");
11448 "Number of scalars must be divisible by NumParts.");
11451 for (
unsigned Part : seq<unsigned>(NumParts)) {
11455 std::optional<TTI::ShuffleKind> SubRes =
11456 isGatherShuffledSingleRegisterEntry(TE, SubVL, Mask, SubEntries, Part,
11459 SubEntries.
clear();
11462 SubEntries.
front()->getVectorFactor() == VL.
size() &&
11463 (SubEntries.
front()->isSame(
TE->Scalars) ||
11464 SubEntries.
front()->isSame(VL))) {
11466 LocalSubEntries.
swap(SubEntries);
11469 std::iota(
Mask.begin(),
Mask.end(), 0);
11471 for (
int I = 0, Sz = VL.
size();
I < Sz; ++
I)
11472 if (isa<PoisonValue>(VL[
I]))
11474 Entries.emplace_back(1, LocalSubEntries.
front());
11480 [](
const std::optional<TTI::ShuffleKind> &SK) {
return !SK; })) {
11488 Type *ScalarTy)
const {
11490 bool DuplicateNonConst =
false;
11498 auto EstimateInsertCost = [&](
unsigned I,
Value *
V) {
11499 if (
V->getType() != ScalarTy) {
11510 for (
unsigned I = 0, E = VL.
size();
I < E; ++
I) {
11513 if ((ForPoisonSrc &&
isConstant(V)) || isa<UndefValue>(V)) {
11521 EstimateInsertCost(
I, V);
11522 ShuffleMask[
I] =
I;
11526 DuplicateNonConst =
true;
11528 ShuffleMask[
I] = Res.first->second;
11534 if (DuplicateNonConst)
11536 VecTy, ShuffleMask);
11548 VLOperands Ops(VL, R);
11551 Left = Ops.getVL(0);
11552 Right = Ops.getVL(1);
11555Instruction &BoUpSLP::getLastInstructionInBundle(
const TreeEntry *E) {
11558 return *Res.second;
11562 auto *Front = E->getMainOp();
11565 if (E->getOpcode() == Instruction::GetElementPtr &&
11566 !isa<GetElementPtrInst>(V))
11568 auto *I = cast<Instruction>(V);
11569 return !E->isOpcodeOrAlt(I) || I->getParent() == BB ||
11570 isVectorLikeInstWithConstOps(I);
11573 auto FindLastInst = [&]() {
11575 for (
Value *V : E->Scalars) {
11576 auto *
I = dyn_cast<Instruction>(V);
11579 if (LastInst->
getParent() ==
I->getParent()) {
11584 assert(((E->getOpcode() == Instruction::GetElementPtr &&
11585 !isa<GetElementPtrInst>(
I)) ||
11588 "Expected vector-like or non-GEP in GEP node insts only.");
11596 auto *NodeB = DT->
getNode(
I->getParent());
11597 assert(NodeA &&
"Should only process reachable instructions");
11598 assert(NodeB &&
"Should only process reachable instructions");
11599 assert((NodeA == NodeB) ==
11600 (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
11601 "Different nodes should have different DFS numbers");
11602 if (NodeA->getDFSNumIn() < NodeB->getDFSNumIn())
11609 auto FindFirstInst = [&]() {
11611 for (
Value *V : E->Scalars) {
11612 auto *
I = dyn_cast<Instruction>(V);
11615 if (FirstInst->
getParent() ==
I->getParent()) {
11616 if (
I->comesBefore(FirstInst))
11620 assert(((E->getOpcode() == Instruction::GetElementPtr &&
11621 !isa<GetElementPtrInst>(
I)) ||
11624 "Expected vector-like or non-GEP in GEP node insts only.");
11632 auto *NodeB = DT->
getNode(
I->getParent());
11633 assert(NodeA &&
"Should only process reachable instructions");
11634 assert(NodeB &&
"Should only process reachable instructions");
11635 assert((NodeA == NodeB) ==
11636 (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
11637 "Different nodes should have different DFS numbers");
11638 if (NodeA->getDFSNumIn() > NodeB->getDFSNumIn())
11648 if ((E->getOpcode() == Instruction::GetElementPtr &&
11651 return !isa<GetElementPtrInst>(V) && isa<Instruction>(V);
11655 return !isVectorLikeInstWithConstOps(V) &&
11656 isUsedOutsideBlock(V);
11658 (E->isGather() && E->Idx == 0 &&
all_of(E->Scalars, [](
Value *V) {
11659 return isa<ExtractElementInst, UndefValue>(V) ||
11660 areAllOperandsNonInsts(V);
11662 Res.second = FindLastInst();
11664 Res.second = FindFirstInst();
11665 return *Res.second;
11672 if (BlocksSchedules.count(BB)) {
11673 Value *
V = E->isOneOf(E->Scalars.back());
11676 auto *Bundle = BlocksSchedules[BB]->getScheduleData(V);
11677 if (Bundle && Bundle->isPartOfBundle())
11678 for (; Bundle; Bundle = Bundle->NextInBundle)
11679 if (Bundle->OpValue == Bundle->Inst)
11680 Res.second = Bundle->Inst;
11702 Res.second = FindLastInst();
11703 assert(Res.second &&
"Failed to find last instruction in bundle");
11704 return *Res.second;
11707void BoUpSLP::setInsertPointAfterBundle(
const TreeEntry *E) {
11708 auto *Front = E->getMainOp();
11709 Instruction *LastInst = &getLastInstructionInBundle(E);
11710 assert(LastInst &&
"Failed to find last instruction in bundle");
11713 bool IsPHI = isa<PHINode>(LastInst);
11715 LastInstIt = LastInst->
getParent()->getFirstNonPHIIt();
11717 Builder.SetInsertPoint(LastInst->
getParent(), LastInstIt);
11721 Builder.SetInsertPoint(
11725 Builder.SetCurrentDebugLocation(Front->getDebugLoc());
11735 Loop *
L = LI->getLoopFor(Builder.GetInsertBlock());
11738 while (InsertBB && InsertBB != InstBB && Visited.
insert(InsertBB).second)
11739 InsertBB = InsertBB->getSinglePredecessor();
11740 return InsertBB && InsertBB == InstBB;
11742 for (
int I = 0, E = VL.
size();
I < E; ++
I) {
11743 if (
auto *Inst = dyn_cast<Instruction>(VL[
I]))
11744 if ((CheckPredecessor(Inst->getParent(), Builder.GetInsertBlock()) ||
11745 getTreeEntry(Inst) ||
11746 (L && (!Root ||
L->isLoopInvariant(Root)) &&
L->contains(Inst))) &&
11747 PostponedIndices.
insert(
I).second)
11751 auto &&CreateInsertElement = [
this](
Value *Vec,
Value *
V,
unsigned Pos,
11754 if (
Scalar->getType() != Ty) {
11758 if (
auto *CI = dyn_cast<CastInst>(Scalar);
11759 isa_and_nonnull<SExtInst, ZExtInst>(CI)) {
11761 if (
auto *IOp = dyn_cast<Instruction>(
Op);
11762 !IOp || !(
isDeleted(IOp) || getTreeEntry(IOp)))
11765 Scalar = Builder.CreateIntCast(
11770 if (
auto *VecTy = dyn_cast<FixedVectorType>(
Scalar->getType())) {
11772 Vec = InsElt = Builder.CreateInsertVector(
11775 auto *
II = dyn_cast<IntrinsicInst>(InsElt);
11776 if (!
II ||
II->getIntrinsicID() != Intrinsic::vector_insert)
11779 Vec = Builder.CreateInsertElement(Vec, Scalar, Builder.getInt32(Pos));
11780 InsElt = dyn_cast<InsertElementInst>(Vec);
11784 GatherShuffleExtractSeq.
insert(InsElt);
11787 if (isa<Instruction>(V)) {
11788 if (TreeEntry *Entry = getTreeEntry(V)) {
11790 User *UserOp =
nullptr;
11792 if (
auto *SI = dyn_cast<Instruction>(Scalar))
11798 unsigned FoundLane =
Entry->findLaneForValue(V);
11799 ExternalUses.emplace_back(V, UserOp, FoundLane);
11809 for (
int I = 0, E = VL.
size();
I < E; ++
I) {
11817 if (!isa<UndefValue>(VL[
I])) {
11821 if (isa<PoisonValue>(VL[
I]))
11823 if (
auto *SV = dyn_cast<ShuffleVectorInst>(Root)) {
11828 Vec = CreateInsertElement(Vec, VL[
I],
I, ScalarTy);
11831 for (
int I : NonConsts)
11832 Vec = CreateInsertElement(Vec, VL[
I],
I, ScalarTy);
11835 for (
const std::pair<Value *, unsigned> &Pair : PostponedInsts)
11836 Vec = CreateInsertElement(Vec, Pair.first, Pair.second, ScalarTy);
11874 bool IsFinalized =
false;
11887 class ShuffleIRBuilder {
11900 : Builder(Builder), GatherShuffleExtractSeq(GatherShuffleExtractSeq),
11901 CSEBlocks(CSEBlocks),
DL(
DL) {}
11902 ~ShuffleIRBuilder() =
default;
11905 if (V1->
getType() != V2->getType()) {
11908 "Expected integer vector types only.");
11909 if (V1->
getType() != V2->getType()) {
11910 if (cast<VectorType>(V2->getType())
11912 ->getIntegerBitWidth() < cast<VectorType>(V1->
getType())
11914 ->getIntegerBitWidth())
11923 if (
auto *
I = dyn_cast<Instruction>(Vec)) {
11924 GatherShuffleExtractSeq.
insert(
I);
11925 CSEBlocks.
insert(
I->getParent());
11934 unsigned VF = Mask.size();
11935 unsigned LocalVF = cast<FixedVectorType>(V1->
getType())->getNumElements();
11939 if (
auto *
I = dyn_cast<Instruction>(Vec)) {
11940 GatherShuffleExtractSeq.
insert(
I);
11941 CSEBlocks.
insert(
I->getParent());
11945 Value *createIdentity(
Value *V) {
return V; }
11946 Value *createPoison(
Type *Ty,
unsigned VF) {
11951 void resizeToMatch(
Value *&V1,
Value *&V2) {
11952 if (V1->
getType() == V2->getType())
11954 int V1VF = cast<FixedVectorType>(V1->
getType())->getNumElements();
11955 int V2VF = cast<FixedVectorType>(V2->getType())->getNumElements();
11956 int VF = std::max(V1VF, V2VF);
11957 int MinVF = std::min(V1VF, V2VF);
11959 std::iota(IdentityMask.
begin(), std::next(IdentityMask.
begin(), MinVF),
11961 Value *&
Op = MinVF == V1VF ? V1 : V2;
11963 if (
auto *
I = dyn_cast<Instruction>(
Op)) {
11964 GatherShuffleExtractSeq.
insert(
I);
11965 CSEBlocks.
insert(
I->getParent());
11978 assert(V1 &&
"Expected at least one vector value.");
11979 ShuffleIRBuilder ShuffleBuilder(Builder, R.GatherShuffleExtractSeq,
11980 R.CSEBlocks, *R.DL);
11981 return BaseShuffleAnalysis::createShuffle<Value *>(V1, V2, Mask,
11989 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
11997 std::optional<bool> IsSigned = std::nullopt) {
11998 auto *VecTy = cast<VectorType>(V->getType());
12009 : BaseShuffleAnalysis(ScalarTy), Builder(Builder), R(R) {}
12013 ArrayRef<std::optional<TTI::ShuffleKind>> ShuffleKinds,
12014 unsigned NumParts,
bool &UseVecBaseAsInput) {
12015 UseVecBaseAsInput =
false;
12017 Value *VecBase =
nullptr;
12018 for (
int I = 0, Sz = Mask.size();
I < Sz; ++
I) {
12022 auto *EI = cast<ExtractElementInst>(E->Scalars[
I]);
12023 VecBase = EI->getVectorOperand();
12024 if (
const TreeEntry *TE = R.getTreeEntry(VecBase))
12025 VecBase = TE->VectorizedValue;
12026 assert(VecBase &&
"Expected vectorized value.");
12027 UniqueBases.
insert(VecBase);
12030 if (!EI->hasOneUse() || (NumParts != 1 &&
count(E->Scalars, EI) > 1) ||
12032 const TreeEntry *UTE = R.getTreeEntry(U);
12033 return !UTE || R.MultiNodeScalars.contains(U) ||
12034 (isa<GetElementPtrInst>(U) &&
12035 !R.areAllUsersVectorized(cast<Instruction>(U))) ||
12036 count_if(R.VectorizableTree,
12037 [&](const std::unique_ptr<TreeEntry> &TE) {
12038 return any_of(TE->UserTreeIndices,
12039 [&](const EdgeInfo &Edge) {
12040 return Edge.UserTE == UTE;
12042 is_contained(TE->Scalars, EI);
12046 R.eraseInstruction(EI);
12048 if (NumParts == 1 || UniqueBases.
size() == 1) {
12049 assert(VecBase &&
"Expected vectorized value.");
12050 return castToScalarTyElem(VecBase);
12052 UseVecBaseAsInput =
true;
12062 Value *Vec =
nullptr;
12065 for (
unsigned Part : seq<unsigned>(NumParts)) {
12066 unsigned Limit =
getNumElems(E->Scalars.size(), SliceSize, Part);
12070 constexpr int MaxBases = 2;
12072 auto VLMask =
zip(VL, SubMask);
12073 const unsigned VF = std::accumulate(
12074 VLMask.begin(), VLMask.end(), 0U, [&](
unsigned S,
const auto &
D) {
12075 if (std::get<1>(D) == PoisonMaskElem)
12078 cast<ExtractElementInst>(std::get<0>(D))->getVectorOperand();
12079 if (const TreeEntry *TE = R.getTreeEntry(VecOp))
12080 VecOp = TE->VectorizedValue;
12081 assert(VecOp &&
"Expected vectorized value.");
12082 const unsigned Size =
12083 cast<FixedVectorType>(VecOp->getType())->getNumElements();
12084 return std::max(S, Size);
12086 for (
const auto [V,
I] : VLMask) {
12089 Value *VecOp = cast<ExtractElementInst>(V)->getVectorOperand();
12090 if (
const TreeEntry *TE = R.getTreeEntry(VecOp))
12091 VecOp = TE->VectorizedValue;
12092 assert(VecOp &&
"Expected vectorized value.");
12093 VecOp = castToScalarTyElem(VecOp);
12094 Bases[
I / VF] = VecOp;
12096 if (!Bases.front())
12099 if (Bases.back()) {
12100 SubVec = createShuffle(Bases.front(), Bases.back(), SubMask);
12101 TransformToIdentity(SubMask);
12103 SubVec = Bases.front();
12110 Mask.slice(
P * SliceSize,
12117 "Expected first part or all previous parts masked.");
12118 copy(SubMask, std::next(VecMask.begin(), Part * SliceSize));
12121 cast<FixedVectorType>(Vec->
getType())->getNumElements();
12123 unsigned SubVecVF =
12124 cast<FixedVectorType>(SubVec->
getType())->getNumElements();
12125 NewVF = std::max(NewVF, SubVecVF);
12128 for (
int &
Idx : SubMask)
12131 copy(SubMask, std::next(VecMask.begin(), Part * SliceSize));
12132 Vec = createShuffle(Vec, SubVec, VecMask);
12133 TransformToIdentity(VecMask);
12141 std::optional<Value *>
12147 TEs, [](
const TreeEntry *TE) {
return TE->VectorizedValue; });
12149 return std::nullopt;
12152 auto *ResVecTy =
getWidenedType(ScalarTy, E->getVectorFactor());
12161 Value *V1 = E1.VectorizedValue;
12163 V1 = castToScalarTyElem(V1,
any_of(E1.Scalars, [&](
Value *V) {
12164 return !isKnownNonNegative(
12165 V, SimplifyQuery(*R.DL));
12167 Value *V2 = E2.VectorizedValue;
12168 if (V2->getType()->isIntOrIntVectorTy())
12169 V2 = castToScalarTyElem(V2,
any_of(E2.Scalars, [&](
Value *V) {
12170 return !isKnownNonNegative(
12171 V, SimplifyQuery(*R.DL));
12178 Value *V1 = E1.VectorizedValue;
12180 V1 = castToScalarTyElem(V1,
any_of(E1.Scalars, [&](
Value *V) {
12181 return !isKnownNonNegative(
12182 V, SimplifyQuery(*R.DL));
12188 assert(V1 && V2 && !Mask.empty() &&
"Expected non-empty input vectors.");
12190 isa<FixedVectorType>(V2->getType()) &&
12191 "castToScalarTyElem expects V1 and V2 to be FixedVectorType");
12192 V1 = castToScalarTyElem(V1);
12193 V2 = castToScalarTyElem(V2);
12194 if (InVectors.
empty()) {
12197 CommonMask.
assign(Mask.begin(), Mask.end());
12201 if (InVectors.
size() == 2) {
12202 Vec = createShuffle(Vec, InVectors.
back(), CommonMask);
12203 transformMaskAfterShuffle(CommonMask, CommonMask);
12204 }
else if (cast<FixedVectorType>(Vec->
getType())->getNumElements() !=
12206 Vec = createShuffle(Vec,
nullptr, CommonMask);
12207 transformMaskAfterShuffle(CommonMask, CommonMask);
12209 V1 = createShuffle(V1, V2, Mask);
12210 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
12212 CommonMask[
Idx] =
Idx + Sz;
12213 InVectors.
front() = Vec;
12214 if (InVectors.
size() == 2)
12215 InVectors.
back() = V1;
12222 "castToScalarTyElem expects V1 to be FixedVectorType");
12223 V1 = castToScalarTyElem(V1);
12224 if (InVectors.
empty()) {
12226 CommonMask.
assign(Mask.begin(), Mask.end());
12229 const auto *It =
find(InVectors, V1);
12230 if (It == InVectors.
end()) {
12231 if (InVectors.
size() == 2 ||
12234 if (InVectors.
size() == 2) {
12235 V = createShuffle(InVectors.
front(), InVectors.
back(), CommonMask);
12236 transformMaskAfterShuffle(CommonMask, CommonMask);
12237 }
else if (cast<FixedVectorType>(V->getType())->getNumElements() !=
12238 CommonMask.
size()) {
12239 V = createShuffle(InVectors.
front(),
nullptr, CommonMask);
12240 transformMaskAfterShuffle(CommonMask, CommonMask);
12242 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
12245 V->getType() != V1->
getType()
12247 : Mask[
Idx] + cast<FixedVectorType>(V1->
getType())
12248 ->getNumElements();
12249 if (V->getType() != V1->
getType())
12250 V1 = createShuffle(V1,
nullptr, Mask);
12251 InVectors.
front() = V;
12252 if (InVectors.
size() == 2)
12253 InVectors.
back() = V1;
12260 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
12266 int VF = getVF(V1);
12267 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
12269 CommonMask[
Idx] = Mask[
Idx] + (It == InVectors.
begin() ? 0 : VF);
12278 Value *Root =
nullptr) {
12279 return R.gather(VL, Root, ScalarTy);
12288 IsFinalized =
true;
12290 if (
auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
12296 ExtMask = NewExtMask;
12300 if (InVectors.
size() == 2) {
12301 Vec = createShuffle(Vec, InVectors.
back(), CommonMask);
12304 Vec = createShuffle(Vec,
nullptr, CommonMask);
12306 for (
unsigned Idx = 0, Sz = CommonMask.
size();
Idx < Sz; ++
Idx)
12310 "Expected vector length for the final value before action.");
12311 unsigned VecVF = cast<FixedVectorType>(Vec->
getType())->getNumElements();
12314 std::iota(ResizeMask.begin(), std::next(ResizeMask.begin(), VecVF), 0);
12315 Vec = createShuffle(Vec,
nullptr, ResizeMask);
12317 Action(Vec, CommonMask);
12318 InVectors.
front() = Vec;
12320 if (!ExtMask.
empty()) {
12321 if (CommonMask.
empty()) {
12325 for (
int I = 0, Sz = ExtMask.
size();
I < Sz; ++
I) {
12328 NewMask[
I] = CommonMask[ExtMask[
I]];
12330 CommonMask.
swap(NewMask);
12333 if (CommonMask.
empty()) {
12334 assert(InVectors.
size() == 1 &&
"Expected only one vector with no mask");
12335 return InVectors.
front();
12337 if (InVectors.
size() == 2)
12338 return createShuffle(InVectors.
front(), InVectors.
back(), CommonMask);
12339 return createShuffle(InVectors.
front(),
nullptr, CommonMask);
12344 "Shuffle construction must be finalized.");
12348Value *BoUpSLP::vectorizeOperand(TreeEntry *E,
unsigned NodeIdx,
12349 bool PostponedPHIs) {
12350 ValueList &VL = E->getOperand(NodeIdx);
12351 const unsigned VF = VL.size();
12354 if (!S.getOpcode() && VL.front()->getType()->isPointerTy()) {
12355 const auto *It =
find_if(VL, IsaPred<GetElementPtrInst>);
12356 if (It != VL.end())
12359 if (S.getOpcode()) {
12360 auto CheckSameVE = [&](
const TreeEntry *VE) {
12361 return VE->isSame(VL) &&
12362 (
any_of(VE->UserTreeIndices,
12363 [E, NodeIdx](
const EdgeInfo &EI) {
12364 return EI.UserTE == E && EI.EdgeIdx == NodeIdx;
12366 any_of(VectorizableTree,
12367 [E, NodeIdx, VE](
const std::unique_ptr<TreeEntry> &TE) {
12368 return TE->isOperandGatherNode({E, NodeIdx}) &&
12369 VE->isSame(TE->Scalars);
12372 TreeEntry *VE = getTreeEntry(S.OpValue);
12373 bool IsSameVE = VE && CheckSameVE(VE);
12375 auto It = MultiNodeScalars.
find(S.OpValue);
12376 if (It != MultiNodeScalars.
end()) {
12377 auto *
I =
find_if(It->getSecond(), [&](
const TreeEntry *TE) {
12378 return TE != VE && CheckSameVE(TE);
12380 if (
I != It->getSecond().end()) {
12388 ShuffleInstructionBuilder ShuffleBuilder(
12389 cast<VectorType>(
V->getType())->getElementType(), Builder, *
this);
12390 ShuffleBuilder.add(V, Mask);
12391 return ShuffleBuilder.finalize(std::nullopt);
12395 cast<FixedVectorType>(
V->getType())->getNumElements()) {
12396 if (!VE->ReuseShuffleIndices.empty()) {
12417 if (isa<PoisonValue>(V))
12419 Mask[
I] = VE->findLaneForValue(V);
12421 V = FinalShuffle(V, Mask);
12423 assert(VF < cast<FixedVectorType>(
V->getType())->getNumElements() &&
12424 "Expected vectorization factor less "
12425 "than original vector size.");
12427 std::iota(UniformMask.begin(), UniformMask.end(), 0);
12428 V = FinalShuffle(V, UniformMask);
12434 if (
find_if(VE->UserTreeIndices, [&](
const EdgeInfo &EI) {
12435 return EI.UserTE == E && EI.EdgeIdx == NodeIdx;
12436 }) == VE->UserTreeIndices.end()) {
12438 VectorizableTree, [&](
const std::unique_ptr<TreeEntry> &TE) {
12439 return TE->isGather() &&
12440 TE->UserTreeIndices.front().UserTE == E &&
12441 TE->UserTreeIndices.front().EdgeIdx == NodeIdx;
12443 assert(It != VectorizableTree.end() &&
"Expected gather node operand.");
12444 (*It)->VectorizedValue =
V;
12453 auto *
I =
find_if(VectorizableTree,
12454 [E, NodeIdx](
const std::unique_ptr<TreeEntry> &TE) {
12455 return TE->isOperandGatherNode({E, NodeIdx});
12457 assert(
I != VectorizableTree.end() &&
"Gather node is not in the graph.");
12458 assert(
I->get()->UserTreeIndices.size() == 1 &&
12459 "Expected only single user for the gather node.");
12460 assert(
I->get()->isSame(VL) &&
"Expected same list of scalars.");
12464template <
typename BVTy,
typename ResTy,
typename...
Args>
12465ResTy BoUpSLP::processBuildVector(
const TreeEntry *E,
Type *ScalarTy,
12467 assert(E->isGather() &&
"Expected gather node.");
12468 unsigned VF = E->getVectorFactor();
12470 bool NeedFreeze =
false;
12472 E->ReuseShuffleIndices.end());
12478 if (!ReorderMask.
empty())
12481 unsigned I,
unsigned SliceSize) {
12483 return isa<UndefValue>(V) && !isa<PoisonValue>(V);
12486 TreeEntry *UserTE = E->UserTreeIndices.back().UserTE;
12487 unsigned EdgeIdx = E->UserTreeIndices.back().EdgeIdx;
12488 if (UserTE->getNumOperands() != 2)
12491 find_if(VectorizableTree, [=](
const std::unique_ptr<TreeEntry> &TE) {
12492 return find_if(
TE->UserTreeIndices, [=](
const EdgeInfo &EI) {
12493 return EI.UserTE == UserTE && EI.EdgeIdx != EdgeIdx;
12494 }) !=
TE->UserTreeIndices.end();
12496 if (It == VectorizableTree.end())
12499 if ((
Mask.size() < InputVF &&
12502 (
Mask.size() == InputVF &&
12505 std::next(
Mask.begin(),
I * SliceSize),
12506 std::next(
Mask.begin(),
12513 std::next(
Mask.begin(),
I * SliceSize),
12514 std::next(
Mask.begin(),
12520 BVTy ShuffleBuilder(ScalarTy, Params...);
12521 ResTy Res = ResTy();
12525 Value *ExtractVecBase =
nullptr;
12526 bool UseVecBaseAsInput =
false;
12529 Type *OrigScalarTy = GatheredScalars.front()->getType();
12532 if (NumParts == 0 || NumParts >= GatheredScalars.size())
12534 if (!
all_of(GatheredScalars, IsaPred<UndefValue>)) {
12536 bool Resized =
false;
12538 tryToGatherExtractElements(GatheredScalars, ExtractMask, NumParts);
12539 if (!ExtractShuffles.
empty()) {
12544 if (
const auto *TE = getTreeEntry(
12545 cast<ExtractElementInst>(E->Scalars[
Idx])->getVectorOperand()))
12548 if (std::optional<ResTy> Delayed =
12549 ShuffleBuilder.needToDelay(E, ExtractEntries)) {
12551 PostponedGathers.
insert(E);
12556 if (
Value *VecBase = ShuffleBuilder.adjustExtracts(
12557 E, ExtractMask, ExtractShuffles, NumParts, UseVecBaseAsInput)) {
12558 ExtractVecBase = VecBase;
12559 if (
auto *VecBaseTy = dyn_cast<FixedVectorType>(VecBase->getType()))
12560 if (VF == VecBaseTy->getNumElements() &&
12561 GatheredScalars.size() != VF) {
12563 GatheredScalars.append(VF - GatheredScalars.size(),
12569 if (!ExtractShuffles.
empty() || E->getOpcode() != Instruction::Load ||
12570 E->isAltShuffle() ||
12571 all_of(E->Scalars, [
this](
Value *V) { return getTreeEntry(V); }) ||
12573 (E->Scalars != GatheredScalars && GatheredScalars.size() <= 2)) {
12575 isGatherShuffledEntry(E, GatheredScalars, Mask, Entries, NumParts);
12577 if (!GatherShuffles.
empty()) {
12578 if (std::optional<ResTy> Delayed =
12579 ShuffleBuilder.needToDelay(E, Entries)) {
12581 PostponedGathers.
insert(E);
12586 if (GatherShuffles.
size() == 1 &&
12588 Entries.front().front()->isSame(E->Scalars)) {
12593 <<
"SLP: perfect diamond match for gather bundle "
12596 Mask.resize(E->Scalars.size());
12597 const TreeEntry *FrontTE = Entries.front().front();
12598 if (FrontTE->ReorderIndices.empty() &&
12599 ((FrontTE->ReuseShuffleIndices.empty() &&
12600 E->Scalars.size() == FrontTE->Scalars.size()) ||
12601 (E->Scalars.size() == FrontTE->ReuseShuffleIndices.size()))) {
12602 std::iota(
Mask.begin(),
Mask.end(), 0);
12605 if (isa<PoisonValue>(V)) {
12609 Mask[
I] = FrontTE->findLaneForValue(V);
12612 ShuffleBuilder.add(*FrontTE, Mask);
12613 Res = ShuffleBuilder.finalize(E->getCommonMask());
12617 if (GatheredScalars.size() != VF &&
12619 return any_of(TEs, [&](
const TreeEntry *TE) {
12620 return TE->getVectorFactor() == VF;
12623 GatheredScalars.append(VF - GatheredScalars.size(),
12627 for (
int I = 0, Sz =
Mask.size();
I < Sz; ++
I) {
12635 bool IsRootPoison) {
12638 bool IsSplat = IsRootPoison &&
isSplat(Scalars) &&
12645 int NumNonConsts = 0;
12648 if (isa<UndefValue>(V)) {
12649 if (!isa<PoisonValue>(V)) {
12664 Scalars.
front() = OrigV;
12667 const auto Res = UniquePositions.
try_emplace(OrigV,
I);
12668 Scalars[Res.first->second] = OrigV;
12669 ReuseMask[
I] = Res.first->second;
12672 if (NumNonConsts == 1) {
12677 if (!UndefPos.
empty() && UndefPos.
front() == 0)
12680 ReuseMask[SinglePos] = SinglePos;
12681 }
else if (!UndefPos.
empty() && IsSplat) {
12686 return !isa<UndefValue>(V) &&
12688 (E->UserTreeIndices.size() == 1 &&
12692 return E->UserTreeIndices.front().EdgeIdx !=
12693 U.getOperandNo() &&
12695 E->UserTreeIndices.front().UserTE->Scalars,
12699 if (It != Scalars.
end()) {
12701 int Pos = std::distance(Scalars.
begin(), It);
12702 for (
int I : UndefPos) {
12704 ReuseMask[
I] = Pos;
12713 for (
int I : UndefPos) {
12715 if (isa<UndefValue>(Scalars[
I]))
12722 if (!ExtractShuffles.
empty() || !GatherShuffles.
empty()) {
12723 bool IsNonPoisoned =
true;
12724 bool IsUsedInExpr =
true;
12725 Value *Vec1 =
nullptr;
12726 if (!ExtractShuffles.
empty()) {
12730 Value *Vec2 =
nullptr;
12731 for (
unsigned I = 0, Sz = ExtractMask.size();
I < Sz; ++
I) {
12735 if (UseVecBaseAsInput) {
12736 Vec1 = ExtractVecBase;
12738 for (
unsigned I = 0, Sz = ExtractMask.size();
I < Sz; ++
I) {
12741 if (isa<UndefValue>(E->Scalars[
I]))
12743 auto *EI = cast<ExtractElementInst>(E->Scalars[
I]);
12744 Value *VecOp = EI->getVectorOperand();
12745 if (
const auto *TE = getTreeEntry(VecOp))
12746 if (
TE->VectorizedValue)
12747 VecOp =
TE->VectorizedValue;
12750 }
else if (Vec1 != VecOp) {
12751 assert((!Vec2 || Vec2 == VecOp) &&
12752 "Expected only 1 or 2 vectors shuffle.");
12758 IsUsedInExpr =
false;
12761 ShuffleBuilder.add(Vec1, Vec2, ExtractMask);
12763 IsUsedInExpr &= FindReusedSplat(
12765 cast<FixedVectorType>(Vec1->
getType())->getNumElements(), 0,
12766 ExtractMask.size());
12767 ShuffleBuilder.add(Vec1, ExtractMask,
true);
12770 IsUsedInExpr =
false;
12775 if (!GatherShuffles.
empty()) {
12778 for (
const auto [
I, TEs] :
enumerate(Entries)) {
12781 "No shuffles with empty entries list expected.");
12785 "Expected shuffle of 1 or 2 entries.");
12789 copy(SubMask, std::next(VecMask.begin(),
I * SliceSize));
12790 if (TEs.
size() == 1) {
12791 IsUsedInExpr &= FindReusedSplat(
12792 VecMask, TEs.
front()->getVectorFactor(),
I, SliceSize);
12793 ShuffleBuilder.add(*TEs.
front(), VecMask);
12794 if (TEs.
front()->VectorizedValue)
12798 IsUsedInExpr =
false;
12799 ShuffleBuilder.add(*TEs.
front(), *TEs.
back(), VecMask);
12800 if (TEs.
front()->VectorizedValue && TEs.
back()->VectorizedValue)
12811 int EMSz = ExtractMask.size();
12812 int MSz =
Mask.size();
12815 bool IsSingleShuffle = ExtractShuffles.
empty() || GatherShuffles.
empty();
12816 bool IsIdentityShuffle =
12817 ((UseVecBaseAsInput ||
12819 [](
const std::optional<TTI::ShuffleKind> &SK) {
12823 none_of(ExtractMask, [&](
int I) {
return I >= EMSz; }) &&
12825 (!GatherShuffles.
empty() &&
12827 [](
const std::optional<TTI::ShuffleKind> &SK) {
12831 none_of(Mask, [&](
int I) {
return I >= MSz; }) &&
12833 bool EnoughConstsForShuffle =
12837 return isa<UndefValue>(V) && !isa<PoisonValue>(V);
12841 return isa<Constant>(V) && !isa<UndefValue>(V);
12843 (!IsIdentityShuffle ||
12844 (GatheredScalars.size() == 2 &&
12846 [](
Value *V) {
return !isa<UndefValue>(V); })) ||
12848 return isa<Constant>(V) && !isa<PoisonValue>(V);
12852 for (
int I = 0, Sz = GatheredScalars.size();
I < Sz; ++
I) {
12853 if (EnoughConstsForShuffle && isa<Constant>(GatheredScalars[
I]))
12859 if (!
all_of(GatheredScalars, IsaPred<PoisonValue>)) {
12861 TryPackScalars(GatheredScalars, BVMask,
true);
12862 Value *BV = ShuffleBuilder.gather(GatheredScalars, BVMask.size());
12863 ShuffleBuilder.add(BV, BVMask);
12866 return isa<PoisonValue>(V) ||
12867 (IsSingleShuffle && ((IsIdentityShuffle &&
12868 IsNonPoisoned) || IsUsedInExpr) && isa<UndefValue>(V));
12870 Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices);
12872 Res = ShuffleBuilder.finalize(
12873 E->ReuseShuffleIndices, E->Scalars.size(),
12875 TryPackScalars(NonConstants, Mask,
false);
12876 Vec = ShuffleBuilder.gather(NonConstants,
Mask.size(), Vec);
12881 TryPackScalars(GatheredScalars, ReuseMask,
true);
12882 Value *BV = ShuffleBuilder.gather(GatheredScalars, ReuseMask.
size());
12883 ShuffleBuilder.add(BV, ReuseMask);
12884 Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices);
12889 if (!isa<PoisonValue>(V))
12892 Value *BV = ShuffleBuilder.gather(E->Scalars);
12893 ShuffleBuilder.add(BV, Mask);
12894 Res = ShuffleBuilder.finalize(E->ReuseShuffleIndices);
12898 Res = ShuffleBuilder.createFreeze(Res);
12902Value *BoUpSLP::createBuildVector(
const TreeEntry *E,
Type *ScalarTy) {
12903 return processBuildVector<ShuffleInstructionBuilder, Value *>(E, ScalarTy,
12910 if (E->VectorizedValue &&
12911 (E->State != TreeEntry::Vectorize || E->getOpcode() != Instruction::PHI ||
12912 E->isAltShuffle())) {
12913 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *E->Scalars[0] <<
".\n");
12914 return E->VectorizedValue;
12917 Value *
V = E->Scalars.front();
12918 Type *ScalarTy =
V->getType();
12919 if (
auto *Store = dyn_cast<StoreInst>(V))
12920 ScalarTy =
Store->getValueOperand()->getType();
12921 else if (
auto *IE = dyn_cast<InsertElementInst>(V))
12922 ScalarTy =
IE->getOperand(1)->getType();
12923 auto It = MinBWs.
find(E);
12924 if (It != MinBWs.
end())
12927 if (E->isGather()) {
12929 if (E->getMainOp() && E->Idx == 0 && !UserIgnoreList)
12930 setInsertPointAfterBundle(E);
12931 Value *Vec = createBuildVector(E, ScalarTy);
12932 E->VectorizedValue = Vec;
12937 auto FinalShuffle = [&](
Value *
V,
const TreeEntry *E,
VectorType *VecTy) {
12938 ShuffleInstructionBuilder ShuffleBuilder(ScalarTy, Builder, *
this);
12939 if (E->getOpcode() == Instruction::Store &&
12940 E->State == TreeEntry::Vectorize) {
12942 ArrayRef(
reinterpret_cast<const int *
>(E->ReorderIndices.begin()),
12943 E->ReorderIndices.size());
12944 ShuffleBuilder.add(V, Mask);
12945 }
else if (E->State == TreeEntry::StridedVectorize && IsReverseOrder) {
12946 ShuffleBuilder.addOrdered(V, std::nullopt);
12948 ShuffleBuilder.addOrdered(V, E->ReorderIndices);
12950 return ShuffleBuilder.finalize(E->ReuseShuffleIndices);
12953 assert((E->State == TreeEntry::Vectorize ||
12954 E->State == TreeEntry::ScatterVectorize ||
12955 E->State == TreeEntry::StridedVectorize) &&
12956 "Unhandled state");
12957 unsigned ShuffleOrOp =
12958 E->isAltShuffle() ? (
unsigned)Instruction::ShuffleVector : E->getOpcode();
12960 auto GetOperandSignedness = [&](
unsigned Idx) {
12961 const TreeEntry *OpE = getOperandEntry(E,
Idx);
12962 bool IsSigned =
false;
12963 auto It = MinBWs.
find(OpE);
12964 if (It != MinBWs.
end())
12965 IsSigned = It->second.second;
12968 return !isKnownNonNegative(R, SimplifyQuery(*DL));
12972 switch (ShuffleOrOp) {
12973 case Instruction::PHI: {
12974 assert((E->ReorderIndices.empty() || !E->ReuseShuffleIndices.empty() ||
12975 E != VectorizableTree.front().get() ||
12976 !E->UserTreeIndices.empty()) &&
12977 "PHI reordering is free.");
12978 if (PostponedPHIs && E->VectorizedValue)
12979 return E->VectorizedValue;
12980 auto *PH = cast<PHINode>(VL0);
12982 PH->getParent()->getFirstNonPHIIt());
12984 if (PostponedPHIs || !E->VectorizedValue) {
12991 PH->getParent()->getFirstInsertionPt());
12994 V = FinalShuffle(V, E, VecTy);
12996 E->VectorizedValue =
V;
13000 PHINode *NewPhi = cast<PHINode>(E->PHI);
13009 for (
unsigned I : seq<unsigned>(0, PH->getNumIncomingValues())) {
13015 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13019 if (!VisitedBBs.
insert(IBB).second) {
13026 Value *Vec = vectorizeOperand(E,
I,
true);
13027 if (VecTy != Vec->
getType()) {
13029 MinBWs.
contains(getOperandEntry(E,
I))) &&
13030 "Expected item in MinBWs.");
13031 Vec = Builder.
CreateIntCast(Vec, VecTy, GetOperandSignedness(
I));
13037 "Invalid number of incoming values");
13041 case Instruction::ExtractElement: {
13042 Value *
V = E->getSingleOperand(0);
13043 if (
const TreeEntry *TE = getTreeEntry(V))
13044 V =
TE->VectorizedValue;
13045 setInsertPointAfterBundle(E);
13046 V = FinalShuffle(V, E, VecTy);
13047 E->VectorizedValue =
V;
13050 case Instruction::ExtractValue: {
13051 auto *LI = cast<LoadInst>(E->getSingleOperand(0));
13056 NewV = FinalShuffle(NewV, E, VecTy);
13057 E->VectorizedValue = NewV;
13060 case Instruction::InsertElement: {
13061 assert(E->ReuseShuffleIndices.empty() &&
"All inserts should be unique");
13063 Value *
V = vectorizeOperand(E, 1, PostponedPHIs);
13065 Type *ScalarTy =
Op.front()->getType();
13066 if (cast<VectorType>(
V->getType())->getElementType() != ScalarTy) {
13068 std::pair<unsigned, bool> Res = MinBWs.
lookup(getOperandEntry(E, 1));
13069 assert(Res.first > 0 &&
"Expected item in MinBWs.");
13074 cast<FixedVectorType>(
V->getType())->getNumElements()),
13079 auto *FirstInsert = cast<Instruction>(*
find_if(E->Scalars, [E](
Value *V) {
13080 return !is_contained(E->Scalars, cast<Instruction>(V)->getOperand(0));
13082 const unsigned NumElts =
13083 cast<FixedVectorType>(FirstInsert->getType())->getNumElements();
13084 const unsigned NumScalars = E->Scalars.size();
13087 assert(
Offset < NumElts &&
"Failed to find vector index offset");
13091 if (!E->ReorderIndices.empty()) {
13096 std::iota(
Mask.begin(), std::next(
Mask.begin(), NumScalars), 0);
13099 bool IsIdentity =
true;
13101 Mask.swap(PrevMask);
13102 for (
unsigned I = 0;
I < NumScalars; ++
I) {
13105 IsIdentity &= InsertIdx -
Offset ==
I;
13108 if (!IsIdentity || NumElts != NumScalars) {
13112 if (NumElts != NumScalars &&
Offset == 0) {
13121 InsertMask[*InsertIdx] = *InsertIdx;
13122 if (!
Ins->hasOneUse())
13124 Ins = dyn_cast_or_null<InsertElementInst>(
13125 Ins->getUniqueUndroppableUser());
13128 buildUseMask(NumElts, InsertMask, UseMask::UndefsAsMask);
13130 isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
13133 if (!IsFirstPoison.
all()) {
13135 for (
unsigned I = 0;
I < NumElts;
I++) {
13137 IsFirstUndef.
test(
I)) {
13138 if (IsVNonPoisonous) {
13139 InsertMask[
I] =
I < NumScalars ?
I : 0;
13144 if (
Idx >= NumScalars)
13145 Idx = NumScalars - 1;
13146 InsertMask[
I] = NumScalars +
Idx;
13160 if (
auto *
I = dyn_cast<Instruction>(V)) {
13161 GatherShuffleExtractSeq.
insert(
I);
13162 CSEBlocks.
insert(
I->getParent());
13167 for (
unsigned I = 0;
I < NumElts;
I++) {
13172 buildUseMask(NumElts, InsertMask, UseMask::UndefsAsMask);
13175 if ((!IsIdentity ||
Offset != 0 || !IsFirstUndef.
all()) &&
13176 NumElts != NumScalars) {
13177 if (IsFirstUndef.
all()) {
13180 isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
13181 if (!IsFirstPoison.
all()) {
13182 for (
unsigned I = 0;
I < NumElts;
I++) {
13184 InsertMask[
I] =
I + NumElts;
13191 InsertMask, cast<Instruction>(E->Scalars.back())->
getName());
13192 if (
auto *
I = dyn_cast<Instruction>(V)) {
13193 GatherShuffleExtractSeq.
insert(
I);
13194 CSEBlocks.
insert(
I->getParent());
13199 isUndefVector<true>(FirstInsert->getOperand(0), UseMask);
13200 for (
unsigned I = 0;
I < NumElts;
I++) {
13204 InsertMask[
I] += NumElts;
13207 FirstInsert->getOperand(0), V, InsertMask,
13208 cast<Instruction>(E->Scalars.back())->getName());
13209 if (
auto *
I = dyn_cast<Instruction>(V)) {
13210 GatherShuffleExtractSeq.
insert(
I);
13211 CSEBlocks.
insert(
I->getParent());
13216 ++NumVectorInstructions;
13217 E->VectorizedValue =
V;
13220 case Instruction::ZExt:
13221 case Instruction::SExt:
13222 case Instruction::FPToUI:
13223 case Instruction::FPToSI:
13224 case Instruction::FPExt:
13225 case Instruction::PtrToInt:
13226 case Instruction::IntToPtr:
13227 case Instruction::SIToFP:
13228 case Instruction::UIToFP:
13229 case Instruction::Trunc:
13230 case Instruction::FPTrunc:
13231 case Instruction::BitCast: {
13232 setInsertPointAfterBundle(E);
13234 Value *InVec = vectorizeOperand(E, 0, PostponedPHIs);
13235 if (E->VectorizedValue) {
13236 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13237 return E->VectorizedValue;
13240 auto *CI = cast<CastInst>(VL0);
13242 Type *SrcScalarTy = cast<VectorType>(InVec->
getType())->getElementType();
13243 auto SrcIt = MinBWs.
find(getOperandEntry(E, 0));
13245 (SrcIt != MinBWs.
end() || It != MinBWs.
end() ||
13248 unsigned SrcBWSz =
DL->getTypeSizeInBits(SrcScalarTy);
13249 if (SrcIt != MinBWs.
end())
13250 SrcBWSz = SrcIt->second.first;
13251 unsigned BWSz =
DL->getTypeSizeInBits(ScalarTy);
13252 if (BWSz == SrcBWSz) {
13253 VecOpcode = Instruction::BitCast;
13254 }
else if (BWSz < SrcBWSz) {
13255 VecOpcode = Instruction::Trunc;
13256 }
else if (It != MinBWs.
end()) {
13257 assert(BWSz > SrcBWSz &&
"Invalid cast!");
13258 VecOpcode = It->second.second ? Instruction::SExt : Instruction::ZExt;
13259 }
else if (SrcIt != MinBWs.
end()) {
13260 assert(BWSz > SrcBWSz &&
"Invalid cast!");
13262 SrcIt->second.second ? Instruction::SExt : Instruction::ZExt;
13264 }
else if (VecOpcode == Instruction::SIToFP && SrcIt != MinBWs.
end() &&
13265 !SrcIt->second.second) {
13266 VecOpcode = Instruction::UIToFP;
13268 Value *
V = (VecOpcode != ShuffleOrOp && VecOpcode == Instruction::BitCast)
13270 : Builder.
CreateCast(VecOpcode, InVec, VecTy);
13271 V = FinalShuffle(V, E, VecTy);
13273 E->VectorizedValue =
V;
13274 ++NumVectorInstructions;
13277 case Instruction::FCmp:
13278 case Instruction::ICmp: {
13279 setInsertPointAfterBundle(E);
13281 Value *
L = vectorizeOperand(E, 0, PostponedPHIs);
13282 if (E->VectorizedValue) {
13283 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13284 return E->VectorizedValue;
13286 Value *
R = vectorizeOperand(E, 1, PostponedPHIs);
13287 if (E->VectorizedValue) {
13288 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13289 return E->VectorizedValue;
13291 if (
L->getType() !=
R->getType()) {
13293 getOperandEntry(E, 1)->
isGather() ||
13294 MinBWs.
contains(getOperandEntry(E, 0)) ||
13295 MinBWs.
contains(getOperandEntry(E, 1))) &&
13296 "Expected item in MinBWs.");
13297 if (cast<VectorType>(
L->getType())
13299 ->getIntegerBitWidth() < cast<VectorType>(
R->getType())
13301 ->getIntegerBitWidth()) {
13302 Type *CastTy =
R->getType();
13305 Type *CastTy =
L->getType();
13314 VecTy = cast<FixedVectorType>(
V->getType());
13315 V = FinalShuffle(V, E, VecTy);
13317 E->VectorizedValue =
V;
13318 ++NumVectorInstructions;
13321 case Instruction::Select: {
13322 setInsertPointAfterBundle(E);
13324 Value *
Cond = vectorizeOperand(E, 0, PostponedPHIs);
13325 if (E->VectorizedValue) {
13326 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13327 return E->VectorizedValue;
13329 Value *True = vectorizeOperand(E, 1, PostponedPHIs);
13330 if (E->VectorizedValue) {
13331 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13332 return E->VectorizedValue;
13334 Value *False = vectorizeOperand(E, 2, PostponedPHIs);
13335 if (E->VectorizedValue) {
13336 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13337 return E->VectorizedValue;
13341 getOperandEntry(E, 2)->
isGather() ||
13342 MinBWs.
contains(getOperandEntry(E, 1)) ||
13343 MinBWs.
contains(getOperandEntry(E, 2))) &&
13344 "Expected item in MinBWs.");
13345 if (True->
getType() != VecTy)
13346 True = Builder.
CreateIntCast(True, VecTy, GetOperandSignedness(1));
13347 if (False->
getType() != VecTy)
13348 False = Builder.
CreateIntCast(False, VecTy, GetOperandSignedness(2));
13353 assert(TrueNumElements >= CondNumElements &&
13354 TrueNumElements % CondNumElements == 0 &&
13355 "Cannot vectorize Instruction::Select");
13357 "Cannot vectorize Instruction::Select");
13358 if (CondNumElements != TrueNumElements) {
13366 "Cannot vectorize Instruction::Select");
13368 V = FinalShuffle(V, E, VecTy);
13370 E->VectorizedValue =
V;
13371 ++NumVectorInstructions;
13374 case Instruction::FNeg: {
13375 setInsertPointAfterBundle(E);
13377 Value *
Op = vectorizeOperand(E, 0, PostponedPHIs);
13379 if (E->VectorizedValue) {
13380 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13381 return E->VectorizedValue;
13387 if (
auto *
I = dyn_cast<Instruction>(V))
13390 V = FinalShuffle(V, E, VecTy);
13392 E->VectorizedValue =
V;
13393 ++NumVectorInstructions;
13397 case Instruction::Freeze: {
13398 setInsertPointAfterBundle(E);
13400 Value *
Op = vectorizeOperand(E, 0, PostponedPHIs);
13402 if (E->VectorizedValue) {
13403 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13404 return E->VectorizedValue;
13408 V = FinalShuffle(V, E, VecTy);
13410 E->VectorizedValue =
V;
13411 ++NumVectorInstructions;
13415 case Instruction::Add:
13416 case Instruction::FAdd:
13417 case Instruction::Sub:
13418 case Instruction::FSub:
13419 case Instruction::Mul:
13420 case Instruction::FMul:
13421 case Instruction::UDiv:
13422 case Instruction::SDiv:
13423 case Instruction::FDiv:
13424 case Instruction::URem:
13425 case Instruction::SRem:
13426 case Instruction::FRem:
13427 case Instruction::Shl:
13428 case Instruction::LShr:
13429 case Instruction::AShr:
13430 case Instruction::And:
13431 case Instruction::Or:
13432 case Instruction::Xor: {
13433 setInsertPointAfterBundle(E);
13435 Value *
LHS = vectorizeOperand(E, 0, PostponedPHIs);
13436 if (E->VectorizedValue) {
13437 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13438 return E->VectorizedValue;
13440 Value *
RHS = vectorizeOperand(E, 1, PostponedPHIs);
13441 if (E->VectorizedValue) {
13442 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13443 return E->VectorizedValue;
13445 if (ShuffleOrOp == Instruction::And && It != MinBWs.
end()) {
13446 for (
unsigned I : seq<unsigned>(0, E->getNumOperands())) {
13449 auto *CI = dyn_cast<ConstantInt>(
Op);
13450 return CI && CI->getValue().countr_one() >= It->second.first;
13452 V = FinalShuffle(
I == 0 ? RHS : LHS, E, VecTy);
13453 E->VectorizedValue =
V;
13454 ++NumVectorInstructions;
13461 getOperandEntry(E, 1)->
isGather() ||
13462 MinBWs.
contains(getOperandEntry(E, 0)) ||
13463 MinBWs.
contains(getOperandEntry(E, 1))) &&
13464 "Expected item in MinBWs.");
13475 if (
auto *
I = dyn_cast<Instruction>(V)) {
13478 if (!MinBWs.
contains(E) && ShuffleOrOp == Instruction::Sub &&
13480 return isCommutative(cast<Instruction>(V));
13482 I->setHasNoUnsignedWrap(
false);
13485 V = FinalShuffle(V, E, VecTy);
13487 E->VectorizedValue =
V;
13488 ++NumVectorInstructions;
13492 case Instruction::Load: {
13495 setInsertPointAfterBundle(E);
13497 LoadInst *LI = cast<LoadInst>(VL0);
13500 if (E->State == TreeEntry::Vectorize) {
13502 }
else if (E->State == TreeEntry::StridedVectorize) {
13503 Value *Ptr0 = cast<LoadInst>(E->Scalars.front())->getPointerOperand();
13504 Value *PtrN = cast<LoadInst>(E->Scalars.back())->getPointerOperand();
13505 PO = IsReverseOrder ? PtrN : Ptr0;
13511 int Stride = *Diff / (
static_cast<int>(E->Scalars.size()) - 1);
13513 ConstantInt::get(StrideTy, (IsReverseOrder ? -1 : 1) * Stride *
13514 DL->getTypeAllocSize(ScalarTy));
13518 return cast<LoadInst>(V)->getPointerOperand();
13521 std::optional<Value *> Stride =
13530 (IsReverseOrder ? -1 : 1) *
13531 static_cast<int>(
DL->getTypeAllocSize(ScalarTy))));
13533 Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
13535 Intrinsic::experimental_vp_strided_load,
13536 {VecTy, PO->
getType(), StrideTy},
13538 Builder.
getInt32(E->Scalars.size())});
13544 assert(E->State == TreeEntry::ScatterVectorize &&
"Unhandled state");
13545 Value *VecPtr = vectorizeOperand(E, 0, PostponedPHIs);
13546 if (E->VectorizedValue) {
13547 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13548 return E->VectorizedValue;
13551 Align CommonAlignment = computeCommonAlignment<LoadInst>(E->Scalars);
13556 V = FinalShuffle(V, E, VecTy);
13557 E->VectorizedValue =
V;
13558 ++NumVectorInstructions;
13561 case Instruction::Store: {
13562 auto *
SI = cast<StoreInst>(VL0);
13564 setInsertPointAfterBundle(E);
13566 Value *VecValue = vectorizeOperand(E, 0, PostponedPHIs);
13567 if (VecValue->
getType() != VecTy)
13569 Builder.
CreateIntCast(VecValue, VecTy, GetOperandSignedness(0));
13570 VecValue = FinalShuffle(VecValue, E, VecTy);
13574 if (E->State == TreeEntry::Vectorize) {
13577 assert(E->State == TreeEntry::StridedVectorize &&
13578 "Expected either strided or conseutive stores.");
13579 if (!E->ReorderIndices.empty()) {
13580 SI = cast<StoreInst>(E->Scalars[E->ReorderIndices.front()]);
13581 Ptr =
SI->getPointerOperand();
13583 Align CommonAlignment = computeCommonAlignment<StoreInst>(E->Scalars);
13584 Type *StrideTy =
DL->getIndexType(
SI->getPointerOperandType());
13586 Intrinsic::experimental_vp_strided_store,
13587 {VecTy,
Ptr->getType(), StrideTy},
13590 StrideTy, -
static_cast<int>(
DL->getTypeAllocSize(ScalarTy))),
13592 Builder.
getInt32(E->Scalars.size())});
13601 E->VectorizedValue =
V;
13602 ++NumVectorInstructions;
13605 case Instruction::GetElementPtr: {
13606 auto *GEP0 = cast<GetElementPtrInst>(VL0);
13607 setInsertPointAfterBundle(E);
13609 Value *Op0 = vectorizeOperand(E, 0, PostponedPHIs);
13610 if (E->VectorizedValue) {
13611 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13612 return E->VectorizedValue;
13616 for (
int J = 1,
N = GEP0->getNumOperands(); J <
N; ++J) {
13617 Value *OpVec = vectorizeOperand(E, J, PostponedPHIs);
13618 if (E->VectorizedValue) {
13619 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13620 return E->VectorizedValue;
13625 Value *
V = Builder.
CreateGEP(GEP0->getSourceElementType(), Op0, OpVecs);
13626 if (
Instruction *
I = dyn_cast<GetElementPtrInst>(V)) {
13628 for (
Value *V : E->Scalars) {
13629 if (isa<GetElementPtrInst>(V))
13635 V = FinalShuffle(V, E, VecTy);
13637 E->VectorizedValue =
V;
13638 ++NumVectorInstructions;
13642 case Instruction::Call: {
13643 CallInst *CI = cast<CallInst>(VL0);
13644 setInsertPointAfterBundle(E);
13650 It != MinBWs.
end() ? It->second.first : 0);
13653 VecCallCosts.first <= VecCallCosts.second;
13655 Value *ScalarArg =
nullptr;
13661 auto *CEI = cast<CallInst>(VL0);
13662 for (
unsigned I : seq<unsigned>(0, CI->
arg_size())) {
13667 ScalarArg = CEI->getArgOperand(
I);
13670 if (
ID == Intrinsic::abs && It != MinBWs.
end() &&
13671 It->second.first <
DL->getTypeSizeInBits(CEI->getType()))
13679 Value *OpVec = vectorizeOperand(E,
I, PostponedPHIs);
13680 if (E->VectorizedValue) {
13681 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13682 return E->VectorizedValue;
13684 ScalarArg = CEI->getArgOperand(
I);
13685 if (cast<VectorType>(OpVec->
getType())->getElementType() !=
13687 It == MinBWs.
end()) {
13690 OpVec = Builder.
CreateIntCast(OpVec, CastTy, GetOperandSignedness(
I));
13691 }
else if (It != MinBWs.
end()) {
13692 OpVec = Builder.
CreateIntCast(OpVec, VecTy, GetOperandSignedness(
I));
13701 if (!UseIntrinsic) {
13717 V = FinalShuffle(V, E, VecTy);
13719 E->VectorizedValue =
V;
13720 ++NumVectorInstructions;
13723 case Instruction::ShuffleVector: {
13724 assert(E->isAltShuffle() &&
13729 (isa<CmpInst>(VL0) && isa<CmpInst>(E->getAltOp()))) &&
13730 "Invalid Shuffle Vector Operand");
13734 setInsertPointAfterBundle(E);
13735 LHS = vectorizeOperand(E, 0, PostponedPHIs);
13736 if (E->VectorizedValue) {
13737 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13738 return E->VectorizedValue;
13740 RHS = vectorizeOperand(E, 1, PostponedPHIs);
13742 setInsertPointAfterBundle(E);
13743 LHS = vectorizeOperand(E, 0, PostponedPHIs);
13745 if (E->VectorizedValue) {
13746 LLVM_DEBUG(
dbgs() <<
"SLP: Diamond merged for " << *VL0 <<
".\n");
13747 return E->VectorizedValue;
13754 getOperandEntry(E, 1)->
isGather() ||
13755 MinBWs.
contains(getOperandEntry(E, 0)) ||
13756 MinBWs.
contains(getOperandEntry(E, 1))) &&
13757 "Expected item in MinBWs.");
13758 Type *CastTy = VecTy;
13762 ->getIntegerBitWidth() < cast<VectorType>(
RHS->
getType())
13764 ->getIntegerBitWidth())
13781 }
else if (
auto *CI0 = dyn_cast<CmpInst>(VL0)) {
13782 V0 = Builder.
CreateCmp(CI0->getPredicate(), LHS, RHS);
13783 auto *AltCI = cast<CmpInst>(E->getAltOp());
13785 V1 = Builder.
CreateCmp(AltPred, LHS, RHS);
13788 unsigned SrcBWSz =
DL->getTypeSizeInBits(
13789 cast<VectorType>(
LHS->
getType())->getElementType());
13790 unsigned BWSz =
DL->getTypeSizeInBits(ScalarTy);
13791 if (BWSz <= SrcBWSz) {
13792 if (BWSz < SrcBWSz)
13795 if (
auto *
I = dyn_cast<Instruction>(LHS))
13797 E->VectorizedValue =
LHS;
13798 ++NumVectorInstructions;
13809 for (
Value *V : {V0, V1}) {
13810 if (
auto *
I = dyn_cast<Instruction>(V)) {
13811 GatherShuffleExtractSeq.
insert(
I);
13812 CSEBlocks.
insert(
I->getParent());
13821 E->buildAltOpShuffleMask(
13823 assert(E->isOpcodeOrAlt(
I) &&
"Unexpected main/alternate opcode");
13827 Mask, &OpScalars, &AltScalars);
13831 auto DropNuwFlag = [&](
Value *Vec,
unsigned Opcode) {
13833 if (
auto *
I = dyn_cast<Instruction>(Vec);
13834 I && Opcode == Instruction::Sub && !MinBWs.
contains(E) &&
13836 auto *IV = cast<Instruction>(V);
13837 return IV->getOpcode() == Instruction::Sub &&
13838 isCommutative(cast<Instruction>(IV));
13840 I->setHasNoUnsignedWrap(
false);
13842 DropNuwFlag(V0, E->getOpcode());
13843 DropNuwFlag(V1, E->getAltOpcode());
13846 if (
auto *
I = dyn_cast<Instruction>(V)) {
13848 GatherShuffleExtractSeq.
insert(
I);
13849 CSEBlocks.
insert(
I->getParent());
13852 E->VectorizedValue =
V;
13853 ++NumVectorInstructions;
13866 return vectorizeTree(ExternallyUsedValues, ReplacedExternals);
13872struct ShuffledInsertData {
13885 for (
auto &BSIter : BlocksSchedules) {
13886 scheduleBlock(BSIter.second.get());
13890 EntryToLastInstruction.
clear();
13900 for (
const std::unique_ptr<TreeEntry> &TE : VectorizableTree)
13901 if (TE->State == TreeEntry::Vectorize &&
13902 TE->getOpcode() == Instruction::PHI && !TE->isAltShuffle() &&
13903 TE->VectorizedValue)
13909 for (
const TreeEntry *E : PostponedNodes) {
13910 auto *TE =
const_cast<TreeEntry *
>(E);
13911 if (
auto *VecTE = getTreeEntry(TE->Scalars.front()))
13912 if (VecTE->isSame(TE->UserTreeIndices.front().UserTE->getOperand(
13913 TE->UserTreeIndices.front().EdgeIdx)) &&
13914 VecTE->isSame(TE->Scalars))
13918 auto *PrevVec = cast<Instruction>(TE->VectorizedValue);
13919 TE->VectorizedValue =
nullptr;
13921 cast<Instruction>(TE->UserTreeIndices.front().UserTE->VectorizedValue);
13930 if (isa<PHINode>(UserI)) {
13933 for (
User *U : PrevVec->users()) {
13936 auto *UI = dyn_cast<Instruction>(U);
13937 if (!UI || isa<PHINode>(UI) || UI->getParent() != InsertPt->
getParent())
13939 if (UI->comesBefore(InsertPt))
13948 if (Vec->
getType() != PrevVec->getType()) {
13950 PrevVec->getType()->isIntOrIntVectorTy() &&
13951 "Expected integer vector types only.");
13952 std::optional<bool> IsSigned;
13953 for (
Value *V : TE->Scalars) {
13954 if (
const TreeEntry *BaseTE = getTreeEntry(V)) {
13955 auto It = MinBWs.
find(BaseTE);
13956 if (It != MinBWs.
end()) {
13957 IsSigned = IsSigned.value_or(
false) || It->second.second;
13961 for (
const TreeEntry *MNTE : MultiNodeScalars.
lookup(V)) {
13962 auto It = MinBWs.
find(MNTE);
13963 if (It != MinBWs.
end()) {
13964 IsSigned = IsSigned.value_or(
false) || It->second.second;
13969 if (IsSigned.value_or(
false))
13972 for (
const TreeEntry *BVE : ValueToGatherNodes.lookup(V)) {
13973 auto It = MinBWs.
find(BVE);
13974 if (It != MinBWs.
end()) {
13975 IsSigned = IsSigned.value_or(
false) || It->second.second;
13980 if (IsSigned.value_or(
false))
13982 if (
auto *EE = dyn_cast<ExtractElementInst>(V)) {
13984 IsSigned.value_or(
false) ||
13988 if (IsSigned.value_or(
false))
13992 if (IsSigned.value_or(
false)) {
13994 auto It = MinBWs.
find(TE->UserTreeIndices.front().UserTE);
13995 if (It != MinBWs.
end())
13996 IsSigned = It->second.second;
13999 "Expected user node or perfect diamond match in MinBWs.");
14003 PostponedValues.
try_emplace(Vec).first->second.push_back(TE);
14006 auto It = PostponedValues.
find(PrevVec);
14007 if (It != PostponedValues.
end()) {
14008 for (TreeEntry *VTE : It->getSecond())
14009 VTE->VectorizedValue = Vec;
14029 for (
const auto &ExternalUse : ExternalUses) {
14030 Value *Scalar = ExternalUse.Scalar;
14037 TreeEntry *E = getTreeEntry(Scalar);
14038 assert(E &&
"Invalid scalar");
14039 assert(!E->isGather() &&
"Extracting from a gather list");
14041 if (E->getOpcode() == Instruction::GetElementPtr &&
14042 !isa<GetElementPtrInst>(Scalar))
14045 Value *Vec = E->VectorizedValue;
14046 assert(Vec &&
"Can't find vectorizable value");
14049 auto ExtractAndExtendIfNeeded = [&](
Value *Vec) {
14050 if (Scalar->getType() != Vec->
getType()) {
14051 Value *Ex =
nullptr;
14052 Value *ExV =
nullptr;
14053 auto *
GEP = dyn_cast<GetElementPtrInst>(Scalar);
14055 auto It = ScalarToEEs.find(Scalar);
14056 if (It != ScalarToEEs.end()) {
14060 if (EEIt != It->second.end()) {
14066 if (
auto *CI = EEIt->second.second)
14070 ExV = EEIt->second.second ? EEIt->second.second : Ex;
14075 if (
auto *ES = dyn_cast<ExtractElementInst>(Scalar);
14076 ES && isa<Instruction>(Vec)) {
14077 Value *V = ES->getVectorOperand();
14078 auto *IVec = cast<Instruction>(Vec);
14079 if (
const TreeEntry *ETE = getTreeEntry(V))
14080 V = ETE->VectorizedValue;
14081 if (
auto *
IV = dyn_cast<Instruction>(V);
14082 !
IV ||
IV == Vec ||
IV->getParent() != IVec->getParent() ||
14083 IV->comesBefore(IVec))
14087 }
else if (ReplaceGEP) {
14090 auto *CloneGEP =
GEP->clone();
14091 if (isa<Instruction>(Vec))
14095 CloneGEP->insertBefore(
GEP);
14096 if (
GEP->hasName())
14097 CloneGEP->takeName(
GEP);
14099 }
else if (
auto *VecTy =
14100 dyn_cast<FixedVectorType>(Scalar->getType())) {
14109 Vec, Builder.
getInt64(ExternalUse.Lane * VecTyNumElements));
14116 if (Scalar->getType() != Ex->
getType())
14118 MinBWs.
find(E)->second.second);
14119 if (
auto *
I = dyn_cast<Instruction>(Ex))
14120 ScalarToEEs[Scalar].try_emplace(
14122 std::make_pair(
I, cast<Instruction>(ExV)));
14126 if (
auto *ExI = dyn_cast<Instruction>(Ex)) {
14127 GatherShuffleExtractSeq.
insert(ExI);
14128 CSEBlocks.
insert(ExI->getParent());
14132 assert(isa<FixedVectorType>(Scalar->getType()) &&
14133 isa<InsertElementInst>(Scalar) &&
14134 "In-tree scalar of vector type is not insertelement?");
14135 auto *IE = cast<InsertElementInst>(Scalar);
14143 if (!ScalarsWithNullptrUser.
insert(Scalar).second)
14149 if (ExternalUsesAsGEPs.contains(U))
14151 TreeEntry *UseEntry = getTreeEntry(U);
14153 (UseEntry->State == TreeEntry::Vectorize ||
14155 TreeEntry::StridedVectorize) &&
14156 (E->State == TreeEntry::Vectorize ||
14157 E->State == TreeEntry::StridedVectorize) &&
14158 doesInTreeUserNeedToExtract(
14160 cast<Instruction>(UseEntry->Scalars.front()),
14163 "Scalar with nullptr User must be registered in "
14164 "ExternallyUsedValues map or remain as scalar in vectorized "
14166 if (
auto *VecI = dyn_cast<Instruction>(Vec)) {
14167 if (
auto *
PHI = dyn_cast<PHINode>(VecI)) {
14168 if (
PHI->getParent()->isLandingPad())
14172 PHI->getParent()->getLandingPadInst()->getIterator()));
14175 PHI->getParent()->getFirstNonPHIIt());
14178 std::next(VecI->getIterator()));
14183 Value *NewInst = ExtractAndExtendIfNeeded(Vec);
14185 Scalar->replaceAllUsesWith(NewInst);
14186 ReplacedExternals.emplace_back(Scalar, NewInst);
14190 if (
auto *VU = dyn_cast<InsertElementInst>(
User);
14193 if (!Scalar->getType()->isVectorTy() && isa<Instruction>(Vec)) {
14194 if (
auto *FTy = dyn_cast<FixedVectorType>(
User->
getType())) {
14195 if (!UsedInserts.
insert(VU).second)
14198 auto BWIt = MinBWs.
find(E);
14200 auto *ScalarTy = FTy->getElementType();
14201 auto Key = std::make_pair(Vec, ScalarTy);
14202 auto VecIt = VectorCasts.
find(Key);
14203 if (VecIt == VectorCasts.
end()) {
14205 if (
auto *IVec = dyn_cast<PHINode>(Vec)) {
14206 if (IVec->getParent()->isLandingPad())
14208 std::next(IVec->getParent()
14209 ->getLandingPadInst()
14213 IVec->getParent()->getFirstNonPHIOrDbgOrLifetime());
14214 }
else if (
auto *IVec = dyn_cast<Instruction>(Vec)) {
14221 cast<FixedVectorType>(Vec->
getType())->getNumElements()),
14222 BWIt->second.second);
14225 Vec = VecIt->second;
14232 find_if(ShuffledInserts, [VU](
const ShuffledInsertData &
Data) {
14239 unsigned Idx = *InsertIdx;
14240 if (It == ShuffledInserts.
end()) {
14242 It = std::next(ShuffledInserts.
begin(),
14243 ShuffledInserts.
size() - 1);
14249 while (
auto *IEBase = dyn_cast<InsertElementInst>(
Base)) {
14250 if (IEBase !=
User &&
14251 (!IEBase->hasOneUse() ||
14255 if (
const TreeEntry *E = getTreeEntry(IEBase)) {
14257 IEBase = cast<InsertElementInst>(
Base);
14260 "InsertElementInstruction used already.");
14261 Mask[IEIdx] = IEIdx;
14262 Base = IEBase->getOperand(0);
14263 }
while (E == getTreeEntry(
Base));
14266 Base = cast<InsertElementInst>(
Base)->getOperand(0);
14270 auto It = VectorToInsertElement.
find(
Base);
14271 if (It != VectorToInsertElement.
end())
14278 Mask[
Idx] = ExternalUse.Lane;
14279 It->InsertElements.push_back(cast<InsertElementInst>(
User));
14288 if (
auto *VecI = dyn_cast<Instruction>(Vec)) {
14290 for (
unsigned I : seq<unsigned>(0, PH->getNumIncomingValues())) {
14291 if (PH->getIncomingValue(
I) == Scalar) {
14293 PH->getIncomingBlock(
I)->getTerminator();
14294 if (isa<CatchSwitchInst>(IncomingTerminator)) {
14296 std::next(VecI->getIterator()));
14300 Value *NewInst = ExtractAndExtendIfNeeded(Vec);
14301 PH->setOperand(
I, NewInst);
14306 Value *NewInst = ExtractAndExtendIfNeeded(Vec);
14311 Value *NewInst = ExtractAndExtendIfNeeded(Vec);
14321 int VF = cast<FixedVectorType>(V1->
getType())->getNumElements();
14322 for (
int I = 0, E = Mask.size();
I < E; ++
I) {
14324 CombinedMask1[
I] = Mask[
I];
14326 CombinedMask2[
I] = Mask[
I] - VF;
14329 cast<VectorType>(V1->
getType())->getElementType(), Builder, *
this);
14330 ShuffleBuilder.
add(V1, CombinedMask1);
14332 ShuffleBuilder.
add(V2, CombinedMask2);
14333 return ShuffleBuilder.
finalize(std::nullopt);
14337 bool ForSingleMask) {
14338 unsigned VF = Mask.size();
14339 unsigned VecVF = cast<FixedVectorType>(Vec->
getType())->getNumElements();
14341 if (
any_of(Mask, [VF](
int Idx) {
return Idx >=
static_cast<int>(VF); })) {
14342 Vec = CreateShuffle(Vec,
nullptr, Mask);
14343 return std::make_pair(Vec,
true);
14345 if (!ForSingleMask) {
14347 for (
unsigned I = 0;
I < VF; ++
I) {
14349 ResizeMask[Mask[
I]] = Mask[
I];
14351 Vec = CreateShuffle(Vec,
nullptr, ResizeMask);
14355 return std::make_pair(Vec,
false);
14359 for (
int I = 0, E = ShuffledInserts.
size();
I < E; ++
I) {
14365 auto Vector = ShuffledInserts[
I].ValueMasks.takeVector();
14366 Value *NewInst = performExtractsShuffleAction<Value>(
14370 return cast<VectorType>(Vec->getType())
14371 ->getElementCount()
14372 .getKnownMinValue();
14377 assert((Vals.size() == 1 || Vals.size() == 2) &&
14378 "Expected exactly 1 or 2 input values.");
14379 if (Vals.size() == 1) {
14382 if (Mask.size() != cast<FixedVectorType>(Vals.front()->getType())
14383 ->getNumElements() ||
14384 !ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
14385 return CreateShuffle(Vals.front(), nullptr, Mask);
14386 return Vals.front();
14388 return CreateShuffle(Vals.
front() ? Vals.
front()
14390 Vals.
back(), Mask);
14392 auto It = ShuffledInserts[
I].InsertElements.
rbegin();
14395 if (It != ShuffledInserts[
I].InsertElements.
rend())
14398 while (It != ShuffledInserts[
I].InsertElements.
rend()) {
14399 assert(
II &&
"Must be an insertelement instruction.");
14404 II = dyn_cast<InsertElementInst>(
II->getOperand(0));
14407 II->replaceUsesOfWith(
II->getOperand(0), NewInst);
14408 if (
auto *NewI = dyn_cast<Instruction>(NewInst))
14409 if (
II->getParent() == NewI->getParent() &&
II->comesBefore(NewI))
14410 II->moveAfter(NewI);
14413 LastInsert->replaceAllUsesWith(NewInst);
14415 IE->replaceUsesOfWith(IE->getOperand(0),
14417 IE->replaceUsesOfWith(IE->getOperand(1),
14421 CSEBlocks.
insert(LastInsert->getParent());
14426 for (
auto &TEPtr : VectorizableTree) {
14427 TreeEntry *Entry = TEPtr.get();
14430 if (Entry->isGather())
14433 assert(Entry->VectorizedValue &&
"Can't find vectorizable value");
14436 for (
int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
14437 Value *Scalar = Entry->Scalars[Lane];
14439 if (Entry->getOpcode() == Instruction::GetElementPtr &&
14440 !isa<GetElementPtrInst>(Scalar))
14443 Type *Ty = Scalar->getType();
14445 for (
User *U : Scalar->users()) {
14449 assert((getTreeEntry(U) ||
14450 (UserIgnoreList && UserIgnoreList->contains(U)) ||
14451 (isa_and_nonnull<Instruction>(U) &&
14452 isDeleted(cast<Instruction>(U)))) &&
14453 "Deleting out-of-tree value");
14457 LLVM_DEBUG(
dbgs() <<
"SLP: \tErasing scalar:" << *Scalar <<
".\n");
14458 auto *
I = cast<Instruction>(Scalar);
14465 if (
auto *V = dyn_cast<Instruction>(VectorizableTree[0]->VectorizedValue))
14466 V->mergeDIAssignID(RemovedInsts);
14469 if (UserIgnoreList) {
14471 if (getTreeEntry(
I)->
Idx != 0)
14476 bool IsPoisoningLogicalOp = isa<SelectInst>(U.getUser()) &&
14477 (match(U.getUser(), m_LogicalAnd()) ||
14478 match(U.getUser(), m_LogicalOr())) &&
14479 U.getOperandNo() == 0;
14480 if (IsPoisoningLogicalOp) {
14481 LogicalOpSelects.push_back(cast<SelectInst>(U.getUser()));
14484 return UserIgnoreList->contains(
U.getUser());
14496 removeInstructionsAndOperands(
ArrayRef(RemovedInsts));
14499 InstrElementSize.
clear();
14501 const TreeEntry &RootTE = *VectorizableTree.front();
14502 Value *Vec = RootTE.VectorizedValue;
14503 if (
auto It = MinBWs.find(&RootTE); ReductionBitWidth != 0 &&
14504 It != MinBWs.end() &&
14505 ReductionBitWidth != It->second.first) {
14508 ReductionRoot->getIterator());
14512 cast<VectorType>(Vec->
getType())->getElementCount()),
14513 It->second.second);
14520 <<
" gather sequences instructions.\n");
14527 Loop *L = LI->getLoopFor(
I->getParent());
14532 BasicBlock *PreHeader = L->getLoopPreheader();
14540 auto *OpI = dyn_cast<Instruction>(V);
14541 return OpI && L->contains(OpI);
14547 CSEBlocks.
insert(PreHeader);
14562 assert((
A ==
B) == (
A->getDFSNumIn() ==
B->getDFSNumIn()) &&
14563 "Different nodes should have different DFS numbers");
14564 return A->getDFSNumIn() <
B->getDFSNumIn();
14574 if (I1->getType() != I2->getType())
14576 auto *SI1 = dyn_cast<ShuffleVectorInst>(I1);
14577 auto *SI2 = dyn_cast<ShuffleVectorInst>(I2);
14579 return I1->isIdenticalTo(I2);
14580 if (SI1->isIdenticalTo(SI2))
14582 for (
int I = 0, E = SI1->getNumOperands();
I < E; ++
I)
14583 if (SI1->getOperand(
I) != SI2->getOperand(
I))
14586 NewMask.
assign(SI2->getShuffleMask().begin(), SI2->getShuffleMask().end());
14590 unsigned LastUndefsCnt = 0;
14591 for (
int I = 0, E = NewMask.
size();
I < E; ++
I) {
14597 NewMask[
I] != SM1[
I])
14600 NewMask[
I] = SM1[
I];
14604 return SM1.
size() - LastUndefsCnt > 1 &&
14608 SM1.
size() - LastUndefsCnt));
14614 for (
auto I = CSEWorkList.
begin(), E = CSEWorkList.
end();
I != E; ++
I) {
14617 "Worklist not sorted properly!");
14623 if (!isa<InsertElementInst, ExtractElementInst, ShuffleVectorInst>(&In) &&
14624 !GatherShuffleExtractSeq.contains(&In))
14629 bool Replaced =
false;
14632 if (IsIdenticalOrLessDefined(&In, V, NewMask) &&
14633 DT->
dominates(V->getParent(), In.getParent())) {
14634 In.replaceAllUsesWith(V);
14636 if (
auto *SI = dyn_cast<ShuffleVectorInst>(V))
14637 if (!NewMask.
empty())
14638 SI->setShuffleMask(NewMask);
14642 if (isa<ShuffleVectorInst>(In) && isa<ShuffleVectorInst>(V) &&
14643 GatherShuffleExtractSeq.contains(V) &&
14644 IsIdenticalOrLessDefined(V, &In, NewMask) &&
14645 DT->
dominates(In.getParent(), V->getParent())) {
14647 V->replaceAllUsesWith(&In);
14649 if (
auto *SI = dyn_cast<ShuffleVectorInst>(&In))
14650 if (!NewMask.
empty())
14651 SI->setShuffleMask(NewMask);
14659 Visited.push_back(&In);
14664 GatherShuffleExtractSeq.clear();
14667BoUpSLP::ScheduleData *
14669 ScheduleData *Bundle =
nullptr;
14670 ScheduleData *PrevInBundle =
nullptr;
14671 for (
Value *V : VL) {
14674 ScheduleData *BundleMember = getScheduleData(V);
14676 "no ScheduleData for bundle member "
14677 "(maybe not in same basic block)");
14678 assert(BundleMember->isSchedulingEntity() &&
14679 "bundle member already part of other bundle");
14680 if (PrevInBundle) {
14681 PrevInBundle->NextInBundle = BundleMember;
14683 Bundle = BundleMember;
14687 BundleMember->FirstInBundle = Bundle;
14688 PrevInBundle = BundleMember;
14690 assert(Bundle &&
"Failed to find schedule bundle");
14696std::optional<BoUpSLP::ScheduleData *>
14698 const InstructionsState &S) {
14709 auto TryScheduleBundleImpl = [
this, OldScheduleEnd, SLP](
bool ReSchedule,
14710 ScheduleData *Bundle) {
14716 if (ScheduleEnd != OldScheduleEnd) {
14717 for (
auto *
I = ScheduleStart;
I != ScheduleEnd;
I =
I->getNextNode())
14718 doForAllOpcodes(
I, [](ScheduleData *SD) { SD->clearDependencies(); });
14723 <<
" in block " << BB->
getName() <<
"\n");
14724 calculateDependencies(Bundle,
true, SLP);
14729 initialFillReadyList(ReadyInsts);
14736 while (((!Bundle && ReSchedule) || (Bundle && !Bundle->isReady())) &&
14737 !ReadyInsts.empty()) {
14738 ScheduleData *Picked = ReadyInsts.pop_back_val();
14739 assert(Picked->isSchedulingEntity() && Picked->isReady() &&
14740 "must be ready to schedule");
14741 schedule(Picked, ReadyInsts);
14747 for (
Value *V : VL) {
14750 if (!extendSchedulingRegion(V, S)) {
14757 TryScheduleBundleImpl(
false,
nullptr);
14758 return std::nullopt;
14762 bool ReSchedule =
false;
14763 for (
Value *V : VL) {
14766 ScheduleData *BundleMember = getScheduleData(V);
14768 "no ScheduleData for bundle member (maybe not in same basic block)");
14772 ReadyInsts.remove(BundleMember);
14774 if (!BundleMember->IsScheduled)
14779 LLVM_DEBUG(
dbgs() <<
"SLP: reset schedule because " << *BundleMember
14780 <<
" was already scheduled\n");
14784 auto *Bundle = buildBundle(VL);
14785 TryScheduleBundleImpl(ReSchedule, Bundle);
14786 if (!Bundle->isReady()) {
14787 cancelScheduling(VL, S.OpValue);
14788 return std::nullopt;
14801 ScheduleData *Bundle = getScheduleData(OpValue);
14802 LLVM_DEBUG(
dbgs() <<
"SLP: cancel scheduling of " << *Bundle <<
"\n");
14803 assert(!Bundle->IsScheduled &&
14804 "Can't cancel bundle which is already scheduled");
14805 assert(Bundle->isSchedulingEntity() &&
14807 "tried to unbundle something which is not a bundle");
14810 if (Bundle->isReady())
14811 ReadyInsts.remove(Bundle);
14814 ScheduleData *BundleMember = Bundle;
14815 while (BundleMember) {
14816 assert(BundleMember->FirstInBundle == Bundle &&
"corrupt bundle links");
14817 BundleMember->FirstInBundle = BundleMember;
14818 ScheduleData *Next = BundleMember->NextInBundle;
14819 BundleMember->NextInBundle =
nullptr;
14820 BundleMember->TE =
nullptr;
14821 if (BundleMember->unscheduledDepsInBundle() == 0) {
14822 ReadyInsts.insert(BundleMember);
14824 BundleMember = Next;
14828BoUpSLP::ScheduleData *BoUpSLP::BlockScheduling::allocateScheduleDataChunks() {
14830 if (ChunkPos >= ChunkSize) {
14831 ScheduleDataChunks.push_back(std::make_unique<ScheduleData[]>(ChunkSize));
14834 return &(ScheduleDataChunks.back()[ChunkPos++]);
14837bool BoUpSLP::BlockScheduling::extendSchedulingRegion(
Value *V,
14838 const InstructionsState &S) {
14839 if (getScheduleData(V,
isOneOf(S, V)))
14842 assert(
I &&
"bundle member must be an instruction");
14845 "phi nodes/insertelements/extractelements/extractvalues don't need to "
14847 auto &&CheckScheduleForI = [
this, &S](
Instruction *
I) ->
bool {
14848 ScheduleData *ISD = getScheduleData(
I);
14851 assert(isInSchedulingRegion(ISD) &&
14852 "ScheduleData not in scheduling region");
14853 ScheduleData *SD = allocateScheduleDataChunks();
14855 SD->init(SchedulingRegionID, S.OpValue);
14856 ExtraScheduleDataMap[
I][S.OpValue] = SD;
14859 if (CheckScheduleForI(
I))
14861 if (!ScheduleStart) {
14863 initScheduleData(
I,
I->getNextNode(),
nullptr,
nullptr);
14865 ScheduleEnd =
I->getNextNode();
14867 CheckScheduleForI(
I);
14868 assert(ScheduleEnd &&
"tried to vectorize a terminator?");
14869 LLVM_DEBUG(
dbgs() <<
"SLP: initialize schedule region to " << *
I <<
"\n");
14877 ++ScheduleStart->getIterator().getReverse();
14882 if (
auto *
II = dyn_cast<IntrinsicInst>(&
I))
14883 return II->isAssumeLikeIntrinsic();
14886 UpIter = std::find_if_not(UpIter, UpperEnd, IsAssumeLikeIntr);
14887 DownIter = std::find_if_not(DownIter, LowerEnd, IsAssumeLikeIntr);
14888 while (UpIter != UpperEnd && DownIter != LowerEnd && &*UpIter !=
I &&
14890 if (++ScheduleRegionSize > ScheduleRegionSizeLimit) {
14891 LLVM_DEBUG(
dbgs() <<
"SLP: exceeded schedule region size limit\n");
14898 UpIter = std::find_if_not(UpIter, UpperEnd, IsAssumeLikeIntr);
14899 DownIter = std::find_if_not(DownIter, LowerEnd, IsAssumeLikeIntr);
14901 if (DownIter == LowerEnd || (UpIter != UpperEnd && &*UpIter ==
I)) {
14902 assert(
I->getParent() == ScheduleStart->getParent() &&
14903 "Instruction is in wrong basic block.");
14904 initScheduleData(
I, ScheduleStart,
nullptr, FirstLoadStoreInRegion);
14907 CheckScheduleForI(
I);
14912 assert((UpIter == UpperEnd || (DownIter != LowerEnd && &*DownIter ==
I)) &&
14913 "Expected to reach top of the basic block or instruction down the "
14915 assert(
I->getParent() == ScheduleEnd->getParent() &&
14916 "Instruction is in wrong basic block.");
14917 initScheduleData(ScheduleEnd,
I->getNextNode(), LastLoadStoreInRegion,
14919 ScheduleEnd =
I->getNextNode();
14921 CheckScheduleForI(
I);
14922 assert(ScheduleEnd &&
"tried to vectorize a terminator?");
14923 LLVM_DEBUG(
dbgs() <<
"SLP: extend schedule region end to " << *
I <<
"\n");
14927void BoUpSLP::BlockScheduling::initScheduleData(
Instruction *FromI,
14929 ScheduleData *PrevLoadStore,
14930 ScheduleData *NextLoadStore) {
14931 ScheduleData *CurrentLoadStore = PrevLoadStore;
14936 ScheduleData *SD = ScheduleDataMap.lookup(
I);
14938 SD = allocateScheduleDataChunks();
14939 ScheduleDataMap[
I] = SD;
14942 assert(!isInSchedulingRegion(SD) &&
14943 "new ScheduleData already in scheduling region");
14944 SD->init(SchedulingRegionID,
I);
14946 if (
I->mayReadOrWriteMemory() &&
14947 (!isa<IntrinsicInst>(
I) ||
14948 (cast<IntrinsicInst>(
I)->getIntrinsicID() != Intrinsic::sideeffect &&
14949 cast<IntrinsicInst>(
I)->getIntrinsicID() !=
14950 Intrinsic::pseudoprobe))) {
14952 if (CurrentLoadStore) {
14953 CurrentLoadStore->NextLoadStore = SD;
14955 FirstLoadStoreInRegion = SD;
14957 CurrentLoadStore = SD;
14960 if (
match(
I, m_Intrinsic<Intrinsic::stacksave>()) ||
14961 match(
I, m_Intrinsic<Intrinsic::stackrestore>()))
14962 RegionHasStackSave =
true;
14964 if (NextLoadStore) {
14965 if (CurrentLoadStore)
14966 CurrentLoadStore->NextLoadStore = NextLoadStore;
14968 LastLoadStoreInRegion = CurrentLoadStore;
14972void BoUpSLP::BlockScheduling::calculateDependencies(ScheduleData *SD,
14973 bool InsertInReadyList,
14975 assert(SD->isSchedulingEntity());
14980 while (!WorkList.
empty()) {
14982 for (ScheduleData *BundleMember = SD; BundleMember;
14983 BundleMember = BundleMember->NextInBundle) {
14984 assert(isInSchedulingRegion(BundleMember));
14985 if (BundleMember->hasValidDependencies())
14990 BundleMember->Dependencies = 0;
14991 BundleMember->resetUnscheduledDeps();
14994 if (BundleMember->OpValue != BundleMember->Inst) {
14995 if (ScheduleData *UseSD = getScheduleData(BundleMember->Inst)) {
14996 BundleMember->Dependencies++;
14997 ScheduleData *DestBundle = UseSD->FirstInBundle;
14998 if (!DestBundle->IsScheduled)
14999 BundleMember->incrementUnscheduledDeps(1);
15000 if (!DestBundle->hasValidDependencies())
15004 for (
User *U : BundleMember->Inst->
users()) {
15005 if (ScheduleData *UseSD = getScheduleData(cast<Instruction>(U))) {
15006 BundleMember->Dependencies++;
15007 ScheduleData *DestBundle = UseSD->FirstInBundle;
15008 if (!DestBundle->IsScheduled)
15009 BundleMember->incrementUnscheduledDeps(1);
15010 if (!DestBundle->hasValidDependencies())
15017 auto *DepDest = getScheduleData(
I);
15018 assert(DepDest &&
"must be in schedule window");
15019 DepDest->ControlDependencies.push_back(BundleMember);
15020 BundleMember->Dependencies++;
15021 ScheduleData *DestBundle = DepDest->FirstInBundle;
15022 if (!DestBundle->IsScheduled)
15023 BundleMember->incrementUnscheduledDeps(1);
15024 if (!DestBundle->hasValidDependencies())
15032 for (
Instruction *
I = BundleMember->Inst->getNextNode();
15033 I != ScheduleEnd;
I =
I->getNextNode()) {
15038 MakeControlDependent(
I);
15046 if (RegionHasStackSave) {
15050 if (
match(BundleMember->Inst, m_Intrinsic<Intrinsic::stacksave>()) ||
15051 match(BundleMember->Inst, m_Intrinsic<Intrinsic::stackrestore>())) {
15052 for (
Instruction *
I = BundleMember->Inst->getNextNode();
15053 I != ScheduleEnd;
I =
I->getNextNode()) {
15054 if (
match(
I, m_Intrinsic<Intrinsic::stacksave>()) ||
15055 match(
I, m_Intrinsic<Intrinsic::stackrestore>()))
15060 if (!isa<AllocaInst>(
I))
15064 MakeControlDependent(
I);
15073 if (isa<AllocaInst>(BundleMember->Inst) ||
15074 BundleMember->Inst->mayReadOrWriteMemory()) {
15075 for (
Instruction *
I = BundleMember->Inst->getNextNode();
15076 I != ScheduleEnd;
I =
I->getNextNode()) {
15077 if (!
match(
I, m_Intrinsic<Intrinsic::stacksave>()) &&
15078 !
match(
I, m_Intrinsic<Intrinsic::stackrestore>()))
15082 MakeControlDependent(
I);
15089 ScheduleData *DepDest = BundleMember->NextLoadStore;
15094 "NextLoadStore list for non memory effecting bundle?");
15096 bool SrcMayWrite = BundleMember->Inst->mayWriteToMemory();
15097 unsigned NumAliased = 0;
15098 unsigned DistToSrc = 1;
15100 for (; DepDest; DepDest = DepDest->NextLoadStore) {
15101 assert(isInSchedulingRegion(DepDest));
15111 ((SrcMayWrite || DepDest->Inst->mayWriteToMemory()) &&
15113 SLP->isAliased(SrcLoc, SrcInst, DepDest->Inst)))) {
15120 DepDest->MemoryDependencies.push_back(BundleMember);
15121 BundleMember->Dependencies++;
15122 ScheduleData *DestBundle = DepDest->FirstInBundle;
15123 if (!DestBundle->IsScheduled) {
15124 BundleMember->incrementUnscheduledDeps(1);
15126 if (!DestBundle->hasValidDependencies()) {
15149 if (InsertInReadyList && SD->isReady()) {
15150 ReadyInsts.insert(SD);
15157void BoUpSLP::BlockScheduling::resetSchedule() {
15159 "tried to reset schedule on block which has not been scheduled");
15160 for (
Instruction *
I = ScheduleStart;
I != ScheduleEnd;
I =
I->getNextNode()) {
15161 doForAllOpcodes(
I, [&](ScheduleData *SD) {
15162 assert(isInSchedulingRegion(SD) &&
15163 "ScheduleData not in scheduling region");
15164 SD->IsScheduled =
false;
15165 SD->resetUnscheduledDeps();
15168 ReadyInsts.clear();
15171void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
15172 if (!BS->ScheduleStart)
15175 LLVM_DEBUG(
dbgs() <<
"SLP: schedule block " << BS->BB->getName() <<
"\n");
15182 BS->resetSchedule();
15189 struct ScheduleDataCompare {
15190 bool operator()(ScheduleData *SD1, ScheduleData *SD2)
const {
15191 return SD2->SchedulingPriority < SD1->SchedulingPriority;
15194 std::set<ScheduleData *, ScheduleDataCompare> ReadyInsts;
15199 for (
auto *
I = BS->ScheduleStart;
I != BS->ScheduleEnd;
15200 I =
I->getNextNode()) {
15201 BS->doForAllOpcodes(
I, [
this, &
Idx, BS](ScheduleData *SD) {
15202 TreeEntry *SDTE = getTreeEntry(SD->Inst);
15205 SD->isPartOfBundle() ==
15207 "scheduler and vectorizer bundle mismatch");
15208 SD->FirstInBundle->SchedulingPriority =
Idx++;
15210 if (SD->isSchedulingEntity() && SD->isPartOfBundle())
15211 BS->calculateDependencies(SD,
false,
this);
15214 BS->initialFillReadyList(ReadyInsts);
15216 Instruction *LastScheduledInst = BS->ScheduleEnd;
15219 while (!ReadyInsts.empty()) {
15220 ScheduleData *Picked = *ReadyInsts.begin();
15221 ReadyInsts.erase(ReadyInsts.begin());
15225 for (ScheduleData *BundleMember = Picked; BundleMember;
15226 BundleMember = BundleMember->NextInBundle) {
15230 LastScheduledInst = PickedInst;
15233 BS->schedule(Picked, ReadyInsts);
15237#ifdef EXPENSIVE_CHECKS
15241#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
15243 for (
auto *
I = BS->ScheduleStart;
I != BS->ScheduleEnd;
I =
I->getNextNode()) {
15244 BS->doForAllOpcodes(
I, [&](ScheduleData *SD) {
15245 if (SD->isSchedulingEntity() && SD->hasValidDependencies()) {
15246 assert(SD->IsScheduled &&
"must be scheduled at this point");
15253 BS->ScheduleStart =
nullptr;
15260 if (
auto *Store = dyn_cast<StoreInst>(V))
15261 return DL->getTypeSizeInBits(Store->getValueOperand()->getType());
15263 if (
auto *IEI = dyn_cast<InsertElementInst>(V))
15266 auto E = InstrElementSize.
find(V);
15267 if (E != InstrElementSize.
end())
15276 if (
auto *
I = dyn_cast<Instruction>(V)) {
15284 Value *FirstNonBool =
nullptr;
15285 while (!Worklist.
empty()) {
15290 auto *Ty =
I->getType();
15291 if (isa<VectorType>(Ty))
15293 if (Ty != Builder.
getInt1Ty() && !FirstNonBool)
15300 if (isa<LoadInst, ExtractElementInst, ExtractValueInst>(
I))
15301 Width = std::max<unsigned>(Width,
DL->getTypeSizeInBits(Ty));
15309 for (
Use &U :
I->operands()) {
15310 if (
auto *J = dyn_cast<Instruction>(U.get()))
15311 if (Visited.
insert(J).second &&
15312 (isa<PHINode>(
I) || J->getParent() == Parent)) {
15316 if (!FirstNonBool && U.get()->getType() != Builder.
getInt1Ty())
15317 FirstNonBool = U.get();
15328 if (V->getType() == Builder.
getInt1Ty() && FirstNonBool)
15330 Width =
DL->getTypeSizeInBits(V->getType());
15334 InstrElementSize[
I] = Width;
15339bool BoUpSLP::collectValuesToDemote(
15340 const TreeEntry &E,
bool IsProfitableToDemoteRoot,
unsigned &
BitWidth,
15342 unsigned &MaxDepthLevel,
bool &IsProfitableToDemote,
15343 bool IsTruncRoot)
const {
15345 if (
all_of(E.Scalars, IsaPred<Constant>))
15348 unsigned OrigBitWidth =
DL->getTypeSizeInBits(E.Scalars.front()->getType());
15357 bool IsSignedNode =
any_of(E.Scalars, [&](
Value *R) {
15358 return !isKnownNonNegative(R, SimplifyQuery(*DL));
15360 auto IsPotentiallyTruncated = [&](
Value *V,
unsigned &
BitWidth) ->
bool {
15367 if ((!IsSignedNode || IsSignedVal) && OrigBitWidth >
BitWidth) {
15373 unsigned BitWidth1 = OrigBitWidth - NumSignBits;
15376 if (
auto *
I = dyn_cast<Instruction>(V)) {
15378 unsigned BitWidth2 =
15379 std::max<unsigned>(1, Mask.getBitWidth() - Mask.countl_zero());
15380 while (!IsSignedNode && BitWidth2 < OrigBitWidth) {
15386 BitWidth1 = std::min(BitWidth1, BitWidth2);
15391 using namespace std::placeholders;
15392 auto FinalAnalysis = [&]() {
15393 if (!IsProfitableToDemote)
15396 E.Scalars, std::bind(IsPotentiallyTruncated, _1, std::ref(
BitWidth)));
15398 if (Res && E.isGather()) {
15402 for (
Value *V : E.Scalars) {
15403 auto *EE = dyn_cast<ExtractElementInst>(V);
15406 UniqueBases.
insert(EE->getVectorOperand());
15408 const unsigned VF = E.Scalars.size();
15409 Type *OrigScalarTy = E.Scalars.front()->getType();
15410 if (UniqueBases.
size() <= 2 ||
15418 if (E.isGather() || !Visited.
insert(&E).second ||
15420 return all_of(V->users(), [&](User *U) {
15421 return isa<InsertElementInst>(U) && !getTreeEntry(U);
15424 return FinalAnalysis();
15427 return !all_of(V->users(), [=](User *U) {
15428 return getTreeEntry(U) ||
15429 (UserIgnoreList && UserIgnoreList->contains(U)) ||
15430 (!isa<CmpInst>(U) && U->getType()->isSized() &&
15431 !U->getType()->isScalableTy() &&
15432 DL->getTypeSizeInBits(U->getType()) <= BitWidth);
15433 }) && !IsPotentiallyTruncated(V,
BitWidth);
15438 bool &NeedToExit) {
15439 NeedToExit =
false;
15440 unsigned InitLevel = MaxDepthLevel;
15442 unsigned Level = InitLevel;
15443 if (!collectValuesToDemote(*
Op, IsProfitableToDemoteRoot,
BitWidth,
15444 ToDemote, Visited, Level, IsProfitableToDemote,
15446 if (!IsProfitableToDemote)
15449 if (!FinalAnalysis())
15453 MaxDepthLevel = std::max(MaxDepthLevel, Level);
15457 auto AttemptCheckBitwidth =
15460 NeedToExit =
false;
15461 unsigned BestFailBitwidth = 0;
15463 if (Checker(
BitWidth, OrigBitWidth))
15465 if (BestFailBitwidth == 0 && FinalAnalysis())
15469 if (BestFailBitwidth == 0) {
15480 auto TryProcessInstruction =
15487 (void)
for_each(E.Scalars, std::bind(IsPotentiallyTruncated, _1,
15492 if (E.UserTreeIndices.size() > 1 &&
15493 !
all_of(E.Scalars, std::bind(IsPotentiallyTruncated, _1,
15496 bool NeedToExit =
false;
15497 if (Checker && !AttemptCheckBitwidth(Checker, NeedToExit))
15501 if (!ProcessOperands(
Operands, NeedToExit))
15510 return IsProfitableToDemote;
15512 switch (E.getOpcode()) {
15516 case Instruction::Trunc:
15517 if (IsProfitableToDemoteRoot)
15518 IsProfitableToDemote =
true;
15519 return TryProcessInstruction(
BitWidth);
15520 case Instruction::ZExt:
15521 case Instruction::SExt:
15522 IsProfitableToDemote =
true;
15523 return TryProcessInstruction(
BitWidth);
15527 case Instruction::Add:
15528 case Instruction::Sub:
15529 case Instruction::Mul:
15530 case Instruction::And:
15531 case Instruction::Or:
15532 case Instruction::Xor: {
15533 return TryProcessInstruction(
15534 BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)});
15536 case Instruction::Shl: {
15541 auto *I = cast<Instruction>(V);
15542 KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL);
15543 return AmtKnownBits.getMaxValue().ult(BitWidth);
15546 return TryProcessInstruction(
15547 BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)}, ShlChecker);
15549 case Instruction::LShr: {
15553 auto LShrChecker = [&](
unsigned BitWidth,
unsigned OrigBitWidth) {
15555 auto *I = cast<Instruction>(V);
15556 KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL);
15557 APInt ShiftedBits = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
15558 return AmtKnownBits.getMaxValue().ult(BitWidth) &&
15559 MaskedValueIsZero(I->getOperand(0), ShiftedBits,
15560 SimplifyQuery(*DL));
15563 return TryProcessInstruction(
15564 BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)},
15567 case Instruction::AShr: {
15571 auto AShrChecker = [&](
unsigned BitWidth,
unsigned OrigBitWidth) {
15573 auto *I = cast<Instruction>(V);
15574 KnownBits AmtKnownBits = computeKnownBits(I->getOperand(1), *DL);
15575 unsigned ShiftedBits = OrigBitWidth - BitWidth;
15576 return AmtKnownBits.getMaxValue().ult(BitWidth) &&
15577 ShiftedBits < ComputeNumSignBits(I->getOperand(0), *DL, 0, AC,
15581 return TryProcessInstruction(
15582 BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)},
15585 case Instruction::UDiv:
15586 case Instruction::URem: {
15588 auto Checker = [&](
unsigned BitWidth,
unsigned OrigBitWidth) {
15591 auto *I = cast<Instruction>(V);
15592 APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
15593 return MaskedValueIsZero(I->getOperand(0), Mask, SimplifyQuery(*DL)) &&
15594 MaskedValueIsZero(I->getOperand(1), Mask, SimplifyQuery(*DL));
15597 return TryProcessInstruction(
15598 BitWidth, {getOperandEntry(&E, 0), getOperandEntry(&E, 1)}, Checker);
15602 case Instruction::Select: {
15603 return TryProcessInstruction(
15604 BitWidth, {getOperandEntry(&E, 1), getOperandEntry(&E, 2)});
15609 case Instruction::PHI: {
15610 const unsigned NumOps = E.getNumOperands();
15613 std::bind(&BoUpSLP::getOperandEntry,
this, &E, _1));
15615 return TryProcessInstruction(
BitWidth, Ops);
15618 case Instruction::Call: {
15619 auto *IC = dyn_cast<IntrinsicInst>(E.getMainOp());
15623 if (
ID != Intrinsic::abs &&
ID != Intrinsic::smin &&
15624 ID != Intrinsic::smax &&
ID != Intrinsic::umin &&
ID != Intrinsic::umax)
15628 auto CompChecker = [&](
unsigned BitWidth,
unsigned OrigBitWidth) {
15631 auto *I = cast<Instruction>(V);
15632 if (ID == Intrinsic::umin || ID == Intrinsic::umax) {
15633 APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth);
15634 return MaskedValueIsZero(I->getOperand(0), Mask,
15635 SimplifyQuery(*DL)) &&
15636 MaskedValueIsZero(I->getOperand(1), Mask, SimplifyQuery(*DL));
15638 assert((
ID == Intrinsic::smin ||
ID == Intrinsic::smax) &&
15639 "Expected min/max intrinsics only.");
15640 unsigned SignBits = OrigBitWidth -
BitWidth;
15646 return SignBits <= Op0SignBits &&
15647 ((SignBits != Op0SignBits &&
15651 SignBits <= Op1SignBits &&
15652 ((SignBits != Op1SignBits &&
15657 if (
ID != Intrinsic::abs) {
15658 Operands.push_back(getOperandEntry(&E, 1));
15659 CallChecker = CompChecker;
15662 std::numeric_limits<InstructionCost::CostType>::max();
15664 unsigned VF = E.Scalars.size();
15673 if (
Cost < BestCost) {
15679 [[maybe_unused]]
bool NeedToExit;
15680 (void)AttemptCheckBitwidth(Checker, NeedToExit);
15690 return FinalAnalysis();
15697 bool IsStoreOrInsertElt =
15698 VectorizableTree.front()->getOpcode() == Instruction::Store ||
15699 VectorizableTree.front()->getOpcode() == Instruction::InsertElement;
15700 if ((IsStoreOrInsertElt || UserIgnoreList) &&
15701 ExtraBitWidthNodes.
size() <= 1 &&
15702 (!CastMaxMinBWSizes || CastMaxMinBWSizes->second == 0 ||
15703 CastMaxMinBWSizes->first / CastMaxMinBWSizes->second <= 2))
15706 unsigned NodeIdx = 0;
15707 if (IsStoreOrInsertElt && !VectorizableTree.front()->isGather())
15711 if (VectorizableTree[NodeIdx]->
isGather() ||
15712 (NodeIdx == 0 && !VectorizableTree[NodeIdx]->UserTreeIndices.
empty()) ||
15713 (NodeIdx != 0 &&
any_of(VectorizableTree[NodeIdx]->UserTreeIndices,
15716 static_cast<int>(NodeIdx);
15722 bool IsTruncRoot =
false;
15723 bool IsProfitableToDemoteRoot = !IsStoreOrInsertElt;
15725 if (NodeIdx != 0 &&
15726 VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
15727 VectorizableTree[NodeIdx]->
getOpcode() == Instruction::Trunc) {
15728 assert(IsStoreOrInsertElt &&
"Expected store/insertelement seeded graph.");
15729 IsTruncRoot =
true;
15731 IsProfitableToDemoteRoot =
true;
15736 if (AnalyzedMinBWVals.
contains(VectorizableTree[NodeIdx]->Scalars.front()))
15740 auto ComputeMaxBitWidth = [&](
const TreeEntry &E,
bool IsTopRoot,
15741 bool IsProfitableToDemoteRoot,
unsigned Opcode,
15742 unsigned Limit,
bool IsTruncRoot,
15743 bool IsSignedCmp) ->
unsigned {
15747 if (E.isGather() && IsTruncRoot && E.UserTreeIndices.size() == 1 &&
15748 E.Idx > (IsStoreOrInsertElt ? 2 : 1) &&
15750 return V->hasOneUse() || isa<Constant>(V) ||
15753 const TreeEntry *TE = getTreeEntry(U);
15754 const TreeEntry *UserTE = E.UserTreeIndices.back().UserTE;
15755 if (TE == UserTE || !TE)
15757 unsigned UserTESz = DL->getTypeSizeInBits(
15758 UserTE->Scalars.front()->getType());
15759 auto It = MinBWs.find(TE);
15760 if (It != MinBWs.end() && It->second.first > UserTESz)
15762 return DL->getTypeSizeInBits(U->getType()) > UserTESz;
15766 const TreeEntry *UserTE = E.UserTreeIndices.back().UserTE;
15767 auto It = MinBWs.
find(UserTE);
15768 if (It != MinBWs.
end())
15769 return It->second.first;
15770 unsigned MaxBitWidth =
15771 DL->getTypeSizeInBits(UserTE->Scalars.front()->getType());
15772 MaxBitWidth =
bit_ceil(MaxBitWidth);
15773 if (MaxBitWidth < 8 && MaxBitWidth > 1)
15775 return MaxBitWidth;
15778 unsigned VF = E.getVectorFactor();
15779 auto *TreeRootIT = dyn_cast<IntegerType>(E.Scalars.front()->getType());
15780 if (!TreeRootIT || !Opcode)
15784 [&](
Value *V) { return AnalyzedMinBWVals.contains(V); }))
15792 unsigned MaxBitWidth = 1u;
15800 bool IsKnownPositive = !IsSignedCmp &&
all_of(E.Scalars, [&](
Value *R) {
15801 KnownBits Known = computeKnownBits(R, *DL);
15802 return Known.isNonNegative();
15807 for (
Value *Root : E.Scalars) {
15810 unsigned BitWidth1 = NumTypeBits - NumSignBits;
15826 if (!IsKnownPositive)
15830 unsigned BitWidth2 = Mask.getBitWidth() - Mask.countl_zero();
15832 std::max<unsigned>(std::min(BitWidth1, BitWidth2), MaxBitWidth);
15835 if (MaxBitWidth < 8 && MaxBitWidth > 1)
15840 if (NumParts > 1 &&
15846 bool IsProfitableToDemote = Opcode == Instruction::Trunc ||
15847 Opcode == Instruction::SExt ||
15848 Opcode == Instruction::ZExt || NumParts > 1;
15853 unsigned MaxDepthLevel = IsTruncRoot ? Limit : 1;
15854 bool NeedToDemote = IsProfitableToDemote;
15856 if (!collectValuesToDemote(E, IsProfitableToDemoteRoot, MaxBitWidth,
15857 ToDemote, Visited, MaxDepthLevel, NeedToDemote,
15859 (MaxDepthLevel <= Limit &&
15860 !(((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
15861 (!IsTopRoot || !(IsStoreOrInsertElt || UserIgnoreList) ||
15862 DL->getTypeSizeInBits(TreeRootIT) /
15863 DL->getTypeSizeInBits(cast<Instruction>(E.Scalars.front())
15869 MaxBitWidth =
bit_ceil(MaxBitWidth);
15871 return MaxBitWidth;
15878 if (UserIgnoreList &&
15879 isa<IntegerType>(VectorizableTree.front()->Scalars.front()->getType())) {
15880 for (
Value *V : *UserIgnoreList) {
15882 auto NumTypeBits =
DL->getTypeSizeInBits(V->getType());
15883 unsigned BitWidth1 = NumTypeBits - NumSignBits;
15886 unsigned BitWidth2 = BitWidth1;
15889 BitWidth2 = Mask.getBitWidth() - Mask.countl_zero();
15891 ReductionBitWidth =
15892 std::max(std::min(BitWidth1, BitWidth2), ReductionBitWidth);
15894 if (ReductionBitWidth < 8 && ReductionBitWidth > 1)
15895 ReductionBitWidth = 8;
15897 ReductionBitWidth =
bit_ceil(ReductionBitWidth);
15899 bool IsTopRoot = NodeIdx == 0;
15900 while (NodeIdx < VectorizableTree.size() &&
15901 VectorizableTree[NodeIdx]->State == TreeEntry::Vectorize &&
15902 VectorizableTree[NodeIdx]->getOpcode() == Instruction::Trunc) {
15905 IsTruncRoot =
true;
15907 bool IsSignedCmp =
false;
15908 while (NodeIdx < VectorizableTree.size()) {
15910 unsigned Limit = 2;
15911 unsigned Opcode = VectorizableTree[NodeIdx]->getOpcode();
15913 ReductionBitWidth ==
15914 DL->getTypeSizeInBits(
15915 VectorizableTree.front()->Scalars.front()->getType()))
15917 unsigned MaxBitWidth = ComputeMaxBitWidth(
15918 *VectorizableTree[NodeIdx], IsTopRoot, IsProfitableToDemoteRoot, Opcode,
15919 Limit, IsTruncRoot, IsSignedCmp);
15920 if (ReductionBitWidth != 0 && (IsTopRoot || !RootDemotes.
empty())) {
15921 if (MaxBitWidth != 0 && ReductionBitWidth < MaxBitWidth)
15922 ReductionBitWidth =
bit_ceil(MaxBitWidth);
15923 else if (MaxBitWidth == 0)
15924 ReductionBitWidth = 0;
15927 for (
unsigned Idx : RootDemotes) {
15929 uint32_t OrigBitWidth =
DL->getTypeSizeInBits(V->getType());
15930 if (OrigBitWidth > MaxBitWidth) {
15938 RootDemotes.clear();
15940 IsProfitableToDemoteRoot =
true;
15942 if (ExtraBitWidthNodes.
empty()) {
15943 NodeIdx = VectorizableTree.size();
15945 unsigned NewIdx = 0;
15947 NewIdx = *ExtraBitWidthNodes.
begin();
15948 ExtraBitWidthNodes.
erase(ExtraBitWidthNodes.
begin());
15949 }
while (NewIdx <= NodeIdx && !ExtraBitWidthNodes.
empty());
15952 NodeIdx < VectorizableTree.size() &&
15953 any_of(VectorizableTree[NodeIdx]->UserTreeIndices,
15956 EI.
UserTE->getOpcode() == Instruction::Trunc &&
15957 !EI.
UserTE->isAltShuffle();
15960 NodeIdx < VectorizableTree.size() &&
15961 any_of(VectorizableTree[NodeIdx]->UserTreeIndices,
15963 return EI.
UserTE->getOpcode() == Instruction::ICmp &&
15965 auto *IC = dyn_cast<ICmpInst>(V);
15968 !isKnownNonNegative(IC->getOperand(0),
15969 SimplifyQuery(*DL)) ||
15970 !isKnownNonNegative(IC->getOperand(1),
15971 SimplifyQuery(*DL)));
15978 if (MaxBitWidth == 0 ||
15980 cast<IntegerType>(TreeRoot.
front()->getType())->getBitWidth()) {
15981 if (UserIgnoreList)
15988 for (
unsigned Idx : ToDemote) {
15989 TreeEntry *TE = VectorizableTree[
Idx].get();
15992 bool IsSigned =
any_of(TE->Scalars, [&](
Value *R) {
15993 return !isKnownNonNegative(R, SimplifyQuery(*DL));
16011 bool Changed =
runImpl(
F, SE,
TTI, TLI, AA, LI, DT, AC, DB, ORE);
16036 DL = &
F.getDataLayout();
16040 bool Changed =
false;
16046 dbgs() <<
"SLP: Didn't find any vector registers for target, abort.\n");
16051 if (
F.hasFnAttribute(Attribute::NoImplicitFloat))
16054 LLVM_DEBUG(
dbgs() <<
"SLP: Analyzing blocks in " <<
F.getName() <<
".\n");
16058 BoUpSLP R(&
F, SE,
TTI, TLI, AA, LI, DT, AC, DB,
DL, ORE_);
16067 for (
auto *BB :
post_order(&
F.getEntryBlock())) {
16069 R.clearReductionData();
16070 collectSeedInstructions(BB);
16073 if (!Stores.empty()) {
16075 <<
" underlying objects.\n");
16076 Changed |= vectorizeStoreChains(R);
16080 Changed |= vectorizeChainsInBlock(BB, R);
16085 if (!GEPs.
empty()) {
16087 <<
" underlying objects.\n");
16088 Changed |= vectorizeGEPIndices(BB, R);
16093 R.optimizeGatherSequence();
16101 unsigned Idx,
unsigned MinVF,
16106 const unsigned Sz = R.getVectorElementSize(Chain[0]);
16107 unsigned VF = Chain.
size();
16121 for (
Value *V : Chain)
16122 ValOps.
insert(cast<StoreInst>(V)->getValueOperand());
16125 if (
all_of(ValOps, IsaPred<Instruction>) && ValOps.
size() > 1) {
16130 if ((!IsPowerOf2 && S.getOpcode() && S.getOpcode() != Instruction::Load &&
16131 (!S.MainOp->isSafeToRemove() ||
16134 return !isa<ExtractElementInst>(V) &&
16135 (V->getNumUses() > Chain.size() ||
16136 any_of(V->users(), [&](User *U) {
16137 return !Stores.contains(U);
16140 (ValOps.
size() > Chain.size() / 2 && !S.getOpcode())) {
16141 Size = (!IsPowerOf2 && S.getOpcode()) ? 1 : 2;
16145 if (
R.isLoadCombineCandidate(Chain))
16147 R.buildTree(Chain);
16149 if (
R.isTreeTinyAndNotFullyVectorizable()) {
16150 if (
R.isGathered(Chain.front()) ||
16151 R.isNotScheduled(cast<StoreInst>(Chain.front())->getValueOperand()))
16152 return std::nullopt;
16153 Size =
R.getTreeSize();
16156 R.reorderTopToBottom();
16157 R.reorderBottomToTop();
16158 R.buildExternalUses();
16160 R.computeMinimumValueSizes();
16161 R.transformNodes();
16163 Size =
R.getTreeSize();
16164 if (S.getOpcode() == Instruction::Load)
16172 using namespace ore;
16175 cast<StoreInst>(Chain[0]))
16176 <<
"Stores SLP vectorized with cost " <<
NV(
"Cost",
Cost)
16177 <<
" and with tree size "
16178 <<
NV(
"TreeSize",
R.getTreeSize()));
16192 Sizes.begin(), Sizes.end(),
static_cast<uint64_t>(0),
16193 [&](
uint64_t V,
const std::pair<unsigned, unsigned> &Val) {
16194 unsigned Size = First ? Val.first : Val.second;
16206 Sizes.begin(), Sizes.end(),
static_cast<uint64_t>(0),
16207 [&](
uint64_t V,
const std::pair<unsigned, unsigned> &Val) {
16208 unsigned P = First ? Val.first : Val.second;
16211 return V + (P - Mean) * (P - Mean);
16214 return Dev * 81 / (Mean * Mean) == 0;
16217bool SLPVectorizerPass::vectorizeStores(
16219 DenseSet<std::tuple<Value *, Value *, Value *, Value *, unsigned>>
16224 bool Changed =
false;
16226 struct StoreDistCompare {
16227 bool operator()(
const std::pair<unsigned, int> &Op1,
16228 const std::pair<unsigned, int> &Op2)
const {
16229 return Op1.second < Op2.second;
16234 using StoreIndexToDistSet =
16235 std::set<std::pair<unsigned, int>, StoreDistCompare>;
16236 auto TryToVectorize = [&](
const StoreIndexToDistSet &
Set) {
16241 if (
Operands.empty() ||
Data.second - PrevDist == 1) {
16243 PrevDist =
Data.second;
16244 if (
Idx !=
Set.size() - 1)
16249 Operands.push_back(Stores[DataVar.first]);
16250 PrevDist = DataVar.second;
16255 .
insert({Operands.front(),
16256 cast<StoreInst>(Operands.front())->getValueOperand(),
16258 cast<StoreInst>(Operands.back())->getValueOperand(),
16263 unsigned MaxVecRegSize =
R.getMaxVecRegSize();
16264 unsigned EltSize =
R.getVectorElementSize(
Operands[0]);
16268 std::min(
R.getMaximumVF(EltSize, Instruction::Store), MaxElts);
16269 unsigned MaxRegVF = MaxVF;
16271 Type *StoreTy =
Store->getValueOperand()->getType();
16272 Type *ValueTy = StoreTy;
16273 if (
auto *Trunc = dyn_cast<TruncInst>(
Store->getValueOperand()))
16274 ValueTy = Trunc->getSrcTy();
16275 if (ValueTy == StoreTy &&
16276 R.getVectorElementSize(
Store->getValueOperand()) <= EltSize)
16278 unsigned MinVF = std::max<unsigned>(
16280 R.getMinVF(
DL->getTypeStoreSizeInBits(StoreTy)), StoreTy,
16283 if (MaxVF < MinVF) {
16284 LLVM_DEBUG(
dbgs() <<
"SLP: Vectorization infeasible as MaxVF (" << MaxVF
16286 <<
"MinVF (" << MinVF <<
")\n");
16290 unsigned NonPowerOf2VF = 0;
16295 unsigned CandVF =
Operands.size();
16297 NonPowerOf2VF = CandVF;
16302 unsigned Size = MinVF;
16304 VF =
Size > MaxVF ? NonPowerOf2VF :
Size;
16308 unsigned Repeat = 0;
16309 constexpr unsigned MaxAttempts = 4;
16311 for_each(RangeSizes, [](std::pair<unsigned, unsigned> &
P) {
16312 P.first =
P.second = 1;
16315 auto IsNotVectorized = [](
bool First,
16316 const std::pair<unsigned, unsigned> &
P) {
16317 return First ?
P.first > 0 :
P.second > 0;
16319 auto IsVectorized = [](
bool First,
16320 const std::pair<unsigned, unsigned> &
P) {
16321 return First ?
P.first == 0 :
P.second == 0;
16323 auto VFIsProfitable = [](
bool First,
unsigned Size,
16324 const std::pair<unsigned, unsigned> &
P) {
16327 auto FirstSizeSame = [](
unsigned Size,
16328 const std::pair<unsigned, unsigned> &
P) {
16329 return Size ==
P.first;
16333 bool RepeatChanged =
false;
16334 bool AnyProfitableGraph =
false;
16335 for (
unsigned Size : CandidateVFs) {
16336 AnyProfitableGraph =
false;
16337 unsigned StartIdx = std::distance(
16338 RangeSizes.begin(),
16339 find_if(RangeSizes, std::bind(IsNotVectorized,
Size >= MaxRegVF,
16340 std::placeholders::_1)));
16341 while (StartIdx <
End) {
16343 std::distance(RangeSizes.begin(),
16344 find_if(RangeSizes.drop_front(StartIdx),
16345 std::bind(IsVectorized,
Size >= MaxRegVF,
16346 std::placeholders::_1)));
16347 unsigned Sz = EndIdx >=
End ?
End : EndIdx;
16348 for (
unsigned Cnt = StartIdx; Cnt +
Size <= Sz;) {
16350 Size >= MaxRegVF)) {
16357 return cast<StoreInst>(V)
16358 ->getValueOperand()
16360 cast<StoreInst>(Slice.
front())
16361 ->getValueOperand()
16364 "Expected all operands of same type.");
16365 if (!NonSchedulable.empty()) {
16366 auto [NonSchedSizeMax, NonSchedSizeMin] =
16367 NonSchedulable.lookup(Slice.
front());
16368 if (NonSchedSizeMax > 0 && NonSchedSizeMin <=
Size) {
16369 Cnt += NonSchedSizeMax;
16374 std::optional<bool> Res =
16375 vectorizeStoreChain(Slice, R, Cnt, MinVF, TreeSize);
16379 .first->getSecond()
16387 AnyProfitableGraph = RepeatChanged = Changed =
true;
16391 [](std::pair<unsigned, unsigned> &
P) {
16392 P.first = P.second = 0;
16394 if (Cnt < StartIdx + MinVF) {
16395 for_each(RangeSizes.slice(StartIdx, Cnt - StartIdx),
16396 [](std::pair<unsigned, unsigned> &
P) {
16397 P.first = P.second = 0;
16399 StartIdx = Cnt +
Size;
16401 if (Cnt > Sz -
Size - MinVF) {
16403 [](std::pair<unsigned, unsigned> &
P) {
16404 P.first = P.second = 0;
16413 if (
Size > 2 && Res &&
16415 std::bind(VFIsProfitable,
Size >= MaxRegVF, TreeSize,
16416 std::placeholders::_1))) {
16422 if (
Size > MaxRegVF && TreeSize > 1 &&
16424 std::bind(FirstSizeSame, TreeSize,
16425 std::placeholders::_1))) {
16427 while (Cnt != Sz && RangeSizes[Cnt].first == TreeSize)
16433 [&](std::pair<unsigned, unsigned> &
P) {
16434 if (Size >= MaxRegVF)
16435 P.second = std::max(P.second, TreeSize);
16437 P.first = std::max(P.first, TreeSize);
16440 AnyProfitableGraph =
true;
16442 if (StartIdx >=
End)
16444 if (Sz - StartIdx < Size && Sz - StartIdx >= MinVF)
16445 AnyProfitableGraph =
true;
16446 StartIdx = std::distance(
16447 RangeSizes.begin(),
16448 find_if(RangeSizes.drop_front(Sz),
16449 std::bind(IsNotVectorized,
Size >= MaxRegVF,
16450 std::placeholders::_1)));
16452 if (!AnyProfitableGraph &&
Size >= MaxRegVF)
16456 if (
all_of(RangeSizes, [](
const std::pair<unsigned, unsigned> &
P) {
16457 return P.first == 0 &&
P.second == 0;
16461 if (Repeat >= MaxAttempts ||
16462 (Repeat > 1 && (RepeatChanged || !AnyProfitableGraph)))
16464 constexpr unsigned StoresLimit = 64;
16465 const unsigned MaxTotalNum =
bit_floor(std::min<unsigned>(
16467 static_cast<unsigned>(
16470 RangeSizes.begin(),
16471 find_if(RangeSizes, std::bind(IsNotVectorized,
true,
16472 std::placeholders::_1))) +
16475 if (VF > MaxTotalNum || VF >= StoresLimit)
16477 for_each(RangeSizes, [&](std::pair<unsigned, unsigned> &
P) {
16479 P.first = std::max(
P.second,
P.first);
16483 CandidateVFs.clear();
16484 CandidateVFs.push_back(VF);
16531 for (std::pair<unsigned, StoreIndexToDistSet> &Set : SortedStores) {
16533 Stores[
Set.first]->getValueOperand()->getType(),
16534 Stores[
Set.first]->getPointerOperand(),
16535 SI->getValueOperand()->getType(),
SI->getPointerOperand(), *
DL, *SE,
16539 auto It =
Set.second.find(std::make_pair(
Idx, *Diff));
16540 if (It ==
Set.second.end()) {
16541 Set.second.emplace(
Idx, *Diff);
16545 TryToVectorize(
Set.second);
16546 StoreIndexToDistSet PrevSet;
16547 PrevSet.swap(
Set.second);
16549 Set.second.emplace(
Idx, 0);
16552 unsigned StartIdx = It->first + 1;
16557 for (
const std::pair<unsigned, int> &Pair :
reverse(PrevSet)) {
16559 if (Pair.first <= It->first ||
16560 VectorizedStores.
contains(Stores[Pair.first]))
16562 unsigned BI = Pair.first - StartIdx;
16563 UsedStores.set(BI);
16564 Dists[BI] = Pair.second - It->second;
16566 for (
unsigned I = StartIdx;
I <
Idx; ++
I) {
16567 unsigned BI =
I - StartIdx;
16568 if (UsedStores.test(BI))
16569 Set.second.emplace(
I, Dists[BI]);
16573 auto &Res = SortedStores.emplace_back();
16575 Res.second.emplace(
Idx, 0);
16577 Type *PrevValTy =
nullptr;
16579 if (
R.isDeleted(SI))
16582 PrevValTy =
SI->getValueOperand()->getType();
16584 if (PrevValTy !=
SI->getValueOperand()->getType()) {
16585 for (
auto &Set : SortedStores)
16586 TryToVectorize(
Set.second);
16587 SortedStores.clear();
16588 PrevValTy =
SI->getValueOperand()->getType();
16590 FillStoresSet(
I, SI);
16594 for (
auto &Set : SortedStores)
16595 TryToVectorize(
Set.second);
16600void SLPVectorizerPass::collectSeedInstructions(
BasicBlock *BB) {
16611 if (
auto *SI = dyn_cast<StoreInst>(&
I)) {
16612 if (!
SI->isSimple())
16622 else if (
auto *
GEP = dyn_cast<GetElementPtrInst>(&
I)) {
16623 if (
GEP->getNumIndices() != 1)
16626 if (isa<Constant>(
Idx))
16630 if (
GEP->getType()->isVectorTy())
16642 LLVM_DEBUG(
dbgs() <<
"SLP: Trying to vectorize a list of length = "
16643 << VL.
size() <<
".\n");
16648 if (!S.getOpcode())
16654 for (
Value *V : VL) {
16655 Type *Ty =
V->getType();
16659 R.getORE()->emit([&]() {
16660 std::string TypeStr;
16664 <<
"Cannot SLP vectorize list: type "
16665 << TypeStr +
" is unsupported by vectorizer";
16671 unsigned Sz =
R.getVectorElementSize(I0);
16672 unsigned MinVF =
R.getMinVF(Sz);
16673 unsigned MaxVF = std::max<unsigned>(
llvm::bit_floor(VL.size()), MinVF);
16674 MaxVF = std::min(
R.getMaximumVF(Sz, S.getOpcode()), MaxVF);
16676 R.getORE()->emit([&]() {
16678 <<
"Cannot SLP vectorize list: vectorization factor "
16679 <<
"less than 2 is not supported";
16684 bool Changed =
false;
16685 bool CandidateFound =
false;
16687 Type *ScalarTy = VL[0]->getType();
16688 if (
auto *IE = dyn_cast<InsertElementInst>(VL[0]))
16689 ScalarTy =
IE->getOperand(1)->getType();
16691 unsigned NextInst = 0, MaxInst = VL.size();
16692 for (
unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF; VF /= 2) {
16699 for (
unsigned I = NextInst;
I < MaxInst; ++
I) {
16700 unsigned ActualVF = std::min(MaxInst -
I, VF);
16705 if (MaxVFOnly && ActualVF < MaxVF)
16707 if ((VF > MinVF && ActualVF <= VF / 2) || (VF == MinVF && ActualVF < 2))
16713 auto *
I = dyn_cast<Instruction>(V);
16714 return I &&
R.isDeleted(
I);
16718 LLVM_DEBUG(
dbgs() <<
"SLP: Analyzing " << ActualVF <<
" operations "
16722 if (
R.isTreeTinyAndNotFullyVectorizable())
16724 R.reorderTopToBottom();
16725 R.reorderBottomToTop(
16726 !isa<InsertElementInst>(Ops.
front()) &&
16727 !
R.doesRootHaveInTreeUses());
16728 R.buildExternalUses();
16730 R.computeMinimumValueSizes();
16731 R.transformNodes();
16733 CandidateFound =
true;
16734 MinCost = std::min(MinCost,
Cost);
16737 <<
" for VF=" << ActualVF <<
"\n");
16741 cast<Instruction>(Ops[0]))
16742 <<
"SLP vectorized with cost " <<
ore::NV(
"Cost",
Cost)
16743 <<
" and with tree size "
16744 <<
ore::NV(
"TreeSize",
R.getTreeSize()));
16755 if (!Changed && CandidateFound) {
16756 R.getORE()->emit([&]() {
16758 <<
"List vectorization was possible but not beneficial with cost "
16759 <<
ore::NV(
"Cost", MinCost) <<
" >= "
16762 }
else if (!Changed) {
16763 R.getORE()->emit([&]() {
16765 <<
"Cannot SLP vectorize list: vectorization was impossible"
16766 <<
" with available vectorization factors";
16776 if (!isa<BinaryOperator, CmpInst>(
I) || isa<VectorType>(
I->getType()))
16782 auto *Op0 = dyn_cast<Instruction>(
I->getOperand(0));
16783 auto *Op1 = dyn_cast<Instruction>(
I->getOperand(1));
16784 if (!Op0 || !Op1 || Op0->getParent() !=
P || Op1->getParent() !=
P)
16791 auto *
A = dyn_cast<BinaryOperator>(Op0);
16792 auto *
B = dyn_cast<BinaryOperator>(Op1);
16794 if (
A &&
B &&
B->hasOneUse()) {
16795 auto *B0 = dyn_cast<BinaryOperator>(
B->getOperand(0));
16796 auto *B1 = dyn_cast<BinaryOperator>(
B->getOperand(1));
16797 if (B0 && B0->getParent() ==
P)
16799 if (B1 && B1->getParent() ==
P)
16803 if (
B &&
A &&
A->hasOneUse()) {
16804 auto *A0 = dyn_cast<BinaryOperator>(
A->getOperand(0));
16805 auto *A1 = dyn_cast<BinaryOperator>(
A->getOperand(1));
16806 if (A0 && A0->getParent() ==
P)
16808 if (A1 && A1->getParent() ==
P)
16812 if (Candidates.
size() == 1)
16813 return tryToVectorizeList({Op0, Op1},
R);
16816 std::optional<int> BestCandidate =
R.findBestRootPair(Candidates);
16817 if (!BestCandidate)
16819 return tryToVectorizeList(
16820 {Candidates[*BestCandidate].first, Candidates[*BestCandidate].second},
R);
16854 ReductionOpsListType ReductionOps;
16864 bool IsSupportedHorRdxIdentityOp =
false;
16875 return isa<SelectInst>(
I) &&
16881 if (Kind == RecurKind::None)
16889 if (Kind == RecurKind::FMax || Kind == RecurKind::FMin) {
16893 return I->getFastMathFlags().noNaNs();
16896 if (Kind == RecurKind::FMaximum || Kind == RecurKind::FMinimum)
16899 return I->isAssociative();
16908 return I->getOperand(2);
16909 return I->getOperand(
Index);
16917 case RecurKind::Or:
16923 case RecurKind::And:
16929 case RecurKind::Add:
16930 case RecurKind::Mul:
16931 case RecurKind::Xor:
16932 case RecurKind::FAdd:
16933 case RecurKind::FMul:
16936 case RecurKind::FMax:
16938 case RecurKind::FMin:
16940 case RecurKind::FMaximum:
16942 case RecurKind::FMinimum:
16944 case RecurKind::SMax:
16950 case RecurKind::SMin:
16956 case RecurKind::UMax:
16962 case RecurKind::UMin:
16977 const ReductionOpsListType &ReductionOps) {
16978 bool UseSelect = ReductionOps.size() == 2 ||
16980 (ReductionOps.size() == 1 &&
16981 any_of(ReductionOps.front(), IsaPred<SelectInst>));
16982 assert((!UseSelect || ReductionOps.size() != 2 ||
16983 isa<SelectInst>(ReductionOps[1][0])) &&
16984 "Expected cmp + select pairs for reduction");
16987 if (
auto *Sel = dyn_cast<SelectInst>(
Op)) {
17001 auto *
I = dyn_cast<Instruction>(V);
17003 return RecurKind::None;
17005 return RecurKind::Add;
17007 return RecurKind::Mul;
17010 return RecurKind::And;
17013 return RecurKind::Or;
17015 return RecurKind::Xor;
17017 return RecurKind::FAdd;
17019 return RecurKind::FMul;
17022 return RecurKind::FMax;
17024 return RecurKind::FMin;
17027 return RecurKind::FMaximum;
17029 return RecurKind::FMinimum;
17035 return RecurKind::SMax;
17037 return RecurKind::SMin;
17039 return RecurKind::UMax;
17041 return RecurKind::UMin;
17043 if (
auto *
Select = dyn_cast<SelectInst>(
I)) {
17065 if (!isa<ExtractElementInst>(
RHS) ||
17067 return RecurKind::None;
17069 if (!isa<ExtractElementInst>(
LHS) ||
17071 return RecurKind::None;
17073 if (!isa<ExtractElementInst>(
LHS) || !isa<ExtractElementInst>(
RHS))
17074 return RecurKind::None;
17078 return RecurKind::None;
17083 return RecurKind::None;
17086 return RecurKind::SMax;
17089 return RecurKind::SMin;
17092 return RecurKind::UMax;
17095 return RecurKind::UMin;
17098 return RecurKind::None;
17102 static unsigned getFirstOperandIndex(
Instruction *
I) {
17103 return isCmpSelMinMax(
I) ? 1 : 0;
17109 return isCmpSelMinMax(
I) ? 3 : 2;
17115 if (isCmpSelMinMax(
I) || isBoolLogicOp(
I)) {
17116 auto *Sel = cast<SelectInst>(
I);
17117 auto *
Cmp = dyn_cast<Instruction>(Sel->getCondition());
17118 return Sel->getParent() == BB &&
Cmp &&
Cmp->getParent() == BB;
17120 return I->getParent() == BB;
17124 static bool hasRequiredNumberOfUses(
bool IsCmpSelMinMax,
Instruction *
I) {
17125 if (IsCmpSelMinMax) {
17128 if (
auto *Sel = dyn_cast<SelectInst>(
I))
17129 return Sel->
hasNUses(2) && Sel->getCondition()->hasOneUse();
17130 return I->hasNUses(2);
17134 return I->hasOneUse();
17139 if (isCmpSelMinMax(
I))
17140 ReductionOps.assign(2, ReductionOpsType());
17142 ReductionOps.assign(1, ReductionOpsType());
17147 if (isCmpSelMinMax(
I)) {
17148 ReductionOps[0].emplace_back(cast<SelectInst>(
I)->getCondition());
17149 ReductionOps[1].emplace_back(
I);
17151 ReductionOps[0].emplace_back(
I);
17156 int Sz = Data.size();
17157 auto *
I = dyn_cast<Instruction>(Data.front());
17158 return Sz > 1 ||
isConstant(Data.front()) ||
17169 RdxKind = HorizontalReduction::getRdxKind(Root);
17170 if (!isVectorizable(RdxKind, Root))
17181 if (
auto *Sel = dyn_cast<SelectInst>(Root))
17182 if (!Sel->getCondition()->hasOneUse())
17185 ReductionRoot = Root;
17190 bool IsCmpSelMinMax = isCmpSelMinMax(Root);
17192 1, std::make_pair(Root, 0));
17200 for (
int I :
reverse(seq<int>(getFirstOperandIndex(TreeN),
17201 getNumberOfOperands(TreeN)))) {
17202 Value *EdgeVal = getRdxOperand(TreeN,
I);
17203 ReducedValsToOps[EdgeVal].push_back(TreeN);
17204 auto *EdgeInst = dyn_cast<Instruction>(EdgeVal);
17211 IsCmpSelMinMax != isCmpSelMinMax(EdgeInst) ||
17212 !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst) ||
17213 !isVectorizable(RdxKind, EdgeInst) ||
17214 (
R.isAnalyzedReductionRoot(EdgeInst) &&
17215 all_of(EdgeInst->operands(), IsaPred<Constant>))) {
17216 PossibleReducedVals.push_back(EdgeVal);
17219 ReductionOps.push_back(EdgeInst);
17228 PossibleReducedVals;
17229 initReductionOps(Root);
17233 auto GenerateLoadsSubkey = [&](
size_t Key,
LoadInst *LI) {
17237 auto LIt = LoadsMap.
find(
Ptr);
17238 if (LIt != LoadsMap.
end()) {
17239 for (
LoadInst *RLI : LIt->second) {
17245 for (
LoadInst *RLI : LIt->second) {
17252 if (LIt->second.size() > 2) {
17254 hash_value(LIt->second.back()->getPointerOperand());
17259 LoadKeyUsed.
insert(Key);
17264 while (!Worklist.empty()) {
17265 auto [TreeN, Level] = Worklist.pop_back_val();
17268 CheckOperands(TreeN, PossibleRedVals, PossibleReductionOps, Level);
17269 addReductionOps(TreeN);
17272 for (
Value *V : PossibleRedVals) {
17276 ++PossibleReducedVals[
Key][
Idx]
17277 .
insert(std::make_pair(V, 0))
17281 Worklist.emplace_back(
I,
I->getParent() == BB ? 0 : Level + 1);
17283 auto PossibleReducedValsVect = PossibleReducedVals.
takeVector();
17286 for (
auto &PossibleReducedVals : PossibleReducedValsVect) {
17287 auto PossibleRedVals = PossibleReducedVals.second.
takeVector();
17289 for (
auto It = PossibleRedVals.begin(),
E = PossibleRedVals.end();
17292 auto RedValsVect = It->second.takeVector();
17294 for (
const std::pair<Value *, unsigned> &Data : RedValsVect)
17295 PossibleRedValsVect.
back().append(Data.second, Data.first);
17297 stable_sort(PossibleRedValsVect, [](
const auto &P1,
const auto &P2) {
17298 return P1.size() > P2.size();
17303 (!isGoodForReduction(Data) &&
17304 (!isa<LoadInst>(Data.front()) ||
17305 !isa<LoadInst>(ReducedVals[NewIdx].front()) ||
17307 cast<LoadInst>(Data.front())->getPointerOperand()) !=
17309 cast<LoadInst>(ReducedVals[NewIdx].front())
17311 NewIdx = ReducedVals.
size();
17314 ReducedVals[NewIdx].
append(Data.rbegin(), Data.rend());
17328 constexpr int ReductionLimit = 4;
17329 constexpr unsigned RegMaxNumber = 4;
17330 constexpr unsigned RedValsMaxNumber = 128;
17334 unsigned NumReducedVals =
17335 std::accumulate(ReducedVals.
begin(), ReducedVals.
end(), 0,
17337 if (!isGoodForReduction(Vals))
17339 return Num + Vals.size();
17341 if (NumReducedVals < ReductionLimit &&
17346 for (ReductionOpsType &RdxOps : ReductionOps)
17347 for (
Value *RdxOp : RdxOps)
17348 V.analyzedReductionRoot(cast<Instruction>(RdxOp));
17359 ReducedVals.
front().size());
17364 auto &&GetCmpForMinMaxReduction = [](
Instruction *RdxRootInst) {
17365 assert(isa<SelectInst>(RdxRootInst) &&
17366 "Expected min/max reduction to have select root instruction");
17367 Value *ScalarCond = cast<SelectInst>(RdxRootInst)->getCondition();
17368 assert(isa<Instruction>(ScalarCond) &&
17369 "Expected min/max reduction to have compare condition");
17370 return cast<Instruction>(ScalarCond);
17374 auto GetNewVectorizedTree = [&](
Value *VectorizedTree,
Value *Res) {
17375 if (VectorizedTree) {
17378 cast<Instruction>(ReductionOps.front().front())->getDebugLoc());
17379 if ((isa<PoisonValue>(VectorizedTree) && !isa<PoisonValue>(Res)) ||
17382 auto It = ReducedValsToOps.
find(Res);
17383 if (It != ReducedValsToOps.
end() &&
17389 return createOp(Builder, RdxKind, VectorizedTree, Res,
"op.rdx",
17395 bool AnyBoolLogicOp =
any_of(ReductionOps.back(), [](
Value *V) {
17396 return isBoolLogicOp(cast<Instruction>(V));
17399 ReductionOps.front().size());
17400 for (ReductionOpsType &RdxOps : ReductionOps)
17401 for (
Value *RdxOp : RdxOps) {
17404 IgnoreList.insert(RdxOp);
17409 for (
Value *U : IgnoreList)
17410 if (
auto *FPMO = dyn_cast<FPMathOperator>(U))
17411 RdxFMF &= FPMO->getFastMathFlags();
17412 bool IsCmpSelMinMax = isCmpSelMinMax(cast<Instruction>(ReductionRoot));
17417 for (
Value *V : Candidates)
17418 TrackedVals.try_emplace(V, V);
17424 Value *VectorizedTree =
nullptr;
17425 bool CheckForReusedReductionOps =
false;
17430 for (
unsigned I = 0,
E = ReducedVals.
size();
I <
E; ++
I) {
17432 InstructionsState S = States[
I];
17436 for (
unsigned Cnt = 0, Sz = OrigReducedVals.
size(); Cnt < Sz; ++Cnt) {
17437 Value *RdxVal = TrackedVals.find(OrigReducedVals[Cnt])->second;
17442 auto *Inst = dyn_cast<Instruction>(RdxVal);
17444 (!S.getOpcode() || !S.isOpcodeOrAlt(Inst))) ||
17445 (S.getOpcode() && !Inst))
17448 TrackedToOrig.try_emplace(RdxVal, OrigReducedVals[Cnt]);
17450 bool ShuffledExtracts =
false;
17452 if (S.getOpcode() == Instruction::ExtractElement && !S.isAltShuffle() &&
17455 for (
Value *RV : ReducedVals[
I + 1]) {
17456 Value *RdxVal = TrackedVals.find(RV)->second;
17460 auto *Inst = dyn_cast<ExtractElementInst>(RdxVal);
17463 CommonCandidates.push_back(RdxVal);
17464 TrackedToOrig.try_emplace(RdxVal, RV);
17469 Candidates.
swap(CommonCandidates);
17470 ShuffledExtracts =
true;
17478 ++VectorizedVals.try_emplace(Candidates.
front(), 0).first->getSecond();
17480 Res = createOp(Builder, RdxKind, Res, VC,
"const.rdx", ReductionOps);
17481 ++VectorizedVals.try_emplace(VC, 0).first->getSecond();
17482 if (
auto *ResI = dyn_cast<Instruction>(Res))
17483 V.analyzedReductionRoot(ResI);
17485 VectorizedTree = GetNewVectorizedTree(VectorizedTree, Res);
17489 unsigned NumReducedVals = Candidates.
size();
17490 if (NumReducedVals < ReductionLimit &&
17497 IsSupportedHorRdxIdentityOp =
17499 RdxKind != RecurKind::FMul && RdxKind != RecurKind::FMulAdd;
17502 if (IsSupportedHorRdxIdentityOp)
17503 for (
Value *V : Candidates)
17504 ++SameValuesCounter.
insert(std::make_pair(V, 0)).first->second;
17515 bool SameScaleFactor =
false;
17516 bool OptReusedScalars = IsSupportedHorRdxIdentityOp &&
17517 SameValuesCounter.
size() != Candidates.size();
17518 if (OptReusedScalars) {
17520 (RdxKind == RecurKind::Add || RdxKind == RecurKind::FAdd ||
17521 RdxKind == RecurKind::Xor) &&
17523 [&SameValuesCounter](
const std::pair<Value *, unsigned> &
P) {
17524 return P.second == SameValuesCounter.
front().second;
17526 Candidates.resize(SameValuesCounter.
size());
17527 transform(SameValuesCounter, Candidates.begin(),
17528 [](
const auto &
P) { return P.first; });
17529 NumReducedVals = Candidates.size();
17531 if (NumReducedVals == 1) {
17532 Value *OrigV = TrackedToOrig.find(Candidates.front())->second;
17533 unsigned Cnt = SameValuesCounter.
lookup(OrigV);
17535 emitScaleForReusedOps(Candidates.front(), Builder, Cnt);
17536 VectorizedTree = GetNewVectorizedTree(VectorizedTree, RedVal);
17537 VectorizedVals.try_emplace(OrigV, Cnt);
17542 unsigned MaxVecRegSize =
V.getMaxVecRegSize();
17543 unsigned EltSize =
V.getVectorElementSize(Candidates[0]);
17547 unsigned ReduxWidth = std::min<unsigned>(
17549 std::clamp<unsigned>(MaxElts, RedValsMaxNumber,
17550 RegMaxNumber * RedValsMaxNumber));
17551 unsigned Start = 0;
17552 unsigned Pos = Start;
17554 unsigned PrevReduxWidth = ReduxWidth;
17555 bool CheckForReusedReductionOpsLocal =
false;
17556 auto &&AdjustReducedVals = [&Pos, &Start, &ReduxWidth, NumReducedVals,
17557 &CheckForReusedReductionOpsLocal,
17558 &PrevReduxWidth, &
V,
17559 &IgnoreList](
bool IgnoreVL =
false) {
17560 bool IsAnyRedOpGathered = !IgnoreVL &&
V.isAnyGathered(IgnoreList);
17561 if (!CheckForReusedReductionOpsLocal && PrevReduxWidth == ReduxWidth) {
17564 CheckForReusedReductionOpsLocal |= IsAnyRedOpGathered;
17567 if (Pos < NumReducedVals - ReduxWidth + 1)
17568 return IsAnyRedOpGathered;
17571 return IsAnyRedOpGathered;
17573 bool AnyVectorized =
false;
17574 while (Pos < NumReducedVals - ReduxWidth + 1 &&
17575 ReduxWidth >= ReductionLimit) {
17578 if (CheckForReusedReductionOpsLocal && PrevReduxWidth != ReduxWidth &&
17580 CheckForReusedReductionOps =
true;
17583 PrevReduxWidth = ReduxWidth;
17586 if (
V.areAnalyzedReductionVals(VL)) {
17587 (void)AdjustReducedVals(
true);
17593 auto *RedValI = dyn_cast<Instruction>(RedVal);
17596 return V.isDeleted(RedValI);
17599 V.buildTree(VL, IgnoreList);
17600 if (
V.isTreeTinyAndNotFullyVectorizable(
true)) {
17601 if (!AdjustReducedVals())
17602 V.analyzedReductionVals(VL);
17605 if (
V.isLoadCombineReductionCandidate(RdxKind)) {
17606 if (!AdjustReducedVals())
17607 V.analyzedReductionVals(VL);
17610 V.reorderTopToBottom();
17612 V.reorderBottomToTop(
true);
17619 LocalExternallyUsedValues[ReductionRoot];
17620 for (
unsigned Cnt = 0, Sz = ReducedVals.
size(); Cnt < Sz; ++Cnt) {
17621 if (Cnt ==
I || (ShuffledExtracts && Cnt ==
I - 1))
17623 for (
Value *V : ReducedVals[Cnt])
17624 if (isa<Instruction>(V))
17625 LocalExternallyUsedValues[TrackedVals[V]];
17627 if (!IsSupportedHorRdxIdentityOp) {
17630 "Reused values counter map is not empty");
17631 for (
unsigned Cnt = 0; Cnt < NumReducedVals; ++Cnt) {
17632 if (Cnt >= Pos && Cnt < Pos + ReduxWidth)
17634 Value *
V = Candidates[Cnt];
17635 Value *OrigV = TrackedToOrig.find(V)->second;
17636 ++SameValuesCounter[OrigV];
17642 for (
unsigned Cnt = 0; Cnt < NumReducedVals; ++Cnt) {
17643 if (Cnt >= Pos && Cnt < Pos + ReduxWidth)
17645 Value *RdxVal = Candidates[Cnt];
17646 if (!Visited.
insert(RdxVal).second)
17650 if (!VLScalars.contains(RdxVal) &&
V.isVectorized(RdxVal)) {
17651 LocalExternallyUsedValues[RdxVal];
17654 Value *OrigV = TrackedToOrig.find(RdxVal)->second;
17656 VectorizedVals.lookup(RdxVal) + SameValuesCounter[OrigV];
17657 if (NumOps != ReducedValsToOps.
find(OrigV)->second.size())
17658 LocalExternallyUsedValues[RdxVal];
17661 if (!IsSupportedHorRdxIdentityOp)
17662 SameValuesCounter.
clear();
17663 for (
Value *RdxVal : VL)
17664 if (RequiredExtract.
contains(RdxVal))
17665 LocalExternallyUsedValues[RdxVal];
17666 V.buildExternalUses(LocalExternallyUsedValues);
17668 V.computeMinimumValueSizes();
17669 V.transformNodes();
17674 getReductionCost(
TTI, VL, IsCmpSelMinMax, ReduxWidth, RdxFMF);
17677 <<
" for reduction\n");
17681 V.getORE()->emit([&]() {
17683 SV_NAME,
"HorSLPNotBeneficial",
17684 ReducedValsToOps.
find(VL[0])->second.front())
17685 <<
"Vectorizing horizontal reduction is possible "
17686 <<
"but not beneficial with cost " <<
ore::NV(
"Cost",
Cost)
17687 <<
" and threshold "
17690 if (!AdjustReducedVals())
17691 V.analyzedReductionVals(VL);
17695 LLVM_DEBUG(
dbgs() <<
"SLP: Vectorizing horizontal reduction at cost:"
17696 <<
Cost <<
". (HorRdx)\n");
17697 V.getORE()->emit([&]() {
17699 SV_NAME,
"VectorizedHorizontalReduction",
17700 ReducedValsToOps.
find(VL[0])->second.front())
17701 <<
"Vectorized horizontal reduction with cost "
17702 <<
ore::NV(
"Cost",
Cost) <<
" and with tree size "
17703 <<
ore::NV(
"TreeSize",
V.getTreeSize());
17710 Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
17712 if (IsCmpSelMinMax)
17713 InsertPt = GetCmpForMinMaxReduction(RdxRootInst);
17716 Value *VectorizedRoot =
V.vectorizeTree(LocalExternallyUsedValues,
17717 ReplacedExternals, InsertPt);
17724 if ((isBoolLogicOp(RdxRootInst) ||
17725 (AnyBoolLogicOp && VL.size() != TrackedVals.size())) &&
17727 VectorizedRoot = Builder.
CreateFreeze(VectorizedRoot);
17730 if (OptReusedScalars && !SameScaleFactor) {
17731 VectorizedRoot = emitReusedOps(VectorizedRoot, Builder, V,
17732 SameValuesCounter, TrackedToOrig);
17735 Value *ReducedSubTree =
17736 emitReduction(VectorizedRoot, Builder, ReduxWidth,
TTI);
17737 if (ReducedSubTree->
getType() != VL.front()->getType()) {
17738 assert(ReducedSubTree->
getType() != VL.front()->getType() &&
17739 "Expected different reduction type.");
17741 Builder.
CreateIntCast(ReducedSubTree, VL.front()->getType(),
17742 V.isSignedMinBitwidthRootNode());
17748 if (OptReusedScalars && SameScaleFactor)
17749 ReducedSubTree = emitScaleForReusedOps(
17750 ReducedSubTree, Builder, SameValuesCounter.
front().second);
17752 VectorizedTree = GetNewVectorizedTree(VectorizedTree, ReducedSubTree);
17754 for (
Value *RdxVal : VL) {
17755 Value *OrigV = TrackedToOrig.find(RdxVal)->second;
17756 if (IsSupportedHorRdxIdentityOp) {
17757 VectorizedVals.try_emplace(OrigV, SameValuesCounter[RdxVal]);
17760 ++VectorizedVals.try_emplace(OrigV, 0).first->getSecond();
17761 if (!
V.isVectorized(RdxVal))
17762 RequiredExtract.
insert(RdxVal);
17767 AnyVectorized =
true;
17769 if (OptReusedScalars && !AnyVectorized) {
17770 for (
const std::pair<Value *, unsigned> &
P : SameValuesCounter) {
17771 Value *RedVal = emitScaleForReusedOps(
P.first, Builder,
P.second);
17772 VectorizedTree = GetNewVectorizedTree(VectorizedTree, RedVal);
17773 Value *OrigV = TrackedToOrig.find(
P.first)->second;
17774 VectorizedVals.try_emplace(OrigV,
P.second);
17779 if (VectorizedTree) {
17800 if (!AnyBoolLogicOp)
17802 if (isBoolLogicOp(RedOp1) &&
17803 ((!InitStep &&
LHS == VectorizedTree) ||
17806 if (isBoolLogicOp(RedOp2) && ((!InitStep &&
RHS == VectorizedTree) ||
17807 getRdxOperand(RedOp2, 0) ==
RHS ||
17812 if (
LHS != VectorizedTree)
17823 unsigned Sz = InstVals.
size();
17826 for (
unsigned I = 0,
E = (Sz / 2) * 2;
I <
E;
I += 2) {
17829 Value *RdxVal1 = InstVals[
I].second;
17830 Value *StableRdxVal1 = RdxVal1;
17831 auto It1 = TrackedVals.find(RdxVal1);
17832 if (It1 != TrackedVals.end())
17833 StableRdxVal1 = It1->second;
17834 Value *RdxVal2 = InstVals[
I + 1].second;
17835 Value *StableRdxVal2 = RdxVal2;
17836 auto It2 = TrackedVals.find(RdxVal2);
17837 if (It2 != TrackedVals.end())
17838 StableRdxVal2 = It2->second;
17842 FixBoolLogicalOps(StableRdxVal1, StableRdxVal2, InstVals[
I].first,
17844 Value *ExtraRed = createOp(Builder, RdxKind, StableRdxVal1,
17845 StableRdxVal2,
"op.rdx", ReductionOps);
17846 ExtraReds[
I / 2] = std::make_pair(InstVals[
I].first, ExtraRed);
17849 ExtraReds[Sz / 2] = InstVals.
back();
17853 ExtraReductions.
emplace_back(cast<Instruction>(ReductionRoot),
17857 for (
Value *RdxVal : Candidates) {
17858 if (!Visited.
insert(RdxVal).second)
17860 unsigned NumOps = VectorizedVals.lookup(RdxVal);
17868 bool InitStep =
true;
17869 while (ExtraReductions.
size() > 1) {
17871 FinalGen(ExtraReductions, InitStep);
17872 ExtraReductions.
swap(NewReds);
17875 VectorizedTree = ExtraReductions.
front().second;
17877 ReductionRoot->replaceAllUsesWith(VectorizedTree);
17886 IgnoreSet.
insert(RdxOps.begin(), RdxOps.end());
17893 for (
auto *U :
Ignore->users()) {
17895 "All users must be either in the reduction ops list.");
17898 if (!
Ignore->use_empty()) {
17900 Ignore->replaceAllUsesWith(
P);
17903 V.removeInstructionsAndOperands(RdxOps);
17905 }
else if (!CheckForReusedReductionOps) {
17906 for (ReductionOpsType &RdxOps : ReductionOps)
17907 for (
Value *RdxOp : RdxOps)
17908 V.analyzedReductionRoot(cast<Instruction>(RdxOp));
17910 return VectorizedTree;
17917 bool IsCmpSelMinMax,
unsigned ReduxWidth,
17920 Type *ScalarTy = ReducedVals.
front()->getType();
17929 int Cnt = ReducedVals.
size();
17930 for (
Value *RdxVal : ReducedVals) {
17935 Cost += GenCostFn();
17940 auto *RdxOp = cast<Instruction>(U);
17941 if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
17949 Cost += ScalarCost;
17951 Cost += GenCostFn();
17956 case RecurKind::Add:
17957 case RecurKind::Mul:
17958 case RecurKind::Or:
17959 case RecurKind::And:
17960 case RecurKind::Xor:
17961 case RecurKind::FAdd:
17962 case RecurKind::FMul: {
17967 ScalarCost = EvaluateScalarCost([&]() {
17972 case RecurKind::FMax:
17973 case RecurKind::FMin:
17974 case RecurKind::FMaximum:
17975 case RecurKind::FMinimum:
17976 case RecurKind::SMax:
17977 case RecurKind::SMin:
17978 case RecurKind::UMax:
17979 case RecurKind::UMin: {
17983 ScalarCost = EvaluateScalarCost([&]() {
17993 LLVM_DEBUG(
dbgs() <<
"SLP: Adding cost " << VectorCost - ScalarCost
17995 <<
" (It is a splitting reduction)\n");
17996 return VectorCost - ScalarCost;
18002 assert(VectorizedValue &&
"Need to have a vectorized tree node");
18004 "We only handle power-of-two reductions for now");
18005 assert(RdxKind != RecurKind::FMulAdd &&
18006 "A call to the llvm.fmuladd intrinsic is not handled yet");
18008 ++NumVectorInstructions;
18015 assert(IsSupportedHorRdxIdentityOp &&
18016 "The optimization of matched scalar identity horizontal reductions "
18017 "must be supported.");
18019 return VectorizedValue;
18021 case RecurKind::Add: {
18023 Value *Scale = ConstantInt::get(VectorizedValue->
getType(), Cnt);
18025 << VectorizedValue <<
". (HorRdx)\n");
18026 return Builder.
CreateMul(VectorizedValue, Scale);
18028 case RecurKind::Xor: {
18030 LLVM_DEBUG(
dbgs() <<
"SLP: Xor " << Cnt <<
"of " << VectorizedValue
18031 <<
". (HorRdx)\n");
18034 return VectorizedValue;
18036 case RecurKind::FAdd: {
18038 Value *Scale = ConstantFP::get(VectorizedValue->
getType(), Cnt);
18040 << VectorizedValue <<
". (HorRdx)\n");
18041 return Builder.
CreateFMul(VectorizedValue, Scale);
18043 case RecurKind::And:
18044 case RecurKind::Or:
18045 case RecurKind::SMax:
18046 case RecurKind::SMin:
18047 case RecurKind::UMax:
18048 case RecurKind::UMin:
18049 case RecurKind::FMax:
18050 case RecurKind::FMin:
18051 case RecurKind::FMaximum:
18052 case RecurKind::FMinimum:
18054 return VectorizedValue;
18055 case RecurKind::Mul:
18056 case RecurKind::FMul:
18057 case RecurKind::FMulAdd:
18058 case RecurKind::IAnyOf:
18059 case RecurKind::FAnyOf:
18060 case RecurKind::None:
18072 assert(IsSupportedHorRdxIdentityOp &&
18073 "The optimization of matched scalar identity horizontal reductions "
18074 "must be supported.");
18076 auto *VTy = cast<FixedVectorType>(VectorizedValue->
getType());
18077 if (VTy->getElementType() != VL.
front()->getType()) {
18081 R.isSignedMinBitwidthRootNode());
18084 case RecurKind::Add: {
18087 for (
Value *V : VL) {
18088 unsigned Cnt = SameValuesCounter.
lookup(TrackedToOrig.
find(V)->second);
18089 Vals.
push_back(ConstantInt::get(
V->getType(), Cnt,
false));
18093 << VectorizedValue <<
". (HorRdx)\n");
18094 return Builder.
CreateMul(VectorizedValue, Scale);
18096 case RecurKind::And:
18097 case RecurKind::Or:
18100 <<
". (HorRdx)\n");
18101 return VectorizedValue;
18102 case RecurKind::SMax:
18103 case RecurKind::SMin:
18104 case RecurKind::UMax:
18105 case RecurKind::UMin:
18106 case RecurKind::FMax:
18107 case RecurKind::FMin:
18108 case RecurKind::FMaximum:
18109 case RecurKind::FMinimum:
18112 <<
". (HorRdx)\n");
18113 return VectorizedValue;
18114 case RecurKind::Xor: {
18120 cast<FixedVectorType>(VectorizedValue->
getType())->getNumElements(),
18122 std::iota(
Mask.begin(),
Mask.end(), 0);
18123 bool NeedShuffle =
false;
18124 for (
unsigned I = 0, VF = VL.size();
I < VF; ++
I) {
18126 unsigned Cnt = SameValuesCounter.
lookup(TrackedToOrig.
find(V)->second);
18127 if (Cnt % 2 == 0) {
18129 NeedShuffle =
true;
18135 dbgs() <<
"> of " << VectorizedValue <<
". (HorRdx)\n");
18139 ConstantVector::getNullValue(VectorizedValue->
getType()),
Mask);
18140 return VectorizedValue;
18142 case RecurKind::FAdd: {
18145 for (
Value *V : VL) {
18146 unsigned Cnt = SameValuesCounter.
lookup(TrackedToOrig.
find(V)->second);
18147 Vals.
push_back(ConstantFP::get(
V->getType(), Cnt));
18150 return Builder.
CreateFMul(VectorizedValue, Scale);
18152 case RecurKind::Mul:
18153 case RecurKind::FMul:
18154 case RecurKind::FMulAdd:
18155 case RecurKind::IAnyOf:
18156 case RecurKind::FAnyOf:
18157 case RecurKind::None:
18167 return HorizontalReduction::getRdxKind(V);
18170 if (
auto *IE = dyn_cast<InsertElementInst>(InsertInst))
18171 return cast<FixedVectorType>(IE->getType())->getNumElements();
18173 unsigned AggregateSize = 1;
18174 auto *
IV = cast<InsertValueInst>(InsertInst);
18175 Type *CurrentType =
IV->getType();
18177 if (
auto *ST = dyn_cast<StructType>(CurrentType)) {
18178 for (
auto *Elt : ST->elements())
18179 if (Elt != ST->getElementType(0))
18180 return std::nullopt;
18181 AggregateSize *= ST->getNumElements();
18182 CurrentType = ST->getElementType(0);
18183 }
else if (
auto *AT = dyn_cast<ArrayType>(CurrentType)) {
18184 AggregateSize *= AT->getNumElements();
18185 CurrentType = AT->getElementType();
18186 }
else if (
auto *VT = dyn_cast<FixedVectorType>(CurrentType)) {
18187 AggregateSize *= VT->getNumElements();
18188 return AggregateSize;
18190 return AggregateSize;
18192 return std::nullopt;
18201 unsigned OperandOffset) {
18204 std::optional<unsigned> OperandIndex =
18208 if (isa<InsertElementInst, InsertValueInst>(InsertedOperand)) {
18210 BuildVectorOpds, InsertElts, *OperandIndex);
18213 BuildVectorOpds[*OperandIndex] = InsertedOperand;
18214 InsertElts[*OperandIndex] = LastInsertInst;
18216 LastInsertInst = dyn_cast<Instruction>(LastInsertInst->
getOperand(0));
18217 }
while (LastInsertInst !=
nullptr &&
18218 isa<InsertValueInst, InsertElementInst>(LastInsertInst) &&
18241 assert((isa<InsertElementInst>(LastInsertInst) ||
18242 isa<InsertValueInst>(LastInsertInst)) &&
18243 "Expected insertelement or insertvalue instruction!");
18246 "Expected empty result vectors!");
18249 if (!AggregateSize)
18251 BuildVectorOpds.
resize(*AggregateSize);
18252 InsertElts.
resize(*AggregateSize);
18257 if (BuildVectorOpds.
size() >= 2)
18275 auto DominatedReduxValue = [&](
Value *R) {
18276 return isa<Instruction>(R) &&
18277 DT->
dominates(
P->getParent(), cast<Instruction>(R)->getParent());
18283 if (
P->getIncomingBlock(0) == ParentBB) {
18284 Rdx = dyn_cast<Instruction>(
P->getIncomingValue(0));
18285 }
else if (
P->getIncomingBlock(1) == ParentBB) {
18286 Rdx = dyn_cast<Instruction>(
P->getIncomingValue(1));
18289 if (Rdx && DominatedReduxValue(Rdx))
18302 if (
P->getIncomingBlock(0) == BBLatch) {
18303 Rdx = dyn_cast<Instruction>(
P->getIncomingValue(0));
18304 }
else if (
P->getIncomingBlock(1) == BBLatch) {
18305 Rdx = dyn_cast<Instruction>(
P->getIncomingValue(1));
18308 if (Rdx && DominatedReduxValue(Rdx))
18342 assert((isa<BinaryOperator>(Root) || isa<SelectInst>(Root) ||
18343 isa<IntrinsicInst>(Root)) &&
18344 "Expected binop, select, or intrinsic for reduction matching");
18346 Root->
getOperand(HorizontalReduction::getFirstOperandIndex(Root));
18348 Root->
getOperand(HorizontalReduction::getFirstOperandIndex(Root) + 1);
18350 return dyn_cast<Instruction>(
RHS);
18352 return dyn_cast<Instruction>(
LHS);
18359 Value *Op0 =
nullptr;
18360 Value *Op1 =
nullptr;
18363 return dyn_cast<Instruction>(Op0 == Phi ? Op1 : Op0);
18369 Value *B0 =
nullptr, *B1 =
nullptr;
18374bool SLPVectorizerPass::vectorizeHorReduction(
18379 bool TryOperandsAsNewSeeds =
P && isa<BinaryOperator>(Root);
18381 if (Root->
getParent() != BB || isa<PHINode>(Root))
18385 auto SelectRoot = [&]() {
18404 std::queue<std::pair<Instruction *, unsigned>>
Stack;
18405 Stack.emplace(SelectRoot(), 0);
18409 if (
R.isAnalyzedReductionRoot(Inst))
18414 if (!HorRdx.matchAssociativeReduction(R, Inst, *SE, *
DL, *TLI))
18416 return HorRdx.tryToReduce(R, *
DL,
TTI, *TLI);
18418 auto TryAppendToPostponedInsts = [&](
Instruction *FutureSeed) {
18419 if (TryOperandsAsNewSeeds && FutureSeed == Root) {
18426 if (!isa<CmpInst, InsertElementInst, InsertValueInst>(FutureSeed))
18431 while (!
Stack.empty()) {
18434 std::tie(Inst, Level) =
Stack.front();
18439 if (
R.isDeleted(Inst))
18441 if (
Value *VectorizedV = TryToReduce(Inst)) {
18443 if (
auto *
I = dyn_cast<Instruction>(VectorizedV)) {
18445 Stack.emplace(
I, Level);
18448 if (
R.isDeleted(Inst))
18452 if (!TryAppendToPostponedInsts(Inst)) {
18463 if (VisitedInstrs.
insert(
Op).second)
18464 if (
auto *
I = dyn_cast<Instruction>(
Op))
18467 if (!isa<PHINode, CmpInst, InsertElementInst, InsertValueInst>(
I) &&
18468 !
R.isDeleted(
I) &&
I->getParent() == BB)
18469 Stack.emplace(
I, Level);
18478 bool Res = vectorizeHorReduction(
P, Root, BB, R,
TTI, PostponedInsts);
18479 Res |= tryToVectorize(PostponedInsts, R);
18486 for (
Value *V : Insts)
18487 if (
auto *Inst = dyn_cast<Instruction>(V); Inst && !
R.isDeleted(Inst))
18488 Res |= tryToVectorize(Inst, R);
18492bool SLPVectorizerPass::vectorizeInsertValueInst(
InsertValueInst *IVI,
18495 if (!
R.canMapToVector(IVI->
getType()))
18503 if (MaxVFOnly && BuildVectorOpds.
size() == 2) {
18504 R.getORE()->emit([&]() {
18506 <<
"Cannot SLP vectorize list: only 2 elements of buildvalue, "
18507 "trying reduction first.";
18511 LLVM_DEBUG(
dbgs() <<
"SLP: array mappable to vector: " << *IVI <<
"\n");
18513 return tryToVectorizeList(BuildVectorOpds, R, MaxVFOnly);
18523 (
llvm::all_of(BuildVectorOpds, IsaPred<ExtractElementInst, UndefValue>) &&
18527 if (MaxVFOnly && BuildVectorInsts.
size() == 2) {
18528 R.getORE()->emit([&]() {
18530 <<
"Cannot SLP vectorize list: only 2 elements of buildvector, "
18531 "trying reduction first.";
18535 LLVM_DEBUG(
dbgs() <<
"SLP: array mappable to vector: " << *IEI <<
"\n");
18536 return tryToVectorizeList(BuildVectorInsts, R, MaxVFOnly);
18539template <
typename T>
18544 bool MaxVFOnly,
BoUpSLP &R) {
18545 bool Changed =
false;
18556 auto *
I = dyn_cast<Instruction>(*IncIt);
18557 if (!
I || R.isDeleted(
I)) {
18561 auto *SameTypeIt = IncIt;
18562 while (SameTypeIt != E && (!isa<Instruction>(*SameTypeIt) ||
18563 R.isDeleted(cast<Instruction>(*SameTypeIt)) ||
18564 AreCompatible(*SameTypeIt, *IncIt))) {
18565 auto *
I = dyn_cast<Instruction>(*SameTypeIt);
18567 if (
I && !R.isDeleted(
I))
18572 unsigned NumElts = VL.
size();
18573 LLVM_DEBUG(
dbgs() <<
"SLP: Trying to vectorize starting at nodes ("
18574 << NumElts <<
")\n");
18584 if (NumElts > 1 && TryToVectorizeHelper(
ArrayRef(VL), MaxVFOnly)) {
18587 VL.
swap(Candidates);
18588 Candidates.
clear();
18590 if (
auto *
I = dyn_cast<Instruction>(V);
I && !R.isDeleted(
I))
18596 auto GetMinNumElements = [&R](
Value *V) {
18597 unsigned EltSize = R.getVectorElementSize(V);
18598 return std::max(2U, R.getMaxVecRegSize() / EltSize);
18600 if (NumElts < GetMinNumElements(*IncIt) &&
18601 (Candidates.
empty() ||
18602 Candidates.
front()->getType() == (*IncIt)->getType())) {
18604 if (
auto *
I = dyn_cast<Instruction>(V);
I && !R.isDeleted(
I))
18610 if (Candidates.
size() > 1 &&
18611 (SameTypeIt == E || (*SameTypeIt)->getType() != (*IncIt)->getType())) {
18612 if (TryToVectorizeHelper(Candidates,
false)) {
18615 }
else if (MaxVFOnly) {
18618 for (
auto *It = Candidates.
begin(), *
End = Candidates.
end(); It !=
End;
18620 auto *
I = dyn_cast<Instruction>(*It);
18621 if (!
I || R.isDeleted(
I)) {
18625 auto *SameTypeIt = It;
18626 while (SameTypeIt !=
End &&
18627 (!isa<Instruction>(*SameTypeIt) ||
18628 R.isDeleted(cast<Instruction>(*SameTypeIt)) ||
18629 AreCompatible(*SameTypeIt, *It))) {
18630 auto *
I = dyn_cast<Instruction>(*SameTypeIt);
18632 if (
I && !R.isDeleted(
I))
18635 unsigned NumElts = VL.
size();
18636 if (NumElts > 1 && TryToVectorizeHelper(
ArrayRef(VL),
18642 Candidates.
clear();
18646 IncIt = SameTypeIt;
18658template <
bool IsCompatibility>
18663 "Expected valid element types only.");
18665 return IsCompatibility;
18666 auto *CI1 = cast<CmpInst>(V);
18667 auto *CI2 = cast<CmpInst>(V2);
18668 if (CI1->getOperand(0)->getType()->getTypeID() <
18670 return !IsCompatibility;
18671 if (CI1->getOperand(0)->getType()->getTypeID() >
18674 if (CI1->getOperand(0)->getType()->getScalarSizeInBits() <
18676 return !IsCompatibility;
18677 if (CI1->getOperand(0)->getType()->getScalarSizeInBits() >
18686 if (BasePred1 < BasePred2)
18687 return !IsCompatibility;
18688 if (BasePred1 > BasePred2)
18691 bool CI1Preds = Pred1 == BasePred1;
18692 bool CI2Preds = Pred2 == BasePred1;
18693 for (
int I = 0, E = CI1->getNumOperands();
I < E; ++
I) {
18694 auto *Op1 = CI1->getOperand(CI1Preds ?
I : E -
I - 1);
18695 auto *Op2 = CI2->
getOperand(CI2Preds ?
I : E -
I - 1);
18699 return !IsCompatibility;
18702 if (
auto *I1 = dyn_cast<Instruction>(Op1))
18703 if (
auto *I2 = dyn_cast<Instruction>(Op2)) {
18704 if (IsCompatibility) {
18705 if (I1->getParent() != I2->getParent())
18712 return NodeI2 !=
nullptr;
18715 assert((NodeI1 == NodeI2) ==
18717 "Different nodes should have different DFS numbers");
18718 if (NodeI1 != NodeI2)
18722 if (S.getOpcode() && (IsCompatibility || !S.isAltShuffle()))
18724 if (IsCompatibility)
18726 if (I1->getOpcode() != I2->getOpcode())
18727 return I1->getOpcode() < I2->getOpcode();
18730 return IsCompatibility;
18733template <
typename ItT>
18736 bool Changed =
false;
18739 if (
R.isDeleted(
I))
18742 if (
auto *RootOp = dyn_cast<Instruction>(
Op))
18743 Changed |= vectorizeRootInstruction(
nullptr, RootOp, BB, R,
TTI);
18747 if (
R.isDeleted(
I))
18749 Changed |= tryToVectorize(
I, R);
18756 return compareCmp<false>(V, V2, *TLI, *DT);
18759 auto AreCompatibleCompares = [&](
Value *V1,
Value *
V2) {
18762 return compareCmp<true>(V1, V2, *TLI, *DT);
18769 if (Vals.
size() <= 1)
18771 Changed |= tryToVectorizeSequence<Value>(
18772 Vals, CompareSorter, AreCompatibleCompares,
18775 bool ArePossiblyReducedInOtherBlock =
any_of(Candidates, [](
Value *V) {
18777 auto *Select = dyn_cast<SelectInst>(U);
18779 Select->getParent() != cast<Instruction>(V)->getParent();
18782 if (ArePossiblyReducedInOtherBlock)
18784 return tryToVectorizeList(Candidates, R, MaxVFOnly);
18790bool SLPVectorizerPass::vectorizeInserts(InstSetVector &Instructions,
18792 assert(
all_of(Instructions, IsaPred<InsertElementInst, InsertValueInst>) &&
18793 "This function only accepts Insert instructions");
18794 bool OpsChanged =
false;
18796 for (
auto *
I :
reverse(Instructions)) {
18798 if (
R.isDeleted(
I) || isa<CmpInst>(
I))
18800 if (
auto *LastInsertValue = dyn_cast<InsertValueInst>(
I)) {
18802 vectorizeInsertValueInst(LastInsertValue, BB, R,
true);
18803 }
else if (
auto *LastInsertElem = dyn_cast<InsertElementInst>(
I)) {
18805 vectorizeInsertElementInst(LastInsertElem, BB, R,
true);
18808 if (
R.isDeleted(
I))
18810 OpsChanged |= vectorizeHorReduction(
nullptr,
I, BB, R,
TTI, PostponedInsts);
18811 if (
R.isDeleted(
I) || isa<CmpInst>(
I))
18814 if (
auto *LastInsertValue = dyn_cast<InsertValueInst>(
I)) {
18816 vectorizeInsertValueInst(LastInsertValue, BB, R,
false);
18817 }
else if (
auto *LastInsertElem = dyn_cast<InsertElementInst>(
I)) {
18818 OpsChanged |= vectorizeInsertElementInst(LastInsertElem, BB, R,
18823 OpsChanged |= tryToVectorize(PostponedInsts, R);
18830 bool Changed =
false;
18837 auto PHICompare = [
this, &PHIToOpcodes](
Value *V1,
Value *
V2) {
18840 "Expected vectorizable types only.");
18848 V2->getType()->getScalarSizeInBits())
18851 V2->getType()->getScalarSizeInBits())
18855 if (Opcodes1.
size() < Opcodes2.
size())
18857 if (Opcodes1.
size() > Opcodes2.
size())
18859 for (
int I = 0, E = Opcodes1.
size();
I < E; ++
I) {
18862 auto *
I1 = dyn_cast<Instruction>(Opcodes1[
I]);
18863 auto *I2 = dyn_cast<Instruction>(Opcodes2[
I]);
18868 return NodeI2 !=
nullptr;
18871 assert((NodeI1 == NodeI2) ==
18873 "Different nodes should have different DFS numbers");
18874 if (NodeI1 != NodeI2)
18877 if (S.getOpcode() && !S.isAltShuffle())
18879 return I1->getOpcode() < I2->getOpcode();
18888 bool C1 = isa<Constant>(Opcodes1[
I]) && !isa<UndefValue>(Opcodes1[
I]);
18889 bool C2 = isa<Constant>(Opcodes2[
I]) && !isa<UndefValue>(Opcodes2[
I]);
18897 bool U1 = isa<UndefValue>(Opcodes1[
I]);
18898 bool U2 = isa<UndefValue>(Opcodes2[
I]);
18902 auto ValID1 = Opcodes1[
I]->getValueID();
18903 auto ValID2 = Opcodes2[
I]->getValueID();
18904 if (ValID1 == ValID2)
18906 if (ValID1 < ValID2)
18908 if (ValID1 > ValID2)
18917 assert(U1 && U2 &&
"The only thing left should be undef & undef.");
18922 auto AreCompatiblePHIs = [&PHIToOpcodes,
this, &
R](
Value *V1,
Value *
V2) {
18925 if (V1->getType() !=
V2->getType())
18929 if (Opcodes1.
size() != Opcodes2.
size())
18931 for (
int I = 0, E = Opcodes1.
size();
I < E; ++
I) {
18933 if (isa<UndefValue>(Opcodes1[
I]) || isa<UndefValue>(Opcodes2[
I]))
18935 if (
auto *I1 = dyn_cast<Instruction>(Opcodes1[
I]))
18936 if (
auto *I2 = dyn_cast<Instruction>(Opcodes2[
I])) {
18937 if (
R.isDeleted(I1) ||
R.isDeleted(I2))
18939 if (
I1->getParent() != I2->getParent())
18946 if (isa<Constant>(Opcodes1[
I]) && isa<Constant>(Opcodes2[
I]))
18948 if (Opcodes1[
I]->getValueID() != Opcodes2[
I]->getValueID())
18954 bool HaveVectorizedPhiNodes =
false;
18959 auto *
P = dyn_cast<PHINode>(&
I);
18965 if (!VisitedInstrs.
count(
P) && !
R.isDeleted(
P) &&
18978 if (!Opcodes.
empty())
18982 while (!Nodes.empty()) {
18983 auto *
PHI = cast<PHINode>(Nodes.pop_back_val());
18986 for (
Value *V :
PHI->incoming_values()) {
18987 if (
auto *PHI1 = dyn_cast<PHINode>((V))) {
18988 Nodes.push_back(PHI1);
18996 HaveVectorizedPhiNodes = tryToVectorizeSequence<Value>(
18997 Incoming, PHICompare, AreCompatiblePHIs,
18999 return tryToVectorizeList(Candidates, R, MaxVFOnly);
19002 Changed |= HaveVectorizedPhiNodes;
19003 if (HaveVectorizedPhiNodes &&
any_of(PHIToOpcodes, [&](
const auto &
P) {
19004 auto *
PHI = dyn_cast<PHINode>(
P.first);
19005 return !
PHI ||
R.isDeleted(
PHI);
19007 PHIToOpcodes.
clear();
19009 }
while (HaveVectorizedPhiNodes);
19011 VisitedInstrs.
clear();
19013 InstSetVector PostProcessInserts;
19017 auto VectorizeInsertsAndCmps = [&](
bool VectorizeCmps) {
19018 bool Changed = vectorizeInserts(PostProcessInserts, BB, R);
19019 if (VectorizeCmps) {
19020 Changed |= vectorizeCmpInsts(
reverse(PostProcessCmps), BB, R);
19021 PostProcessCmps.
clear();
19023 PostProcessInserts.clear();
19028 if (
auto *Cmp = dyn_cast<CmpInst>(
I))
19029 return PostProcessCmps.
contains(Cmp);
19030 return isa<InsertElementInst, InsertValueInst>(
I) &&
19031 PostProcessInserts.contains(
I);
19037 return I->use_empty() &&
19038 (
I->getType()->isVoidTy() || isa<CallInst, InvokeInst>(
I));
19043 if (isa<ScalableVectorType>(It->getType()))
19047 if (
R.isDeleted(&*It))
19050 if (!VisitedInstrs.
insert(&*It).second) {
19051 if (HasNoUsers(&*It) &&
19052 VectorizeInsertsAndCmps(It->isTerminator())) {
19062 if (isa<DbgInfoIntrinsic>(It))
19066 if (
PHINode *
P = dyn_cast<PHINode>(It)) {
19068 if (
P->getNumIncomingValues() == 2) {
19071 if (Root && vectorizeRootInstruction(
P, Root, BB, R,
TTI)) {
19080 for (
unsigned I : seq<unsigned>(
P->getNumIncomingValues())) {
19085 if (BB ==
P->getIncomingBlock(
I) ||
19091 if (
auto *PI = dyn_cast<Instruction>(
P->getIncomingValue(
I));
19092 PI && !IsInPostProcessInstrs(PI)) {
19093 bool Res = vectorizeRootInstruction(
nullptr, PI,
19094 P->getIncomingBlock(
I), R,
TTI);
19096 if (Res &&
R.isDeleted(
P)) {
19106 if (HasNoUsers(&*It)) {
19107 bool OpsChanged =
false;
19108 auto *
SI = dyn_cast<StoreInst>(It);
19118 TryToVectorizeRoot |= (
I == Stores.
end() ||
I->second.size() == 1) &&
19119 SI->getValueOperand()->hasOneUse();
19121 if (TryToVectorizeRoot) {
19122 for (
auto *V : It->operand_values()) {
19125 if (
auto *VI = dyn_cast<Instruction>(V);
19126 VI && !IsInPostProcessInstrs(VI))
19128 OpsChanged |= vectorizeRootInstruction(
nullptr, VI, BB, R,
TTI);
19135 VectorizeInsertsAndCmps(It->isTerminator());
19146 if (isa<InsertElementInst, InsertValueInst>(It))
19147 PostProcessInserts.insert(&*It);
19148 else if (isa<CmpInst>(It))
19149 PostProcessCmps.
insert(cast<CmpInst>(&*It));
19156 auto Changed =
false;
19157 for (
auto &Entry : GEPs) {
19160 if (
Entry.second.size() < 2)
19163 LLVM_DEBUG(
dbgs() <<
"SLP: Analyzing a getelementptr list of length "
19164 <<
Entry.second.size() <<
".\n");
19172 return !R.isDeleted(GEP);
19174 if (It ==
Entry.second.end())
19176 unsigned MaxVecRegSize =
R.getMaxVecRegSize();
19177 unsigned EltSize =
R.getVectorElementSize(*(*It)->idx_begin());
19178 if (MaxVecRegSize < EltSize)
19181 unsigned MaxElts = MaxVecRegSize / EltSize;
19182 for (
unsigned BI = 0, BE =
Entry.second.size(); BI < BE; BI += MaxElts) {
19183 auto Len = std::min<unsigned>(BE - BI, MaxElts);
19196 Candidates.remove_if([&R](
Value *
I) {
19197 return R.isDeleted(cast<Instruction>(
I)) ||
19198 isa<Constant>(cast<GetElementPtrInst>(
I)->idx_begin()->
get());
19206 for (
int I = 0, E = GEPList.size();
I < E && Candidates.
size() > 1; ++
I) {
19207 auto *GEPI = GEPList[
I];
19208 if (!Candidates.count(GEPI))
19211 for (
int J =
I + 1; J < E && Candidates.
size() > 1; ++J) {
19212 auto *GEPJ = GEPList[J];
19214 if (isa<SCEVConstant>(SE->
getMinusSCEV(SCEVI, SCEVJ))) {
19215 Candidates.remove(GEPI);
19216 Candidates.remove(GEPJ);
19217 }
else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
19218 Candidates.remove(GEPJ);
19225 if (Candidates.
size() < 2)
19232 auto BundleIndex = 0
u;
19233 for (
auto *V : Candidates) {
19234 auto *
GEP = cast<GetElementPtrInst>(V);
19235 auto *GEPIdx =
GEP->idx_begin()->get();
19236 assert(
GEP->getNumIndices() == 1 && !isa<Constant>(GEPIdx));
19237 Bundle[BundleIndex++] = GEPIdx;
19249 Changed |= tryToVectorizeList(Bundle, R);
19255bool SLPVectorizerPass::vectorizeStoreChains(
BoUpSLP &R) {
19256 bool Changed =
false;
19261 if (
V->getValueOperand()->getType()->getTypeID() <
19262 V2->getValueOperand()->getType()->getTypeID())
19264 if (
V->getValueOperand()->getType()->getTypeID() >
19265 V2->getValueOperand()->getType()->getTypeID())
19267 if (
V->getPointerOperandType()->getTypeID() <
19268 V2->getPointerOperandType()->getTypeID())
19270 if (
V->getPointerOperandType()->getTypeID() >
19271 V2->getPointerOperandType()->getTypeID())
19273 if (
V->getValueOperand()->getType()->getScalarSizeInBits() <
19274 V2->getValueOperand()->getType()->getScalarSizeInBits())
19276 if (
V->getValueOperand()->getType()->getScalarSizeInBits() >
19277 V2->getValueOperand()->getType()->getScalarSizeInBits())
19280 if (isa<UndefValue>(
V->getValueOperand()) ||
19281 isa<UndefValue>(
V2->getValueOperand()))
19283 if (
auto *I1 = dyn_cast<Instruction>(
V->getValueOperand()))
19284 if (
auto *I2 = dyn_cast<Instruction>(
V2->getValueOperand())) {
19288 DT->
getNode(I2->getParent());
19289 assert(NodeI1 &&
"Should only process reachable instructions");
19290 assert(NodeI2 &&
"Should only process reachable instructions");
19291 assert((NodeI1 == NodeI2) ==
19293 "Different nodes should have different DFS numbers");
19294 if (NodeI1 != NodeI2)
19299 return I1->getOpcode() < I2->getOpcode();
19301 if (isa<Constant>(
V->getValueOperand()) &&
19302 isa<Constant>(
V2->getValueOperand()))
19304 return V->getValueOperand()->getValueID() <
19305 V2->getValueOperand()->getValueID();
19317 isa<UndefValue>(
V2->getValueOperand()))
19320 if (
auto *I2 = dyn_cast<Instruction>(
V2->getValueOperand())) {
19321 if (
I1->getParent() != I2->getParent())
19324 return S.getOpcode() > 0;
19327 isa<Constant>(
V2->getValueOperand()))
19330 V2->getValueOperand()->getValueID();
19335 for (
auto &Pair : Stores) {
19336 if (Pair.second.size() < 2)
19340 << Pair.second.size() <<
".\n");
19349 Pair.second.rend());
19350 Changed |= tryToVectorizeSequence<StoreInst>(
19351 ReversedStores, StoreSorter, AreCompatibleStores,
19353 return vectorizeStores(Candidates, R, Attempted);
static bool isConstant(const MachineInstr &MI)
AMDGPU Lower Kernel Arguments
amdgpu AMDGPU Register Bank Select
ReachingDefAnalysis InstSet InstSet & Ignore
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static cl::opt< TargetTransformInfo::TargetCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(TargetTransformInfo::TCK_RecipThroughput), cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(TargetTransformInfo::TCK_Latency, "latency", "Instruction latency"), clEnumValN(TargetTransformInfo::TCK_CodeSize, "code-size", "Code size"), clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency")))
static APInt getElementIndex(TypeSize ElemSize, APInt &Offset)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
DenseMap< Block *, BlockRelaxAux > Blocks
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool runImpl(Function &F, const TargetLowering &TLI)
This is the interface for a simple mod/ref and alias analysis over globals.
This defines the Use class.
iv Induction Variable Users
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
Loop::LoopBounds::Direction Direction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
This file provides utility analysis objects describing memory locations.
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static bool IsSelect(MachineInstr &MI)
This file defines the PriorityQueue class.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isLoadCombineCandidateImpl(Value *Root, unsigned NumElts, TargetTransformInfo *TTI, bool MustMatchOrInst)
static cl::opt< bool > RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden, cl::desc("Run the SLP vectorization passes"))
static bool clusterSortPtrAccesses(ArrayRef< Value * > VL, Type *ElemTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< unsigned > &SortedIndices)
static FixedVectorType * getWidenedType(Type *ScalarTy, unsigned VF)
static bool isVectorLikeInstWithConstOps(Value *V)
Checks if V is one of vector-like instructions, i.e.
static std::optional< Value * > calculateRtStride(ArrayRef< Value * > PointerOps, Type *ElemTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< unsigned > &SortedIndices, Instruction *Inst=nullptr)
Checks if the provided list of pointers Pointers represents the strided pointers for type ElemTy.
static cl::opt< bool > AllowHorRdxIdenityOptimization("slp-optimize-identity-hor-reduction-ops", cl::init(true), cl::Hidden, cl::desc("Allow optimization of original scalar identity operations on " "matched horizontal reductions."))
static bool isRepeatedNonIdentityClusteredMask(ArrayRef< int > Mask, unsigned Sz)
Checks if the given mask is a "clustered" mask with the same clusters of size Sz, which are not ident...
static Value * isOneOf(const InstructionsState &S, Value *Op)
Chooses the correct key for scheduling data.
static const unsigned MaxPHINumOperands
Maximum allowed number of operands in the PHI nodes.
static cl::opt< int > MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
static cl::opt< unsigned > MaxProfitableLoadStride("slp-max-stride", cl::init(8), cl::Hidden, cl::desc("The maximum stride, considered to be profitable."))
static bool needToScheduleSingleInstruction(ArrayRef< Value * > VL)
static unsigned getNumElements(Type *Ty)
static SmallBitVector buildUseMask(int VF, ArrayRef< int > Mask, UseMask MaskArg)
Prepares a use bitset for the given mask either for the first argument or for the second.
static bool areCompatibleCmpOps(Value *BaseOp0, Value *BaseOp1, Value *Op0, Value *Op1, const TargetLibraryInfo &TLI)
Checks if the provided operands of 2 cmp instructions are compatible, i.e.
static std::string shortBundleName(ArrayRef< Value * > VL)
Print a short descriptor of the instruction bundle suitable for debug output.
static unsigned getNumElems(unsigned Size, unsigned PartNumElems, unsigned Part)
Returns correct remaining number of elements, considering total amount Size, (power-of-2 number) of e...
static bool isSimple(Instruction *I)
static const int MinScheduleRegionSize
If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling regions to be handled.
static cl::opt< unsigned > MinProfitableStridedLoads("slp-min-strided-loads", cl::init(2), cl::Hidden, cl::desc("The minimum number of loads, which should be considered strided, " "if the stride is > 1 or is runtime value"))
static bool isFirstInsertElement(const InsertElementInst *IE1, const InsertElementInst *IE2)
Checks if the IE1 instructions is followed by IE2 instruction in the buildvector sequence.
static cl::opt< int > LookAheadMaxDepth("slp-max-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for operand reordering scores"))
static cl::opt< unsigned > MaxVFOption("slp-max-vf", cl::init(0), cl::Hidden, cl::desc("Maximum SLP vectorization factor (0=unlimited)"))
static void reorderReuses(SmallVectorImpl< int > &Reuses, ArrayRef< int > Mask)
Reorders the given Reuses mask according to the given Mask.
static void combineOrders(MutableArrayRef< unsigned > Order, ArrayRef< unsigned > SecondaryOrder)
static const unsigned MaxMemDepDistance
SmallBitVector getAltInstrMask(ArrayRef< Value * > VL, unsigned Opcode0, unsigned Opcode1)
static cl::opt< bool > ViewSLPTree("view-slp-tree", cl::Hidden, cl::desc("Display the SLP trees with Graphviz"))
static bool doesInTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst, TargetLibraryInfo *TLI)
static cl::opt< bool > VectorizeNonPowerOf2("slp-vectorize-non-power-of-2", cl::init(false), cl::Hidden, cl::desc("Try to vectorize with non-power-of-2 number of elements."))
static cl::opt< unsigned > MinTreeSize("slp-min-tree-size", cl::init(3), cl::Hidden, cl::desc("Only vectorize small trees if they are fully vectorizable"))
static void reorderOrder(SmallVectorImpl< unsigned > &Order, ArrayRef< int > Mask, bool BottomOrder=false)
Reorders the given Order according to the given Mask.
static T * performExtractsShuffleAction(MutableArrayRef< std::pair< T *, SmallVector< int > > > ShuffleMask, Value *Base, function_ref< unsigned(T *)> GetVF, function_ref< std::pair< T *, bool >(T *, ArrayRef< int >, bool)> ResizeAction, function_ref< T *(ArrayRef< int >, ArrayRef< T * >)> Action)
Does the analysis of the provided shuffle masks and performs the requested actions on the vectors wit...
static cl::opt< bool > ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions"))
static bool isConstant(Value *V)
static bool isSplat(ArrayRef< Value * > VL)
static cl::opt< int > SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden, cl::desc("Only vectorize if you gain more than this " "number "))
static unsigned getPartNumElems(unsigned Size, unsigned NumParts)
Returns power-of-2 number of elements in a single register (part), given the total number of elements...
static bool allConstant(ArrayRef< Value * > VL)
static constexpr int UsesLimit
static std::optional< unsigned > getElementIndex(const Value *Inst, unsigned Offset=0)
static bool isReductionCandidate(Instruction *I)
\Returns true if I is a candidate instruction for reduction vectorization.
static bool checkTreeSizes(ArrayRef< std::pair< unsigned, unsigned > > Sizes, bool First)
Checks if the quadratic mean deviation is less than 90% of the mean size.
static bool isCmpSameOrSwapped(const CmpInst *BaseCI, const CmpInst *CI, const TargetLibraryInfo &TLI)
static cl::opt< bool > SLPSkipEarlyProfitabilityCheck("slp-skip-early-profitability-check", cl::init(false), cl::Hidden, cl::desc("When true, SLP vectorizer bypasses profitability checks based on " "heuristics and makes vectorization decision via cost modeling."))
static std::pair< size_t, size_t > generateKeySubkey(Value *V, const TargetLibraryInfo *TLI, function_ref< hash_code(size_t, LoadInst *)> LoadsSubkeyGenerator, bool AllowAlternate)
Generates key/subkey pair for the given value to provide effective sorting of the values and better d...
static cl::opt< bool > ShouldStartVectorizeHorAtStore("slp-vectorize-hor-store", cl::init(false), cl::Hidden, cl::desc("Attempt to vectorize horizontal reductions feeding into a store"))
static std::pair< InstructionCost, InstructionCost > getVectorCallCosts(CallInst *CI, FixedVectorType *VecTy, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, ArrayRef< Type * > ArgTys)
static void transformScalarShuffleIndiciesToVector(unsigned VecTyNumElements, SmallVectorImpl< int > &Mask)
static cl::opt< bool > SLPReVec("slp-revec", cl::init(false), cl::Hidden, cl::desc("Enable vectorization for wider vector utilization"))
static bool isValidForAlternation(unsigned Opcode)
static std::optional< unsigned > getExtractIndex(Instruction *E)
static cl::opt< int > RootLookAheadMaxDepth("slp-max-root-look-ahead-depth", cl::init(2), cl::Hidden, cl::desc("The maximum look-ahead depth for searching best rooting option"))
static const unsigned AliasedCheckLimit
static bool findBuildAggregate(Instruction *LastInsertInst, TargetTransformInfo *TTI, SmallVectorImpl< Value * > &BuildVectorOpds, SmallVectorImpl< Value * > &InsertElts)
Recognize construction of vectors like ra = insertelement <4 x float> poison, float s0,...
static LLVM_DUMP_METHOD void dumpOrder(const BoUpSLP::OrdersType &Order)
static std::optional< TargetTransformInfo::ShuffleKind > isFixedVectorShuffle(ArrayRef< Value * > VL, SmallVectorImpl< int > &Mask)
Checks if the vector of instructions can be represented as a shuffle, like: x0 = extractelement <4 x ...
static bool isValidElementType(Type *Ty)
Predicate for the element types that the SLP vectorizer supports.
static Instruction * getReductionInstr(const DominatorTree *DT, PHINode *P, BasicBlock *ParentBB, LoopInfo *LI)
Try and get a reduction instruction from a phi node.
static InstructionCost getShuffleCost(const TargetTransformInfo &TTI, TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef< int > Mask=std::nullopt, TTI::TargetCostKind CostKind=TTI::TCK_RecipThroughput, int Index=0, VectorType *SubTp=nullptr, ArrayRef< const Value * > Args=std::nullopt)
Returns the cost of the shuffle instructions with the given Kind, vector type Tp and optional Mask.
static bool allSameType(ArrayRef< Value * > VL)
static MemoryLocation getLocation(Instruction *I)
static void findBuildAggregate_rec(Instruction *LastInsertInst, TargetTransformInfo *TTI, SmallVectorImpl< Value * > &BuildVectorOpds, SmallVectorImpl< Value * > &InsertElts, unsigned OperandOffset)
static bool isCommutative(Instruction *I)
static bool allSameBlock(ArrayRef< Value * > VL)
static bool areTwoInsertFromSameBuildVector(InsertElementInst *VU, InsertElementInst *V, function_ref< Value *(InsertElementInst *)> GetBaseOperand)
Check if two insertelement instructions are from the same buildvector.
static bool arePointersCompatible(Value *Ptr1, Value *Ptr2, const TargetLibraryInfo &TLI, bool CompareOpcodes=true)
static std::pair< InstructionCost, InstructionCost > getGEPCosts(const TargetTransformInfo &TTI, ArrayRef< Value * > Ptrs, Value *BasePtr, unsigned Opcode, TTI::TargetCostKind CostKind, Type *ScalarTy, VectorType *VecTy)
Calculate the scalar and the vector costs from vectorizing set of GEPs.
static SmallBitVector isUndefVector(const Value *V, const SmallBitVector &UseMask={})
Checks if the given value is actually an undefined constant vector.
static bool tryToVectorizeSequence(SmallVectorImpl< T * > &Incoming, function_ref< bool(T *, T *)> Comparator, function_ref< bool(T *, T *)> AreCompatible, function_ref< bool(ArrayRef< T * >, bool)> TryToVectorizeHelper, bool MaxVFOnly, BoUpSLP &R)
static cl::opt< int > ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden, cl::desc("Limit the size of the SLP scheduling region per block"))
Limits the size of scheduling regions in a block.
static Instruction * tryGetSecondaryReductionRoot(PHINode *Phi, Instruction *Root)
We could have an initial reduction that is not an add.
static RecurKind getRdxKind(Value *V)
Gets recurrence kind from the specified value.
static bool matchRdxBop(Instruction *I, Value *&V0, Value *&V1)
static cl::opt< int > MinVectorRegSizeOption("slp-min-reg-size", cl::init(128), cl::Hidden, cl::desc("Attempt to vectorize for this register size in bits"))
static std::optional< unsigned > getAggregateSize(Instruction *InsertInst)
static std::optional< unsigned > getInsertExtractIndex(const Value *Inst, unsigned Offset)
static cl::opt< unsigned > RecursionMaxDepth("slp-recursion-max-depth", cl::init(12), cl::Hidden, cl::desc("Limit the recursion depth when building a vectorizable tree"))
static Align computeCommonAlignment(ArrayRef< Value * > VL)
Calculates minimal alignment as a common alignment.
static void addMask(SmallVectorImpl< int > &Mask, ArrayRef< int > SubMask, bool ExtendingManyInputs=false)
Shuffles Mask in accordance with the given SubMask.
static void fixupOrderingIndices(MutableArrayRef< unsigned > Order)
Order may have elements assigned special value (size) which is out of bounds.
static InstructionsState getSameOpcode(ArrayRef< Value * > VL, const TargetLibraryInfo &TLI, unsigned BaseIndex=0)
static SmallVector< Type * > buildIntrinsicArgTypes(const CallInst *CI, const Intrinsic::ID ID, const unsigned VF, unsigned MinBW)
Builds the arguments types vector for the given call instruction with the given ID for the specified ...
static Instruction * getNonPhiOperand(Instruction *I, PHINode *Phi)
Returns the first operand of I that does not match Phi.
static bool compareCmp(Value *V, Value *V2, TargetLibraryInfo &TLI, const DominatorTree &DT)
Compare two cmp instructions.
static bool isReverseOrder(ArrayRef< unsigned > Order)
Check if Order represents reverse order.
static bool isAlternateInstruction(const Instruction *I, const Instruction *MainOp, const Instruction *AltOp, const TargetLibraryInfo &TLI)
Checks if the specified instruction I is an alternate operation for the given MainOp and AltOp instru...
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines generic set operations that may be used on set's of different types,...
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallString class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static SmallVector< VPValue *, 4 > getOperands(ArrayRef< VPValue * > Values, unsigned OperandIndex)
static const uint32_t IV[8]
Merges shuffle masks and emits final shuffle instruction, if required.
ShuffleCostEstimator(Type *ScalarTy, TargetTransformInfo &TTI, ArrayRef< Value * > VectorizedVals, BoUpSLP &R, SmallPtrSetImpl< Value * > &CheckedExtracts)
void add(Value *V1, Value *V2, ArrayRef< int > Mask)
Adds 2 input vectors and the mask for their shuffling.
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef< int > Mask)
std::optional< InstructionCost > needToDelay(const TreeEntry *, ArrayRef< SmallVector< const TreeEntry * > >) const
Checks if the specified entry E needs to be delayed because of its dependency nodes.
InstructionCost finalize(ArrayRef< int > ExtMask, unsigned VF=0, function_ref< void(Value *&, SmallVectorImpl< int > &)> Action={})
Finalize emission of the shuffles.
Value * gather(ArrayRef< Value * > VL, unsigned MaskVF=0, Value *Root=nullptr)
InstructionCost createFreeze(InstructionCost Cost)
void add(const TreeEntry &E1, ArrayRef< int > Mask)
Value * adjustExtracts(const TreeEntry *E, MutableArrayRef< int > Mask, ArrayRef< std::optional< TTI::ShuffleKind > > ShuffleKinds, unsigned NumParts, bool &UseVecBaseAsInput)
void add(Value *V1, ArrayRef< int > Mask, bool ForExtracts=false)
Adds another one input vector and the mask for the shuffling.
Merges shuffle masks and emits final shuffle instruction, if required.
void add(Value *V1, ArrayRef< int > Mask, bool=false)
Adds another one input vector and the mask for the shuffling.
void addOrdered(Value *V1, ArrayRef< unsigned > Order)
Adds another one input vector and the mask for the shuffling.
std::optional< Value * > needToDelay(const TreeEntry *E, ArrayRef< SmallVector< const TreeEntry * > > Deps) const
Checks if the specified entry E needs to be delayed because of its dependency nodes.
void add(const TreeEntry &E1, ArrayRef< int > Mask)
Adds single input vector (in form of tree entry) and the mask for its shuffling.
Value * gather(ArrayRef< Value * > VL, unsigned MaskVF=0, Value *Root=nullptr)
void add(Value *V1, Value *V2, ArrayRef< int > Mask)
Adds 2 input vectors and the mask for their shuffling.
Value * createFreeze(Value *V)
ShuffleInstructionBuilder(Type *ScalarTy, IRBuilderBase &Builder, BoUpSLP &R)
void add(const TreeEntry &E1, const TreeEntry &E2, ArrayRef< int > Mask)
Adds 2 input vectors (in form of tree entries) and the mask for their shuffling.
Value * adjustExtracts(const TreeEntry *E, MutableArrayRef< int > Mask, ArrayRef< std::optional< TTI::ShuffleKind > > ShuffleKinds, unsigned NumParts, bool &UseVecBaseAsInput)
Adjusts extractelements after reusing them.
Value * finalize(ArrayRef< int > ExtMask, unsigned VF=0, function_ref< void(Value *&, SmallVectorImpl< int > &)> Action={})
Finalize emission of the shuffles.
~ShuffleInstructionBuilder()
A manager for alias analyses.
Class for arbitrary precision integers.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
A container for analyses that lazily runs them and caches their results.
PassT::Result * getCachedResult(IRUnitT &IR) const
Get the cached result of an analysis pass for a given IR unit.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool equals(ArrayRef RHS) const
equals - Check for element-wise equality.
const T & back() const
back - Get the last element.
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
ArrayRef< T > drop_back(size_t N=1) const
Drop the last N elements of the array.
bool empty() const
empty - Check if the array is empty.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
InstListType::reverse_iterator reverse_iterator
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
ModRefInfo getModRefInfo(const Instruction *I, const std::optional< MemoryLocation > &OptLoc)
Represents analyses that only rely on functions' control flow.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
unsigned getBundleOperandsEndIndex() const
Return the index of the last bundle operand in the Use array.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
bool isNoBuiltin() const
Return true if the call should not be treated as a call to a builtin.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasIdenticalOperandBundleSchema(const CallBase &Other) const
Return true if Other has the same sequence of operand bundle tags with the same number of operands on...
unsigned getBundleOperandsStartIndex() const
Return the index of the first bundle operand in the Use array.
Value * getArgOperand(unsigned i) const
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned arg_size() const
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
bool hasOperandBundles() const
Return true if this User has any operand bundles.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
This class is the base class for the comparison instructions.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ ICMP_ULT
unsigned less than
@ ICMP_SGE
signed greater or equal
@ ICMP_ULE
unsigned less or equal
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Predicate getPredicate() const
Return the predicate for this instruction.
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeStoreSizeInBits(Type *Ty) const
Returns the maximum number of bits that may be overwritten by storing the specified type; always a mu...
IntegerType * getIndexType(LLVMContext &C, unsigned AddressSpace) const
Returns the type of a GEP index in AddressSpace.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
An analysis that produces DemandedBits for a function.
APInt getDemandedBits(Instruction *I)
Return the bits demanded from instruction I.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
const ValueT & at(const_arg_type_t< KeyT > Val) const
at - Return the entry for the specified key, or abort if no such entry exists.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
value_type & FindAndConstruct(const KeyT &Key)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
unsigned getDFSNumIn() const
getDFSNumIn/getDFSNumOut - These return the DFS visitation order for nodes in the dominator tree.
Analysis pass which computes a DominatorTree.
void updateDFSNumbers() const
updateDFSNumbers - Assign In and Out numbers to the nodes while walking dominator tree in dfs order.
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Convenience struct for specifying and reasoning about fast-math flags.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
ArrayRef< Type * > params() const
Type * getReturnType() const
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
nodes_iterator operator++()
nodes_iterator(const ItTy &It2)
bool operator!=(const nodes_iterator &N2) const
Common base class shared among various IRBuilders.
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateExtractVector(Type *DstType, Value *SrcVec, Value *Idx, const Twine &Name="")
Create a call to the vector.extract intrinsic.
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Value * CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
ConstantInt * getTrue()
Get the constant value for i1 true.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateFreeze(Value *V, const Twine &Name="")
BasicBlock * GetInsertBlock() const
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Value * getAllOnesMask(ElementCount NumElts)
Return an all true boolean vector (mask) with NumElts lanes.
Value * CreateUnOp(Instruction::UnaryOps Opc, Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
ConstantInt * getFalse()
Get the constant value for i1 false.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
void ClearInsertionPoint()
Clear the insertion point: created instructions will not be inserted into a block.
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
This instruction inserts a struct field of array element value into an aggregate value.
static InstructionCost getInvalid(CostType Val=0)
bool mayReadOrWriteMemory() const
Return true if this instruction may read or write memory.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
void moveAfter(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
const Instruction * getNextNonDebugInstruction(bool SkipPseudoOp=false) const
Return a pointer to the next non-debug instruction in the same basic block as 'this',...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
bool isIdenticalTo(const Instruction *I) const LLVM_READONLY
Return true if the specified instruction is exactly identical to the current one.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
Analysis pass that exposes the LoopInfo for a function.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
This class implements a map that also provides access to all stored values in a deterministic order.
size_type count(const KeyT &Key) const
VectorType takeVector()
Clear the MapVector and return the underlying vector.
iterator find(const KeyT &Key)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
ValueT lookup(const KeyT &Key) const
std::pair< KeyT, ValueT > & front()
This is the common base class for memset/memcpy/memmove.
Representation for a specific memory location.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
const Value * Ptr
The address of the start of the location.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
T & front() const
front - Get the first element.
MutableArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is a MutableArrayRef that owns its array.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
Pass interface - Implemented by all 'passes'.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...
bool isNull() const
Test if the pointer held in the union is null, regardless of which type it is.
T get() const
Returns the value of the specified pointer type.
T dyn_cast() const
Returns the current pointer if it is of the specified pointer type, otherwise returns null.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
PriorityQueue - This class behaves like std::priority_queue and provides a few additional convenience...
unsigned getOpcode() const
static bool isIntMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is an integer min/max kind.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class uses information about analyze scalars to rewrite expressions in canonical form.
Value * expandCodeFor(const SCEV *SH, Type *Ty, BasicBlock::iterator I)
Insert code to directly compute the specified SCEV expression into the program.
This class represents an analyzed expression in the program.
bool isZero() const
Return true if the expression is a constant zero.
bool isNonConstantNegative() const
Return true if the specified scev is negated, but not a constant.
Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
const SCEV * getConstant(ConstantInt *V)
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
void forgetValue(Value *V)
This method should be called by the client when it has changed a value in a way that may effect its v...
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Return LHS-RHS.
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
const SCEV * getUDivExactExpr(const SCEV *LHS, const SCEV *RHS)
Get a canonical unsigned division expression, or something simpler if possible.
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
size_type size() const
Determine the number of elements in the SetVector.
void clear()
Completely clear the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
This instruction constructs a fixed permutation of two input vectors.
static bool isZeroEltSplatMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses all elements with the same value as the first element of exa...
static bool isOneUseSingleSourceMask(ArrayRef< int > Mask, int VF)
Return true if this shuffle mask represents "clustered" mask of size VF, i.e.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
static bool isExtractSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &Index)
Return true if this shuffle mask is an extract subvector mask.
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
int find_first() const
Returns the index of the first set bit, -1 if none of the bits are set.
bool test(unsigned Idx) const
int find_next(unsigned Prev) const
Returns the index of the next set bit following the "Prev" bit.
bool all() const
Returns true if all bits are set.
size_type size() const
Returns the number of bits in this bitvector.
bool any() const
Returns true if any bit is set.
size_type count() const
Returns the number of bits which are set.
bool none() const
Returns true if none of the bits are set.
Implements a dense probed hash-table based set with some number of buckets stored inline.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
bool contains(const T &V) const
Check if the SmallSet contains the given element.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
reverse_iterator rbegin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
Type * getPointerOperandType() const
Value * getValueOperand()
Value * getPointerOperand()
StringRef - Represent a constant reference to a string, i.e.
TargetFolder - Create constants with target dependent folding.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isX86_FP80Ty() const
Return true if this is x86 long double.
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
bool isPointerTy() const
True if this is an instance of PointerType.
unsigned getStructNumElements() const
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
void print(raw_ostream &O, bool IsForDebug=false, bool NoDetails=false) const
Print the current type.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getWithNewType(Type *EltTy) const
Given vector type, change the element type, whilst keeping the old number of elements.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeID getTypeID() const
Return the type id for the type.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
User(Type *ty, unsigned vty, Use *, unsigned NumOps)
Value * getOperand(unsigned i) const
iterator_range< value_op_iterator > operand_values()
The Vector Function Database.
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
unsigned getValueID() const
Return an ID for the concrete type of this object.
bool hasNUsesOrMore(unsigned N) const
Return true if this value has N uses or more.
bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
LLVMContext & getContext() const
All values hold a context through their type.
unsigned getNumUses() const
This method computes the number of uses of this Value.
StringRef getName() const
Return a constant reference to the value's name.
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Type * getElementType() const
Value handle that is nullable, but tries to track the Value.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
bool erase(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
constexpr ScalarTy getFixedValue() const
An efficient, type-erasing, non-owning reference to a callable.
An opaque object representing a hash code.
const ParentTy * getParent() const
self_iterator getIterator()
CRTP base class for adapting an iterator to a different type.
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
A raw_ostream that writes to an std::string.
A raw_ostream that writes to an SmallVector or SmallString.
A helper class used for scoring candidates for two consecutive lanes.
static const int ScoreConsecutiveExtracts
ExtractElementInst from same vector and consecutive indexes.
int getShallowScore(Value *V1, Value *V2, Instruction *U1, Instruction *U2, ArrayRef< Value * > MainAltOps) const
static const int ScoreAllUserVectorized
Score if all users are vectorized.
static const int ScoreSameOpcode
Instructions with the same opcode.
static const int ScoreUndef
Matching with an undef is preferable to failing.
int getScoreAtLevelRec(Value *LHS, Value *RHS, Instruction *U1, Instruction *U2, int CurrLevel, ArrayRef< Value * > MainAltOps) const
Go through the operands of LHS and RHS recursively until MaxLevel, and return the cummulative score.
static const int ScoreFail
Score for failing to find a decent match.
static const int ScoreMaskedGatherCandidate
A load candidate for masked gather.
static const int ScoreSplat
Identical instructions (a.k.a. splat or broadcast).
LookAheadHeuristics(const TargetLibraryInfo &TLI, const DataLayout &DL, ScalarEvolution &SE, const BoUpSLP &R, int NumLanes, int MaxLevel)
static const int ScoreSplatLoads
The same load multiple times.
static const int ScoreReversedLoads
Loads from reversed memory addresses, e.g. load(A[i+1]), load(A[i]).
static const int ScoreConstants
Constants.
static const int ScoreAltOpcodes
Instructions with alt opcodes (e.g, add + sub).
static const int ScoreConsecutiveLoads
Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]).
static const int ScoreReversedExtracts
ExtractElementInst from same vector and reversed indices.
A helper data structure to hold the operands of a vector of instructions.
ValueList getVL(unsigned OpIdx) const
\Returns a value vector with the operands across all lanes for the opearnd at OpIdx.
static LLVM_DUMP_METHOD StringRef getModeStr(ReorderingMode RMode)
VLOperands(ArrayRef< Value * > RootVL, const BoUpSLP &R)
Initialize with all the operands of the instruction vector RootVL.
static LLVM_DUMP_METHOD void dumpMode(ReorderingMode RMode)
Debug print.
LLVM_DUMP_METHOD void dump() const
Debug print.
friend raw_ostream & operator<<(raw_ostream &OS, ReorderingMode RMode)
static LLVM_DUMP_METHOD raw_ostream & printMode(ReorderingMode RMode, raw_ostream &OS)
LLVM_DUMP_METHOD raw_ostream & print(raw_ostream &OS) const
Bottom Up SLP Vectorizer.
SmallVector< unsigned, 4 > OrdersType
std::optional< OrdersType > findPartiallyOrderedLoads(const TreeEntry &TE)
Sort loads into increasing pointers offsets to allow greater clustering.
LoadsState
Tracks the state we can represent the loads in the given sequence.
friend raw_ostream & operator<<(raw_ostream &os, const BoUpSLP::ScheduleData &SD)
void reorderTopToBottom()
Reorders the current graph to the most profitable order starting from the root node to the leaf nodes...
InstructionCost getTreeCost(ArrayRef< Value * > VectorizedVals=std::nullopt)
void reorderBottomToTop(bool IgnoreReorder=false)
Reorders the current graph to the most profitable order starting from leaves to the root.
unsigned getTreeSize() const
bool areAnalyzedReductionVals(ArrayRef< Value * > VL) const
Checks if the provided list of reduced values was checked already for vectorization.
bool isLoadCombineCandidate(ArrayRef< Value * > Stores) const
Assume that a vector of stores of bitwise-or/shifted/zexted loaded values can be load combined in the...
void analyzedReductionVals(ArrayRef< Value * > VL)
Adds the list of reduced values to list of already checked values for the vectorization.
bool isLoadCombineReductionCandidate(RecurKind RdxKind) const
Assume that a legal-sized 'or'-reduction of shifted/zexted loaded values can be load combined in the ...
unsigned getVectorElementSize(Value *V)
bool isSignedMinBitwidthRootNode() const
Checks if the root graph node can be emitted with narrower bitwidth at codegen and returns it signedn...
void analyzedReductionRoot(Instruction *I)
Register given instruction as already analyzed for being possible reduction root.
ArrayRef< Value * > getRootNodeScalars() const
Return the scalars of the root node.
void computeMinimumValueSizes()
Compute the minimum type sizes required to represent the entries in a vectorizable tree.
void deleteTree()
Clear the internal data structures that are created by 'buildTree'.
MapVector< Value *, SmallVector< Instruction *, 2 > > ExtraValueToDebugLocsMap
unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const
LoadsState canVectorizeLoads(ArrayRef< Value * > VL, const Value *VL0, SmallVectorImpl< unsigned > &Order, SmallVectorImpl< Value * > &PointerOps, bool TryRecursiveCheck=true) const
Checks if the given array of loads can be represented as a vectorized, scatter or just simple gather.
SmallPtrSet< Value *, 16 > ValueSet
BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti, TargetLibraryInfo *TLi, AAResults *Aa, LoopInfo *Li, DominatorTree *Dt, AssumptionCache *AC, DemandedBits *DB, const DataLayout *DL, OptimizationRemarkEmitter *ORE)
bool isNotScheduled(const Value *V) const
Checks if the specified value was not schedule.
void transformNodes()
Transforms graph nodes to target specific representations, if profitable.
bool isDeleted(Instruction *I) const
Checks if the instruction is marked for deletion.
void buildExternalUses(const ExtraValueToDebugLocsMap &ExternallyUsedValues={})
Builds external uses of the vectorized scalars, i.e.
bool isTreeTinyAndNotFullyVectorizable(bool ForReduction=false) const
void removeInstructionsAndOperands(ArrayRef< T * > DeadVals)
Remove instructions from the parent function and clear the operands of DeadVals instructions,...
unsigned canMapToVector(Type *T) const
Check if homogeneous aggregate is isomorphic to some VectorType.
unsigned getMinVF(unsigned Sz) const
bool isAnalyzedReductionRoot(Instruction *I) const
Checks if the instruction was already analyzed for being possible reduction root.
std::optional< OrdersType > getReorderingData(const TreeEntry &TE, bool TopToBottom)
Gets reordering data for the given tree entry.
void eraseInstruction(Instruction *I)
Removes an instruction from its block and eventually deletes it.
bool doesRootHaveInTreeUses() const
Returns whether the root node has in-tree uses.
OptimizationRemarkEmitter * getORE()
bool isAnyGathered(const SmallDenseSet< Value * > &Vals) const
Checks if the given value is gathered in one of the nodes.
SmallVector< Value *, 8 > ValueList
void buildTree(ArrayRef< Value * > Roots, const SmallDenseSet< Value * > &UserIgnoreLst)
Construct a vectorizable tree that starts at Roots, ignoring users for the purpose of scheduling and ...
unsigned getMaxVecRegSize() const
bool isVectorized(Value *V) const
Check if the value is vectorized in the tree.
bool isGathered(const Value *V) const
Checks if the given value is gathered in one of the nodes.
InstructionCost getSpillCost() const
unsigned getMinVecRegSize() const
Value * vectorizeTree()
Vectorize the tree that starts with the elements in VL.
std::optional< int > findBestRootPair(ArrayRef< std::pair< Value *, Value * > > Candidates, int Limit=LookAheadHeuristics::ScoreFail) const
Evaluate each pair in Candidates and return index into Candidates for a pair which have highest score...
std::optional< OrdersType > findReusedOrderedScalars(const TreeEntry &TE)
Checks if the specified gather tree entry TE can be represented as a shuffled vector entry + (possibl...
void clearReductionData()
Clear the list of the analyzed reduction root instructions.
void optimizeGatherSequence()
Perform LICM and CSE on the newly generated gather sequences.
Function * getVectorizedFunction(const VFShape &Shape) const
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
TwoOps_match< ValueOpTy, PointerOpTy, Instruction::Store > m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp)
Matches StoreInst.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::FMul > m_FMul(const LHS &L, const RHS &R)
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::FAdd > m_FAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
OneUse_match< T > m_OneUse(const T &SubPattern)
auto m_LogicalOr()
Matches L || R where L and R are arbitrary values.
OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)
Matches LoadInst.
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty > m_UMax(const LHS &L, const RHS &R)
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty > m_SMax(const LHS &L, const RHS &R)
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
match_combine_or< CastInst_match< OpTy, ZExtInst >, CastInst_match< OpTy, SExtInst > > m_ZExtOrSExt(const OpTy &Op)
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
auto m_LogicalAnd()
Matches L && R where L and R are arbitrary values.
auto m_Undef()
Match an arbitrary undef constant.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
std::optional< int > getPointersDiff(Type *ElemTyA, Value *PtrA, Type *ElemTyB, Value *PtrB, const DataLayout &DL, ScalarEvolution &SE, bool StrictCheck=false, bool CheckType=true)
Returns the distance between the pointers PtrA and PtrB iff they are compatible and it is possible to...
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
static bool doesNotNeedToBeScheduled(Value *V)
Checks if the specified value does not require scheduling.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
void stable_sort(R &&Range)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
hash_code hash_value(const FixedPointSemantics &Val)
Intrinsic::ID getMinMaxReductionIntrinsicOp(Intrinsic::ID RdxID)
Returns the min/max intrinsic used when expanding a min/max reduction.
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
If the specified value is a trivially dead instruction, delete it.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
bool MaskedValueIsZero(const Value *V, const APInt &Mask, const SimplifyQuery &DL, unsigned Depth=0)
Return true if 'V & Mask' is known to be zero.
static void reorderScalars(SmallVectorImpl< Value * > &Scalars, ArrayRef< int > Mask)
Reorders the list of scalars in accordance with the given Mask.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Value * createSimpleTargetReduction(IRBuilderBase &B, Value *Src, RecurKind RdxKind)
Create a target reduction of the given vector.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
void set_intersect(S1Ty &S1, const S2Ty &S2)
set_intersect(A, B) - Compute A := A ^ B Identical to set_intersection, except that it works on set<>...
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
testing::Matcher< const detail::ErrorHolder & > Failed()
static bool isUsedOutsideBlock(Value *V)
Checks if the provided value does not require scheduling.
std::pair< Intrinsic::ID, bool > canConvertToMinOrMaxIntrinsic(ArrayRef< Value * > VL)
Check if the values in VL are select instructions that can be converted to a min or max (vector) intr...
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
iterator_range< po_iterator< T > > post_order(const T &G)
MaybeAlign getAlign(const Function &F, unsigned Index)
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
bool isGather(IntrinsicInst *IntInst)
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
auto reverse(ContainerTy &&C)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
static void inversePermutation(ArrayRef< unsigned > Indices, SmallVectorImpl< int > &Mask)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)
Create a mask with replicated elements.
auto find_if_not(R &&Range, UnaryPredicate P)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
bool isModOrRefSet(const ModRefInfo MRI)
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool sortPtrAccesses(ArrayRef< Value * > VL, Type *ElemTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< unsigned > &SortedIndices)
Attempt to sort the pointers in VL and return the sorted indices in SortedIndices,...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
void propagateIRFlags(Value *I, ArrayRef< Value * > VL, Value *OpValue=nullptr, bool IncludeWrapFlags=true)
Get the intersection (logical and) of all of the potential IR flags of each scalar operation (VL) tha...
constexpr int PoisonMaskElem
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
RecurKind
These are the kinds of recurrences that we support.
@ Or
Bitwise or logical OR of integers.
static bool areAllOperandsNonInsts(Value *V)
Checks if the provided value does not require scheduling.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
DWARFExpression::Operation Op
auto max_element(R &&Range)
void ViewGraph(const GraphType &G, const Twine &Name, bool ShortNames=false, const Twine &Title="", GraphProgram::Name Program=GraphProgram::DOT)
ViewGraph - Emit a dot graph, run 'dot', run gv on the postscript file, then cleanup.
OutputIt copy(R &&Range, OutputIt Out)
static bool doesNotNeedToSchedule(ArrayRef< Value * > VL)
Checks if the specified array of instructions does not require scheduling.
constexpr unsigned BitWidth
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
unsigned ComputeNumSignBits(const Value *Op, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return the number of times the sign bit of the register is replicated into the other bits.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be poison, but may be undef.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Constant * ConstantFoldIntegerCast(Constant *C, Type *DestTy, bool IsSigned, const DataLayout &DL)
Constant fold a zext, sext or trunc, depending on IsSigned and whether the DestTy is wider or narrowe...
bool isKnownNonNegative(const Value *V, const SimplifyQuery &SQ, unsigned Depth=0)
Returns true if the give value is known to be non-negative.
bool mayHaveNonDefUseDependency(const Instruction &I)
Returns true if the result or effects of the given instructions I depend values not reachable through...
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Used to keep track of an operand bundle.
static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
BoUpSLP::TreeEntry TreeEntry
std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R)
static std::string getNodeAttributes(const TreeEntry *Entry, const BoUpSLP *)
DOTGraphTraits(bool IsSimple=false)
DOTGraphTraits - Template class that can be specialized to customize how graphs are converted to 'dot...
DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...
Used in the streaming interface as the general argument type.
ChildIteratorType(SmallVector< BoUpSLP::EdgeInfo, 1 >::iterator W, ContainerTy &VT)
ContainerTy & VectorizableTree
static ChildIteratorType child_end(NodeRef N)
static NodeRef getEntryNode(BoUpSLP &R)
static ChildIteratorType child_begin(NodeRef N)
static nodes_iterator nodes_begin(BoUpSLP *R)
TreeEntry * NodeRef
NodeRef has to be a pointer per the GraphWriter.
static unsigned size(BoUpSLP *R)
BoUpSLP::TreeEntry TreeEntry
static nodes_iterator nodes_end(BoUpSLP *R)
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
Direction
An enum for the direction of the loop.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
bool runImpl(Function &F, ScalarEvolution *SE_, TargetTransformInfo *TTI_, TargetLibraryInfo *TLI_, AAResults *AA_, LoopInfo *LI_, DominatorTree *DT_, AssumptionCache *AC_, DemandedBits *DB_, OptimizationRemarkEmitter *ORE_)
Contains the information about the kind of vectorization available.
static VFShape get(const FunctionType *FTy, ElementCount EC, bool HasGlobalPred)
Retrieve the basic vectorization shape of the function, where all parameters are mapped to VFParamKin...
Function object to check whether the first component of a container supported by std::get (like std::...
Function object to check whether the second component of a container supported by std::get (like std:...
This structure holds any data we need about the edges being traversed during buildTree_rec().
unsigned EdgeIdx
The operand index of the use.
EdgeInfo(TreeEntry *UserTE, unsigned EdgeIdx)
LLVM_DUMP_METHOD void dump() const
TreeEntry * UserTE
The user TreeEntry.
friend raw_ostream & operator<<(raw_ostream &OS, const BoUpSLP::EdgeInfo &EI)
void dump(raw_ostream &OS) const
Debug print.