31#define DEBUG_TYPE "vectorutils"
39 cl::desc(
"Maximum factor for an interleaved access group (default = 8)"),
49 case Intrinsic::bswap:
50 case Intrinsic::bitreverse:
51 case Intrinsic::ctpop:
60 case Intrinsic::sadd_sat:
61 case Intrinsic::ssub_sat:
62 case Intrinsic::uadd_sat:
63 case Intrinsic::usub_sat:
64 case Intrinsic::smul_fix:
65 case Intrinsic::smul_fix_sat:
66 case Intrinsic::umul_fix:
67 case Intrinsic::umul_fix_sat:
74 case Intrinsic::log10:
77 case Intrinsic::minnum:
78 case Intrinsic::maxnum:
79 case Intrinsic::minimum:
80 case Intrinsic::maximum:
81 case Intrinsic::copysign:
82 case Intrinsic::floor:
84 case Intrinsic::trunc:
86 case Intrinsic::nearbyint:
87 case Intrinsic::round:
88 case Intrinsic::roundeven:
91 case Intrinsic::fmuladd:
92 case Intrinsic::is_fpclass:
94 case Intrinsic::canonicalize:
95 case Intrinsic::fptosi_sat:
96 case Intrinsic::fptoui_sat:
97 case Intrinsic::lrint:
98 case Intrinsic::llrint:
107 unsigned ScalarOpdIdx) {
110 case Intrinsic::ctlz:
111 case Intrinsic::cttz:
112 case Intrinsic::is_fpclass:
113 case Intrinsic::powi:
114 return (ScalarOpdIdx == 1);
115 case Intrinsic::smul_fix:
116 case Intrinsic::smul_fix_sat:
117 case Intrinsic::umul_fix:
118 case Intrinsic::umul_fix_sat:
119 return (ScalarOpdIdx == 2);
130 case Intrinsic::fptosi_sat:
131 case Intrinsic::fptoui_sat:
132 case Intrinsic::lrint:
133 case Intrinsic::llrint:
134 return OpdIdx == -1 || OpdIdx == 0;
135 case Intrinsic::is_fpclass:
137 case Intrinsic::powi:
138 return OpdIdx == -1 || OpdIdx == 1;
154 ID == Intrinsic::lifetime_end ||
ID == Intrinsic::assume ||
155 ID == Intrinsic::experimental_noalias_scope_decl ||
156 ID == Intrinsic::sideeffect ||
ID == Intrinsic::pseudoprobe)
165 assert(V->getType()->isVectorTy() &&
"Not looking at a vector?");
166 VectorType *VTy = cast<VectorType>(V->getType());
168 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
169 unsigned Width = FVTy->getNumElements();
175 return C->getAggregateElement(EltNo);
179 if (!isa<ConstantInt>(III->getOperand(2)))
181 unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue();
186 return III->getOperand(1);
189 if (III == III->getOperand(0))
199 if (SVI && isa<FixedVectorType>(SVI->
getType())) {
205 if (InEl < (
int)LHSWidth)
214 if (
Constant *Elt =
C->getAggregateElement(EltNo))
215 if (Elt->isNullValue())
219 if (isa<ScalableVectorType>(VTy))
221 if (EltNo < VTy->getElementCount().getKnownMinValue())
236 if (SplatIndex != -1 && SplatIndex != M)
242 assert((SplatIndex == -1 || SplatIndex >= 0) &&
"Negative index?");
251 if (isa<VectorType>(V->getType()))
252 if (
auto *
C = dyn_cast<Constant>(V))
253 return C->getSplatValue();
268 if (isa<VectorType>(V->getType())) {
269 if (isa<UndefValue>(V))
273 if (
auto *
C = dyn_cast<Constant>(V))
274 return C->getSplatValue() !=
nullptr;
277 if (
auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) {
312 const APInt &DemandedElts,
APInt &DemandedLHS,
313 APInt &DemandedRHS,
bool AllowUndefElts) {
317 if (DemandedElts.
isZero())
321 if (
all_of(Mask, [](
int Elt) {
return Elt == 0; })) {
326 for (
unsigned I = 0, E = Mask.size();
I != E; ++
I) {
328 assert((-1 <= M) && (M < (SrcWidth * 2)) &&
329 "Invalid shuffle mask constant");
331 if (!DemandedElts[
I] || (AllowUndefElts && (M < 0)))
342 DemandedRHS.
setBit(M - SrcWidth);
350 assert(Scale > 0 &&
"Unexpected scaling factor");
354 ScaledMask.
assign(Mask.begin(), Mask.end());
359 for (
int MaskElt : Mask) {
362 "Overflowed 32-bits");
364 for (
int SliceElt = 0; SliceElt != Scale; ++SliceElt)
365 ScaledMask.
push_back(MaskElt < 0 ? MaskElt : Scale * MaskElt + SliceElt);
371 assert(Scale > 0 &&
"Unexpected scaling factor");
375 ScaledMask.
assign(Mask.begin(), Mask.end());
380 int NumElts = Mask.size();
381 if (NumElts % Scale != 0)
385 ScaledMask.
reserve(NumElts / Scale);
390 assert((
int)MaskSlice.
size() == Scale &&
"Expected Scale-sized slice.");
393 int SliceFront = MaskSlice.
front();
394 if (SliceFront < 0) {
402 if (SliceFront % Scale != 0)
405 for (
int i = 1; i < Scale; ++i)
406 if (MaskSlice[i] != SliceFront + i)
408 ScaledMask.
push_back(SliceFront / Scale);
410 Mask = Mask.drop_front(Scale);
411 }
while (!Mask.empty());
413 assert((
int)ScaledMask.
size() * Scale == NumElts &&
"Unexpected scaled mask");
422 std::array<SmallVector<int, 16>, 2> TmpMasks;
425 for (
unsigned Scale = 2; Scale <= InputMask.
size(); ++Scale) {
435 ArrayRef<int> Mask,
unsigned NumOfSrcRegs,
unsigned NumOfDestRegs,
436 unsigned NumOfUsedRegs,
function_ref<
void()> NoInputAction,
444 int Sz = Mask.size();
445 unsigned SzDest = Sz / NumOfDestRegs;
446 unsigned SzSrc = Sz / NumOfSrcRegs;
447 for (
unsigned I = 0;
I < NumOfDestRegs; ++
I) {
448 auto &RegMasks = Res[
I];
449 RegMasks.
assign(NumOfSrcRegs, {});
452 for (
unsigned K = 0; K < SzDest; ++K) {
453 int Idx =
I * SzDest + K;
458 int SrcRegIdx = Mask[
Idx] / SzSrc;
461 if (RegMasks[SrcRegIdx].empty())
463 RegMasks[SrcRegIdx][K] = Mask[
Idx] % SzSrc;
467 for (
unsigned I = 0;
I < NumOfUsedRegs; ++
I) {
471 switch (NumSrcRegs) {
480 unsigned SrcReg = std::distance(Dest.begin(), It);
481 SingleInputAction(*It, SrcReg,
I);
496 "Expected undefined mask element.");
497 FirstMask[
Idx] = SecondMask[
Idx] + VF;
502 for (
int Idx = 0, VF = Mask.size();
Idx < VF; ++
Idx) {
512 for (
unsigned I = 0;
I < NumOfDestRegs; ++
I) {
517 if (FirstIdx == SecondIdx) {
523 SecondMask = RegMask;
524 CombineMasks(FirstMask, SecondMask);
525 ManyInputsAction(FirstMask, FirstIdx, SecondIdx);
526 NormalizeMask(FirstMask);
528 SecondMask = FirstMask;
529 SecondIdx = FirstIdx;
531 if (FirstIdx != SecondIdx && SecondIdx >= 0) {
532 CombineMasks(SecondMask, FirstMask);
533 ManyInputsAction(SecondMask, SecondIdx, FirstIdx);
534 Dest[FirstIdx].clear();
535 NormalizeMask(SecondMask);
537 }
while (SecondIdx >= 0);
560 bool SeenExtFromIllegalType =
false;
562 for (
auto &
I : *BB) {
565 if (
TTI && (isa<ZExtInst>(&
I) || isa<SExtInst>(&
I)) &&
567 SeenExtFromIllegalType =
true;
570 if ((isa<TruncInst>(&
I) || isa<ICmpInst>(&
I)) &&
571 !
I.getType()->isVectorTy() &&
572 I.getOperand(0)->getType()->getScalarSizeInBits() <= 64) {
583 if (Worklist.
empty() || (
TTI && !SeenExtFromIllegalType))
587 while (!Worklist.
empty()) {
591 if (!Visited.
insert(Val).second)
595 if (!isa<Instruction>(Val))
601 if (DB.getDemandedBits(
I).getBitWidth() > 64)
604 uint64_t V = DB.getDemandedBits(
I).getZExtValue();
610 if (isa<SExtInst>(
I) || isa<ZExtInst>(
I) || isa<LoadInst>(
I) ||
617 if (isa<BitCastInst>(
I) || isa<PtrToIntInst>(
I) || isa<IntToPtrInst>(
I) ||
618 !
I->getType()->isIntegerTy()) {
619 DBits[Leader] |= ~0ULL;
629 if (DBits[Leader] == ~0ULL)
633 for (
Value *O : cast<User>(
I)->operands()) {
642 for (
auto &
I : DBits)
643 for (
auto *U :
I.first->users())
644 if (U->getType()->isIntegerTy() && DBits.
count(U) == 0)
647 for (
auto I = ECs.
begin(), E = ECs.
end();
I != E; ++
I) {
650 LeaderDemandedBits |= DBits[M];
670 auto *
MI = dyn_cast<Instruction>(M);
673 Type *Ty = M->getType();
675 Ty =
MI->getOperand(0)->getType();
683 auto *CI = dyn_cast<ConstantInt>(U);
687 isa<ShlOperator, LShrOperator, AShrOperator>(U.getUser()) &&
688 U.getOperandNo() == 1)
689 return CI->uge(MinBW);
690 uint64_t BW = bit_width(DB.getDemandedBits(&U).getZExtValue());
691 return bit_ceil(BW) > MinBW;
703template <
typename ListT>
708 List.insert(AccGroups);
712 for (
const auto &AccGroupListOp : AccGroups->
operands()) {
713 auto *Item = cast<MDNode>(AccGroupListOp.get());
724 if (AccGroups1 == AccGroups2)
731 if (Union.size() == 0)
733 if (Union.size() == 1)
734 return cast<MDNode>(Union.front());
745 if (!MayAccessMem1 && !MayAccessMem2)
748 return Inst2->
getMetadata(LLVMContext::MD_access_group);
750 return Inst1->
getMetadata(LLVMContext::MD_access_group);
766 if (AccGroupSet2.
count(MD1))
770 auto *Item = cast<MDNode>(Node.get());
772 if (AccGroupSet2.
count(Item))
777 if (Intersection.
size() == 0)
779 if (Intersection.
size() == 1)
780 return cast<MDNode>(Intersection.
front());
794 for (
auto Kind : {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
795 LLVMContext::MD_noalias, LLVMContext::MD_fpmath,
796 LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load,
797 LLVMContext::MD_access_group, LLVMContext::MD_mmra}) {
799 for (
int J = 1, E = VL.
size(); MD && J != E; ++J) {
804 case LLVMContext::MD_mmra: {
808 case LLVMContext::MD_tbaa:
811 case LLVMContext::MD_alias_scope:
814 case LLVMContext::MD_fpmath:
817 case LLVMContext::MD_noalias:
818 case LLVMContext::MD_nontemporal:
819 case LLVMContext::MD_invariant_load:
822 case LLVMContext::MD_access_group:
847 for (
unsigned i = 0; i < VF; i++)
848 for (
unsigned j = 0; j < Group.
getFactor(); ++j) {
849 unsigned HasMember = Group.
getMember(j) ? 1 : 0;
850 Mask.push_back(Builder.
getInt1(HasMember));
859 for (
unsigned i = 0; i < VF; i++)
860 for (
unsigned j = 0; j < ReplicationFactor; j++)
869 for (
unsigned i = 0; i < VF; i++)
870 for (
unsigned j = 0; j < NumVecs; j++)
871 Mask.push_back(j * VF + i);
879 for (
unsigned i = 0; i < VF; i++)
880 Mask.push_back(Start + i * Stride);
887 unsigned NumUndefs) {
889 for (
unsigned i = 0; i < NumInts; i++)
890 Mask.push_back(Start + i);
892 for (
unsigned i = 0; i < NumUndefs; i++)
901 int NumEltsSigned = NumElts;
902 assert(NumEltsSigned > 0 &&
"Expected smaller or non-zero element count");
907 for (
int MaskElt : Mask) {
908 assert((MaskElt < NumEltsSigned * 2) &&
"Expected valid shuffle mask");
909 int UnaryElt = MaskElt >= NumEltsSigned ? MaskElt - NumEltsSigned : MaskElt;
921 VectorType *VecTy2 = dyn_cast<VectorType>(V2->getType());
922 assert(VecTy1 && VecTy2 &&
923 VecTy1->getScalarType() == VecTy2->getScalarType() &&
924 "Expect two vectors with the same element type");
926 unsigned NumElts1 = cast<FixedVectorType>(VecTy1)->getNumElements();
927 unsigned NumElts2 = cast<FixedVectorType>(VecTy2)->getNumElements();
928 assert(NumElts1 >= NumElts2 &&
"Unexpect the first vector has less elements");
930 if (NumElts1 > NumElts2) {
942 unsigned NumVecs = Vecs.
size();
943 assert(NumVecs > 1 &&
"Should be at least two vectors");
949 for (
unsigned i = 0; i < NumVecs - 1; i += 2) {
950 Value *V0 = ResList[i], *V1 = ResList[i + 1];
951 assert((V0->
getType() == V1->getType() || i == NumVecs - 2) &&
952 "Only the last vector may have a different type");
958 if (NumVecs % 2 != 0)
962 NumVecs = ResList.
size();
963 }
while (NumVecs > 1);
969 assert(isa<VectorType>(Mask->getType()) &&
970 isa<IntegerType>(Mask->getType()->getScalarType()) &&
971 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==
973 "Mask must be a vector of i1");
975 auto *ConstMask = dyn_cast<Constant>(Mask);
978 if (ConstMask->isNullValue() || isa<UndefValue>(ConstMask))
980 if (isa<ScalableVectorType>(ConstMask->getType()))
984 E = cast<FixedVectorType>(ConstMask->getType())->getNumElements();
986 if (
auto *MaskElt = ConstMask->getAggregateElement(
I))
987 if (MaskElt->isNullValue() || isa<UndefValue>(MaskElt))
995 assert(isa<VectorType>(Mask->getType()) &&
996 isa<IntegerType>(Mask->getType()->getScalarType()) &&
997 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==
999 "Mask must be a vector of i1");
1001 auto *ConstMask = dyn_cast<Constant>(Mask);
1004 if (ConstMask->isAllOnesValue() || isa<UndefValue>(ConstMask))
1006 if (isa<ScalableVectorType>(ConstMask->getType()))
1010 E = cast<FixedVectorType>(ConstMask->getType())->getNumElements();
1012 if (
auto *MaskElt = ConstMask->getAggregateElement(
I))
1013 if (MaskElt->isAllOnesValue() || isa<UndefValue>(MaskElt))
1021 assert(isa<VectorType>(Mask->getType()) &&
1022 isa<IntegerType>(Mask->getType()->getScalarType()) &&
1023 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==
1025 "Mask must be a vector of i1");
1027 auto *ConstMask = dyn_cast<Constant>(Mask);
1030 if (ConstMask->isAllOnesValue() || isa<UndefValue>(ConstMask))
1032 if (isa<ScalableVectorType>(ConstMask->getType()))
1036 E = cast<FixedVectorType>(ConstMask->getType())->getNumElements();
1038 if (
auto *MaskElt = ConstMask->getAggregateElement(
I))
1039 if (MaskElt->isAllOnesValue() || isa<UndefValue>(MaskElt))
1048 assert(isa<FixedVectorType>(Mask->getType()) &&
1049 isa<IntegerType>(Mask->getType()->getScalarType()) &&
1050 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==
1052 "Mask must be a fixed width vector of i1");
1054 const unsigned VWidth =
1055 cast<FixedVectorType>(Mask->getType())->getNumElements();
1057 if (
auto *CV = dyn_cast<ConstantVector>(Mask))
1058 for (
unsigned i = 0; i < VWidth; i++)
1059 if (CV->getAggregateElement(i)->isNullValue())
1061 return DemandedElts;
1064bool InterleavedAccessInfo::isStrided(
int Stride) {
1065 unsigned Factor = std::abs(Stride);
1069void InterleavedAccessInfo::collectConstStrideAccesses(
1083 for (
auto &
I : *BB) {
1092 if (
Size * 8 !=
DL.getTypeSizeInBits(ElementTy))
1104 true,
false).value_or(0);
1107 AccessStrideInfo[&
I] = StrideDescriptor(Stride, Scev,
Size,
1149 bool EnablePredicatedInterleavedMemAccesses) {
1155 collectConstStrideAccesses(AccessStrideInfo, Strides);
1157 if (AccessStrideInfo.
empty())
1161 collectDependences();
1182 for (
auto BI = AccessStrideInfo.
rbegin(), E = AccessStrideInfo.
rend();
1185 StrideDescriptor DesB = BI->second;
1191 if (isStrided(DesB.Stride) &&
1192 (!isPredicated(
B->getParent()) || EnablePredicatedInterleavedMemAccesses)) {
1197 GroupB = createInterleaveGroup(
B, DesB.Stride, DesB.Alignment);
1198 if (
B->mayWriteToMemory())
1199 StoreGroups.
insert(GroupB);
1201 LoadGroups.
insert(GroupB);
1205 for (
auto AI = std::next(BI); AI != E; ++AI) {
1207 StrideDescriptor DesA = AI->second;
1232 if (MemberOfGroupB && !canReorderMemAccessesForInterleavedGroups(
1233 A, &*AccessStrideInfo.
find(MemberOfGroupB)))
1234 return MemberOfGroupB;
1244 if (
A->mayWriteToMemory() && GroupA != GroupB) {
1252 if (GroupB && LoadGroups.
contains(GroupB))
1253 DependentInst = DependentMember(GroupB, &*AI);
1254 else if (!canReorderMemAccessesForInterleavedGroups(&*AI, &*BI))
1257 if (DependentInst) {
1262 if (GroupA && StoreGroups.
contains(GroupA)) {
1264 "dependence between "
1265 << *
A <<
" and " << *DependentInst <<
'\n');
1266 StoreGroups.
remove(GroupA);
1267 releaseGroup(GroupA);
1273 if (GroupB && LoadGroups.
contains(GroupB)) {
1275 <<
" as complete.\n");
1276 CompletedLoadGroups.
insert(GroupB);
1280 if (CompletedLoadGroups.
contains(GroupB)) {
1288 if (!isStrided(DesA.Stride) || !isStrided(DesB.Stride))
1298 (
A->mayReadFromMemory() !=
B->mayReadFromMemory()) ||
1299 (
A->mayWriteToMemory() !=
B->mayWriteToMemory()))
1304 if (DesA.Stride != DesB.Stride || DesA.Size != DesB.Size)
1321 if (DistanceToB %
static_cast<int64_t
>(DesB.Size))
1328 if ((isPredicated(BlockA) || isPredicated(BlockB)) &&
1329 (!EnablePredicatedInterleavedMemAccesses || BlockA != BlockB))
1335 GroupB->
getIndex(
B) + DistanceToB /
static_cast<int64_t
>(DesB.Size);
1340 <<
" into the interleave group with" << *
B
1342 InterleaveGroupMap[
A] = GroupB;
1345 if (
A->mayReadFromMemory())
1353 std::string FirstOrLast) ->
bool {
1355 assert(Member &&
"Group member does not exist");
1358 if (
getPtrStride(PSE, AccessTy, MemberPtr, TheLoop, Strides,
1359 false,
true).value_or(0))
1361 LLVM_DEBUG(
dbgs() <<
"LV: Invalidate candidate interleaved group due to "
1363 <<
" group member potentially pointer-wrapping.\n");
1364 releaseGroup(Group);
1382 for (
auto *Group : LoadGroups) {
1394 if (InvalidateGroupIfMemberMayWrap(Group, 0, std::string(
"first")))
1397 InvalidateGroupIfMemberMayWrap(Group, Group->
getFactor() - 1,
1398 std::string(
"last"));
1407 dbgs() <<
"LV: Invalidate candidate interleaved group due to "
1408 "a reverse access with gaps.\n");
1409 releaseGroup(Group);
1413 dbgs() <<
"LV: Interleaved group requires epilogue iteration.\n");
1414 RequiresScalarEpilogue =
true;
1418 for (
auto *Group : StoreGroups) {
1428 if (!EnablePredicatedInterleavedMemAccesses) {
1430 dbgs() <<
"LV: Invalidate candidate interleaved store group due "
1432 releaseGroup(Group);
1442 if (InvalidateGroupIfMemberMayWrap(Group, 0, std::string(
"first")))
1446 InvalidateGroupIfMemberMayWrap(Group,
Index, std::string(
"last"));
1458 bool ReleasedGroup =
false;
1462 if (!Group->requiresScalarEpilogue())
1466 <<
"LV: Invalidate candidate interleaved group due to gaps that "
1467 "require a scalar epilogue (not allowed under optsize) and cannot "
1468 "be masked (not enabled). \n");
1469 releaseGroup(Group);
1470 ReleasedGroup =
true;
1472 assert(ReleasedGroup &&
"At least one group must be invalidated, as a "
1473 "scalar epilogue was required");
1474 (void)ReleasedGroup;
1475 RequiresScalarEpilogue =
false;
1478template <
typename InstT>
1487 std::transform(Members.begin(), Members.end(), std::back_inserter(VL),
1488 [](std::pair<int, Instruction *> p) { return p.second; });
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
DenseMap< Block *, BlockRelaxAux > Blocks
Generic implementation of equivalence classes through the use Tarjan's efficient union-find algorithm...
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file provides utility for Memory Model Relaxation Annotations (MMRAs).
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
static Value * concatenateTwoVectors(IRBuilderBase &Builder, Value *V1, Value *V2)
A helper function for concatenating vectors.
static cl::opt< unsigned > MaxInterleaveGroupFactor("max-interleave-group-factor", cl::Hidden, cl::desc("Maximum factor for an interleaved access group (default = 8)"), cl::init(8))
Maximum factor for an interleaved memory access.
static void addToAccessGroupList(ListT &List, MDNode *AccGroups)
Add all access groups in AccGroups to List.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
int64_t getSExtValue() const
Get sign extended value.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
LLVM Basic Block Representation.
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
const ElemTy & getOrInsertLeaderValue(const ElemTy &V)
getOrInsertLeaderValue - Return the leader for the specified value that is in the set.
member_iterator member_end() const
member_iterator member_begin(iterator I) const
member_iterator unionSets(const ElemTy &V1, const ElemTy &V2)
union - Merge the two equivalence sets for the specified values, inserting them if they do not alread...
Common base class shared among various IRBuilders.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
This instruction inserts a single (scalar) element into a VectorType value.
bool mayReadOrWriteMemory() const
Return true if this instruction may read or write memory.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void getAllMetadataOtherThanDebugLoc(SmallVectorImpl< std::pair< unsigned, MDNode * > > &MDs) const
This does the same thing as getAllMetadata, except that it filters out the debug location.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
uint32_t getIndex(const InstTy *Instr) const
Get the index for the given member.
void setInsertPos(InstTy *Inst)
void addMetadata(InstTy *NewInst) const
Add metadata (e.g.
bool insertMember(InstTy *Instr, int32_t Index, Align NewAlign)
Try to insert a new member Instr with index Index and alignment NewAlign.
uint32_t getNumMembers() const
InterleaveGroup< Instruction > * getInterleaveGroup(const Instruction *Instr) const
Get the interleave group that Instr belongs to.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
bool isInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleave group.
void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
void invalidateGroupsRequiringScalarEpilogue()
Invalidate groups that require a scalar epilogue (due to gaps).
This is an important class for using LLVM in a threaded context.
const DenseMap< Value *, const SCEV * > & getSymbolicStrides() const
If an access has a symbolic strides, this maps the pointer value to the stride symbol.
BlockT * getHeader() const
Store the result of a depth first search within basic blocks contained by a single loop.
static MDNode * getMostGenericAliasScope(MDNode *A, MDNode *B)
static MDNode * getMostGenericTBAA(MDNode *A, MDNode *B)
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDNode * getMostGenericFPMath(MDNode *A, MDNode *B)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDNode * intersect(MDNode *A, MDNode *B)
LLVMContext & getContext() const
Tracking metadata reference owned by Metadata.
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
reverse_iterator rbegin()
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
This class represents a constant integer value.
const APInt & getAPInt() const
This class represents an analyzed expression in the program.
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Return LHS-RHS.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
This instruction constructs a fixed permutation of two input vectors.
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
VectorType * getType() const
Overload to return most specific vector type.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Provides information about what library functions are available for the current target.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
Type * getElementType() const
An efficient, type-erasing, non-owning reference to a callable.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
bool match(Val *V, const Pattern &P)
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
APInt possiblyDemandedEltsInMask(Value *Mask)
Given a mask vector of the form <Y x i1>, return an APInt (of bitwidth Y) for each lane which may be ...
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
unsigned getLoadStoreAddressSpace(Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
MDNode * intersectAccessGroups(const Instruction *Inst1, const Instruction *Inst2)
Compute the access-group list of access groups that Inst1 and Inst2 are both in.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)
Create a mask that filters the members of an interleave group where there are gaps.
constexpr unsigned MaxAnalysisRecursionDepth
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)
Create a mask with replicated elements.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
std::optional< int64_t > getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp, const DenseMap< Value *, const SCEV * > &StridesMap=DenseMap< Value *, const SCEV * >(), bool Assume=false, bool ShouldCheckWrap=true)
If the pointer has a constant stride return it in units of the access type size.
Align getLoadStoreAlignment(Value *I)
A helper function that returns the alignment of load or store instruction.
bool maskIsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if all of the elements of this predicate mask are known to be ...
constexpr int PoisonMaskElem
bool isValidAsAccessGroup(MDNode *AccGroup)
Return whether an MDNode might represent an access group.
Intrinsic::ID getIntrinsicForCallSite(const CallBase &CB, const TargetLibraryInfo *TLI)
Map a call instruction to an intrinsic ID.
void processShuffleMasks(ArrayRef< int > Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, unsigned NumOfUsedRegs, function_ref< void()> NoInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned)> SingleInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned)> ManyInputsAction)
Splits and processes shuffle mask depending on the number of input and output registers.
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
const SCEV * replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &PtrToStride, Value *Ptr)
Return the SCEV corresponding to a pointer with the symbolic stride replaced with constant one,...
Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
MDNode * uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2)
Compute the union of two access-group lists.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool maskIsAllZeroOrUndef(Value *Mask)
Given a mask vector of i1, Return true if all of the elements of this predicate mask are known to be ...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
void getShuffleMaskWithWidestElts(ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Repetitively apply widenShuffleMaskElts() for as long as it succeeds, to get the shuffle mask with wi...
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
bool maskContainsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if any of the elements of this predicate mask are known to be ...
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
Type * getLoadStoreType(Value *I)
A helper function that returns the type of a load or store instruction.
MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.