71#define DEBUG_TYPE "loop-accesses"
75 cl::desc(
"Sets the SIMD width. Zero is autoselect."),
81 cl::desc(
"Sets the vectorization interleave count. "
82 "Zero is autoselect."),
89 cl::desc(
"When performing memory disambiguation checks at runtime do not "
90 "generate more than this number of comparisons (default = 8)."),
97 cl::desc(
"Maximum number of comparisons done when trying to merge "
98 "runtime memory checks. (default = 100)"),
107 cl::desc(
"Maximum number of dependences collected by "
108 "loop-access analysis (default = 100)"),
124 cl::desc(
"Enable symbolic stride memory access versioning"));
129 "store-to-load-forwarding-conflict-detection",
cl::Hidden,
130 cl::desc(
"Enable conflict detection in loop-access analysis"),
135 cl::desc(
"Maximum recursion depth when finding forked SCEVs (default = 5)"),
140 cl::desc(
"Speculate that non-constant strides are unit in LAA"),
146 "Hoist inner loop runtime memory checks to outer loop if possible"),
151 return ::VectorizationInterleave.getNumOccurrences() > 0;
162 if (SI == PtrToStride.
end())
166 const SCEV *StrideSCEV = SI->second;
171 assert(isa<SCEVUnknown>(StrideSCEV) &&
"shouldn't be in map");
179 <<
" by: " << *Expr <<
"\n");
189 NeedsFreeze(RtCheck.Pointers[
Index].NeedsFreeze) {
209 DenseMap<std::pair<const SCEV *, Type *>,
214 {{PtrExpr, AccessTy},
223 ScStart = ScEnd = PtrExpr;
224 }
else if (
auto *AR = dyn_cast<SCEVAddRecExpr>(PtrExpr)) {
227 ScStart = AR->getStart();
228 ScEnd = AR->evaluateAtIteration(Ex, *SE);
229 const SCEV *Step = AR->getStepRecurrence(*SE);
233 if (
const auto *CStep = dyn_cast<SCEVConstant>(Step)) {
234 if (CStep->getValue()->isNegative())
255 Iter->second = {ScStart, ScEnd};
262 Type *AccessTy,
bool WritePtr,
263 unsigned DepSetId,
unsigned ASId,
268 assert(!isa<SCEVCouldNotCompute>(ScStart) &&
269 !isa<SCEVCouldNotCompute>(ScEnd) &&
270 "must be able to compute both start and end expressions");
271 Pointers.emplace_back(
Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, PtrExpr,
275bool RuntimePointerChecking::tryToCreateDiffCheck(
298 if (AccSrc.
size() != 1 || AccSink.
size() != 1)
302 if (AccSink[0] < AccSrc[0])
305 auto *SrcAR = dyn_cast<SCEVAddRecExpr>(Src->Expr);
306 auto *SinkAR = dyn_cast<SCEVAddRecExpr>(Sink->Expr);
317 if (isa<ScalableVectorType>(SrcTy) || isa<ScalableVectorType>(DstTy))
321 SinkAR->getLoop()->getHeader()->getDataLayout();
323 std::max(
DL.getTypeAllocSize(SrcTy),
DL.getTypeAllocSize(DstTy));
328 auto *Step = dyn_cast<SCEVConstant>(SinkAR->getStepRecurrence(*SE));
329 if (!Step || Step != SrcAR->getStepRecurrence(*SE) ||
330 Step->getAPInt().abs() != AllocSize)
338 if (Step->getValue()->isNegative())
343 if (isa<SCEVCouldNotCompute>(SinkStartInt) ||
344 isa<SCEVCouldNotCompute>(SrcStartInt))
347 const Loop *InnerLoop = SrcAR->getLoop();
353 isa<SCEVAddRecExpr>(SinkStartInt) && isa<SCEVAddRecExpr>(SrcStartInt)) {
354 auto *SrcStartAR = cast<SCEVAddRecExpr>(SrcStartInt);
355 auto *SinkStartAR = cast<SCEVAddRecExpr>(SinkStartInt);
356 const Loop *StartARLoop = SrcStartAR->getLoop();
357 if (StartARLoop == SinkStartAR->getLoop() &&
362 SrcStartAR->getStepRecurrence(*SE) !=
363 SinkStartAR->getStepRecurrence(*SE)) {
364 LLVM_DEBUG(
dbgs() <<
"LAA: Not creating diff runtime check, since these "
365 "cannot be hoisted out of the outer loop\n");
371 <<
"SrcStart: " << *SrcStartInt <<
'\n'
372 <<
"SinkStartInt: " << *SinkStartInt <<
'\n');
373 DiffChecks.emplace_back(SrcStartInt, SinkStartInt, AllocSize,
374 Src->NeedsFreeze ||
Sink->NeedsFreeze);
387 CanUseDiffCheck = CanUseDiffCheck && tryToCreateDiffCheck(CGI, CGJ);
388 Checks.
push_back(std::make_pair(&CGI, &CGJ));
395void RuntimePointerChecking::generateChecks(
398 groupChecks(DepCands, UseDependencies);
404 for (
const auto &
I : M.Members)
405 for (
const auto &J :
N.Members)
420 return C->getValue()->isNegative() ? J :
I;
427 RtCheck.
Pointers[
Index].PointerValue->getType()->getPointerAddressSpace(),
436 "all pointers in a checking group must be in the same address space");
462void RuntimePointerChecking::groupChecks(
508 if (!UseDependencies) {
514 unsigned TotalComparisons = 0;
519 It->second.push_back(
Index);
548 auto PointerI = PositionMap.
find(
MI->getPointer());
550 "pointer in equivalence class not found in PositionMap");
551 for (
unsigned Pointer : PointerI->second) {
568 if (Group.addPointer(Pointer, *
this)) {
591 return (PtrToPartition[PtrIdx1] != -1 &&
592 PtrToPartition[PtrIdx1] == PtrToPartition[PtrIdx2]);
616 unsigned Depth)
const {
618 for (
const auto &[Check1, Check2] : Checks) {
619 const auto &
First = Check1->Members, &Second = Check2->Members;
623 OS.
indent(
Depth + 2) <<
"Comparing group (" << Check1 <<
"):\n";
624 for (
unsigned K :
First)
627 OS.
indent(
Depth + 2) <<
"Against group (" << Check2 <<
"):\n";
628 for (
unsigned K : Second)
641 OS.
indent(
Depth + 4) <<
"(Low: " << *CG.Low <<
" High: " << *CG.High
643 for (
unsigned Member : CG.Members) {
655class AccessAnalysis {
665 : TheLoop(TheLoop), BAA(*AA), AST(BAA), LI(LI), DepCands(DA), PSE(PSE),
666 LoopAliasScopes(LoopAliasScopes) {
668 BAA.enableCrossIterationMode();
674 AST.add(adjustLoc(Loc));
675 Accesses[MemAccessInfo(
Ptr,
false)].insert(AccessTy);
677 ReadOnlyPtr.insert(
Ptr);
683 AST.add(adjustLoc(Loc));
684 Accesses[MemAccessInfo(
Ptr,
true)].insert(AccessTy);
695 MemAccessInfo Access,
Type *AccessTy,
698 Loop *TheLoop,
unsigned &RunningDepId,
699 unsigned ASId,
bool ShouldCheckStride,
bool Assume);
708 Value *&UncomputablePtr,
bool ShouldCheckWrap =
false);
712 void buildDependenceSets() {
713 processMemAccesses();
721 bool isDependencyCheckNeeded() {
return !CheckDeps.empty(); }
729 MemAccessInfoList &getDependenciesToCheck() {
return CheckDeps; }
733 return UnderlyingObjects;
758 return LoopAliasScopes.contains(cast<MDNode>(Scope));
767 void processMemAccesses();
771 PtrAccessMap Accesses;
777 MemAccessInfoList CheckDeps;
804 bool IsRTCheckAnalysisNeeded =
false;
822 const SCEV *PtrScev,
Loop *L,
bool Assume) {
846 int64_t Stride =
getPtrStride(PSE, AccessTy,
Ptr, L, Strides).value_or(0);
859 while (!WorkList.
empty()) {
863 auto *PN = dyn_cast<PHINode>(
Ptr);
867 if (PN && InnermostLoop.
contains(PN->getParent()) &&
868 PN->getParent() != InnermostLoop.
getHeader()) {
869 for (
const Use &Inc : PN->incoming_values())
902 if (isa<SCEVAddRecExpr>(Scev) || L->isLoopInvariant(
Ptr) ||
903 !isa<Instruction>(
Ptr) ||
Depth == 0) {
914 auto GetBinOpExpr = [&SE](
unsigned Opcode,
const SCEV *L,
const SCEV *R) {
916 case Instruction::Add:
918 case Instruction::Sub:
926 unsigned Opcode =
I->getOpcode();
928 case Instruction::GetElementPtr: {
930 Type *SourceTy =
GEP->getSourceElementType();
933 if (
I->getNumOperands() != 2 || SourceTy->
isVectorTy()) {
943 bool NeedsFreeze =
any_of(BaseScevs, UndefPoisonCheck) ||
944 any_of(OffsetScevs, UndefPoisonCheck);
949 if (OffsetScevs.
size() == 2 && BaseScevs.
size() == 1)
951 else if (BaseScevs.
size() == 2 && OffsetScevs.
size() == 1)
954 ScevList.emplace_back(Scev, NeedsFreeze);
972 ScevList.emplace_back(SE->
getAddExpr(get<0>(BaseScevs[0]), Scaled1),
974 ScevList.emplace_back(SE->
getAddExpr(get<0>(BaseScevs[1]), Scaled2),
978 case Instruction::Select: {
985 if (ChildScevs.
size() == 2) {
986 ScevList.push_back(ChildScevs[0]);
987 ScevList.push_back(ChildScevs[1]);
992 case Instruction::PHI: {
997 if (
I->getNumOperands() == 2) {
1001 if (ChildScevs.
size() == 2) {
1002 ScevList.push_back(ChildScevs[0]);
1003 ScevList.push_back(ChildScevs[1]);
1008 case Instruction::Add:
1009 case Instruction::Sub: {
1017 any_of(LScevs, UndefPoisonCheck) ||
any_of(RScevs, UndefPoisonCheck);
1022 if (LScevs.
size() == 2 && RScevs.
size() == 1)
1024 else if (RScevs.
size() == 2 && LScevs.
size() == 1)
1027 ScevList.emplace_back(Scev, NeedsFreeze);
1031 ScevList.emplace_back(
1032 GetBinOpExpr(Opcode, get<0>(LScevs[0]), get<0>(RScevs[0])),
1034 ScevList.emplace_back(
1035 GetBinOpExpr(Opcode, get<0>(LScevs[1]), get<0>(RScevs[1])),
1041 LLVM_DEBUG(
dbgs() <<
"ForkedPtr unhandled instruction: " << *
I <<
"\n");
1058 if (Scevs.
size() == 2 &&
1059 (isa<SCEVAddRecExpr>(get<0>(Scevs[0])) ||
1061 (isa<SCEVAddRecExpr>(get<0>(Scevs[1])) ||
1073 MemAccessInfo Access,
Type *AccessTy,
1076 Loop *TheLoop,
unsigned &RunningDepId,
1077 unsigned ASId,
bool ShouldCheckWrap,
1084 for (
auto &
P : TranslatedPtrs) {
1085 const SCEV *PtrExpr = get<0>(
P);
1091 if (ShouldCheckWrap) {
1093 if (TranslatedPtrs.size() > 1)
1096 if (!
isNoWrap(PSE, StridesMap,
Ptr, AccessTy, TheLoop)) {
1098 if (!Assume || !isa<SCEVAddRecExpr>(Expr))
1105 if (TranslatedPtrs.size() == 1)
1110 for (
auto [PtrExpr, NeedsFreeze] : TranslatedPtrs) {
1114 if (isDependencyCheckNeeded()) {
1116 unsigned &LeaderId = DepSetId[Leader];
1118 LeaderId = RunningDepId++;
1122 DepId = RunningDepId++;
1124 bool IsWrite = Access.getInt();
1125 RtCheck.
insert(TheLoop,
Ptr, PtrExpr, AccessTy, IsWrite, DepId, ASId, PSE,
1136 Value *&UncomputablePtr,
bool ShouldCheckWrap) {
1139 bool CanDoRT =
true;
1141 bool MayNeedRTCheck =
false;
1142 if (!IsRTCheckAnalysisNeeded)
return true;
1144 bool IsDepCheckNeeded = isDependencyCheckNeeded();
1149 for (
auto &AS : AST) {
1150 int NumReadPtrChecks = 0;
1151 int NumWritePtrChecks = 0;
1152 bool CanDoAliasSetRT =
true;
1154 auto ASPointers = AS.getPointers();
1158 unsigned RunningDepId = 1;
1166 for (
const Value *ConstPtr : ASPointers) {
1168 bool IsWrite = Accesses.count(MemAccessInfo(
Ptr,
true));
1170 ++NumWritePtrChecks;
1178 if (NumWritePtrChecks == 0 ||
1179 (NumWritePtrChecks == 1 && NumReadPtrChecks == 0)) {
1180 assert((ASPointers.size() <= 1 ||
1183 MemAccessInfo AccessWrite(
const_cast<Value *
>(
Ptr),
1185 return DepCands.
findValue(AccessWrite) == DepCands.
end();
1187 "Can only skip updating CanDoRT below, if all entries in AS "
1188 "are reads or there is at most 1 entry");
1192 for (
auto &Access : AccessInfos) {
1193 for (
const auto &AccessTy : Accesses[Access]) {
1194 if (!createCheckForAccess(RtCheck, Access, AccessTy, StridesMap,
1195 DepSetId, TheLoop, RunningDepId, ASId,
1196 ShouldCheckWrap,
false)) {
1198 << *Access.getPointer() <<
'\n');
1200 CanDoAliasSetRT =
false;
1214 bool NeedsAliasSetRTCheck = RunningDepId > 2 || !Retries.
empty();
1218 if (NeedsAliasSetRTCheck && !CanDoAliasSetRT) {
1222 CanDoAliasSetRT =
true;
1223 for (
const auto &[Access, AccessTy] : Retries) {
1224 if (!createCheckForAccess(RtCheck, Access, AccessTy, StridesMap,
1225 DepSetId, TheLoop, RunningDepId, ASId,
1226 ShouldCheckWrap,
true)) {
1227 CanDoAliasSetRT =
false;
1228 UncomputablePtr = Access.getPointer();
1234 CanDoRT &= CanDoAliasSetRT;
1235 MayNeedRTCheck |= NeedsAliasSetRTCheck;
1244 unsigned NumPointers = RtCheck.
Pointers.size();
1245 for (
unsigned i = 0; i < NumPointers; ++i) {
1246 for (
unsigned j = i + 1;
j < NumPointers; ++
j) {
1248 if (RtCheck.
Pointers[i].DependencySetId ==
1249 RtCheck.
Pointers[j].DependencySetId)
1262 dbgs() <<
"LAA: Runtime check would require comparison between"
1263 " different address spaces\n");
1269 if (MayNeedRTCheck && CanDoRT)
1273 <<
" pointer comparisons.\n");
1280 bool CanDoRTIfNeeded = !RtCheck.
Need || CanDoRT;
1281 if (!CanDoRTIfNeeded)
1283 return CanDoRTIfNeeded;
1286void AccessAnalysis::processMemAccesses() {
1293 LLVM_DEBUG(
dbgs() <<
"LAA: Accesses(" << Accesses.size() <<
"):\n");
1295 for (
const auto &[
A,
_] : Accesses)
1296 dbgs() <<
"\t" << *
A.getPointer() <<
" ("
1297 << (
A.getInt() ?
"write"
1298 : (ReadOnlyPtr.count(
A.getPointer()) ?
"read-only"
1307 for (
const auto &AS : AST) {
1311 auto ASPointers = AS.getPointers();
1313 bool SetHasWrite =
false;
1317 UnderlyingObjToAccessMap ObjToLastAccess;
1320 PtrAccessMap DeferredAccesses;
1324 for (
int SetIteration = 0; SetIteration < 2; ++SetIteration) {
1325 bool UseDeferred = SetIteration > 0;
1326 PtrAccessMap &S = UseDeferred ? DeferredAccesses : Accesses;
1328 for (
const Value *ConstPtr : ASPointers) {
1333 for (
const auto &[AC,
_] : S) {
1334 if (AC.getPointer() !=
Ptr)
1337 bool IsWrite = AC.getInt();
1341 bool IsReadOnlyPtr = ReadOnlyPtr.count(
Ptr) && !IsWrite;
1342 if (UseDeferred && !IsReadOnlyPtr)
1346 assert(((IsReadOnlyPtr && UseDeferred) || IsWrite ||
1347 S.count(MemAccessInfo(
Ptr,
false))) &&
1348 "Alias-set pointer not in the access set?");
1350 MemAccessInfo Access(
Ptr, IsWrite);
1358 if (!UseDeferred && IsReadOnlyPtr) {
1361 DeferredAccesses.insert({Access, {}});
1369 if ((IsWrite || IsReadOnlyPtr) && SetHasWrite) {
1370 CheckDeps.push_back(Access);
1371 IsRTCheckAnalysisNeeded =
true;
1380 ValueVector TempObjects;
1382 UnderlyingObjects[
Ptr] = {};
1386 <<
"Underlying objects for pointer " << *
Ptr <<
"\n");
1387 for (
const Value *UnderlyingObj : UOs) {
1390 if (isa<ConstantPointerNull>(UnderlyingObj) &&
1396 UnderlyingObjToAccessMap::iterator Prev =
1397 ObjToLastAccess.find(UnderlyingObj);
1398 if (Prev != ObjToLastAccess.end())
1399 DepCands.
unionSets(Access, Prev->second);
1401 ObjToLastAccess[UnderlyingObj] = Access;
1430 auto *
GEP = dyn_cast<GetElementPtrInst>(
Ptr);
1431 if (!
GEP || !
GEP->isInBounds())
1435 Value *NonConstIndex =
nullptr;
1437 if (!isa<ConstantInt>(
Index)) {
1440 NonConstIndex =
Index;
1448 if (
auto *OBO = dyn_cast<OverflowingBinaryOperator>(NonConstIndex))
1449 if (OBO->hasNoSignedWrap() &&
1452 isa<ConstantInt>(OBO->getOperand(1))) {
1453 auto *OpScev = PSE.
getSCEV(OBO->getOperand(0));
1455 if (
auto *OpAR = dyn_cast<SCEVAddRecExpr>(OpScev))
1456 return OpAR->getLoop() == L && OpAR->getNoWrapFlags(
SCEV::FlagNSW);
1467 bool Assume,
bool ShouldCheckWrap) {
1471 if (isa<ScalableVectorType>(AccessTy)) {
1472 LLVM_DEBUG(
dbgs() <<
"LAA: Bad stride - Scalable object: " << *AccessTy
1474 return std::nullopt;
1485 <<
" SCEV: " << *PtrScev <<
"\n");
1486 return std::nullopt;
1491 LLVM_DEBUG(
dbgs() <<
"LAA: Bad stride - Not striding over innermost loop "
1492 << *
Ptr <<
" SCEV: " << *AR <<
"\n");
1493 return std::nullopt;
1503 <<
" SCEV: " << *AR <<
"\n");
1504 return std::nullopt;
1508 TypeSize AllocSize =
DL.getTypeAllocSize(AccessTy);
1510 const APInt &APStepVal =
C->getAPInt();
1514 return std::nullopt;
1519 int64_t Stride = StepVal /
Size;
1520 int64_t Rem = StepVal %
Size;
1522 return std::nullopt;
1524 if (!ShouldCheckWrap)
1536 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(
Ptr);
1537 GEP &&
GEP->isInBounds() && (Stride == 1 || Stride == -1))
1545 (Stride == 1 || Stride == -1))
1551 <<
"LAA: Pointer: " << *
Ptr <<
"\n"
1552 <<
"LAA: SCEV: " << *AR <<
"\n"
1553 <<
"LAA: Added an overflow assumption\n");
1557 dbgs() <<
"LAA: Bad stride - Pointer may wrap in the address space "
1558 << *
Ptr <<
" SCEV: " << *AR <<
"\n");
1559 return std::nullopt;
1567 assert(PtrA && PtrB &&
"Expected non-nullptr pointers.");
1575 return std::nullopt;
1582 return std::nullopt;
1583 unsigned IdxWidth =
DL.getIndexSizeInBits(ASA);
1585 APInt OffsetA(IdxWidth, 0), OffsetB(IdxWidth, 0);
1590 if (PtrA1 == PtrB1) {
1593 ASA = cast<PointerType>(PtrA1->
getType())->getAddressSpace();
1594 ASB = cast<PointerType>(PtrB1->
getType())->getAddressSpace();
1597 return std::nullopt;
1599 IdxWidth =
DL.getIndexSizeInBits(ASA);
1600 OffsetA = OffsetA.sextOrTrunc(IdxWidth);
1610 dyn_cast<SCEVConstant>(SE.
getMinusSCEV(PtrSCEVB, PtrSCEVA));
1612 return std::nullopt;
1613 Val = Diff->getAPInt().getSExtValue();
1615 int Size =
DL.getTypeStoreSize(ElemTyA);
1616 int Dist = Val /
Size;
1620 if (!StrictCheck || Dist *
Size == Val)
1622 return std::nullopt;
1630 "Expected list of pointer operands.");
1633 Value *Ptr0 = VL[0];
1635 using DistOrdPair = std::pair<int64_t, int>;
1637 std::set<DistOrdPair,
decltype(Compare)> Offsets(Compare);
1638 Offsets.emplace(0, 0);
1639 bool IsConsecutive =
true;
1648 auto [It, IsInserted] = Offsets.emplace(
Offset,
Idx);
1652 IsConsecutive &= std::next(It) == Offsets.end();
1654 SortedIndices.
clear();
1655 if (!IsConsecutive) {
1659 SortedIndices[
Idx] = Off.second;
1673 std::optional<int> Diff =
1676 return Diff && *Diff == 1;
1682 Accesses[MemAccessInfo(Ptr, true)].push_back(AccessIdx);
1683 InstMap.push_back(SI);
1691 Accesses[MemAccessInfo(Ptr, false)].push_back(AccessIdx);
1692 InstMap.push_back(LI);
1720 case ForwardButPreventsForwarding:
1722 case IndirectUnsafe:
1725 case BackwardVectorizable:
1727 case BackwardVectorizableButPreventsForwarding:
1740 case ForwardButPreventsForwarding:
1745 case BackwardVectorizable:
1747 case BackwardVectorizableButPreventsForwarding:
1748 case IndirectUnsafe:
1754bool MemoryDepChecker::couldPreventStoreLoadForward(
uint64_t Distance,
1768 const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
1770 uint64_t MaxVFWithoutSLForwardIssues = std::min(
1774 for (
uint64_t VF = 2 * TypeByteSize; VF <= MaxVFWithoutSLForwardIssues;
1778 if (Distance % VF && Distance / VF < NumItersForStoreLoadThroughMemory) {
1779 MaxVFWithoutSLForwardIssues = (VF >> 1);
1784 if (MaxVFWithoutSLForwardIssues < 2 * TypeByteSize) {
1786 dbgs() <<
"LAA: Distance " << Distance
1787 <<
" that could cause a store-load forwarding conflict\n");
1791 if (MaxVFWithoutSLForwardIssues < MinDepDistBytes &&
1792 MaxVFWithoutSLForwardIssues !=
1794 MinDepDistBytes = MaxVFWithoutSLForwardIssues;
1816 const SCEV &MaxBTC,
const SCEV &Dist,
1837 const uint64_t ByteStride = MaxStride * TypeByteSize;
1841 const SCEV *CastedDist = &Dist;
1842 const SCEV *CastedProduct = Product;
1849 if (DistTypeSizeBits > ProductTypeSizeBits)
1874 assert(Stride > 1 &&
"The stride must be greater than 1");
1875 assert(TypeByteSize > 0 &&
"The type size in byte must be non-zero");
1876 assert(Distance > 0 &&
"The distance must be non-zero");
1879 if (Distance % TypeByteSize)
1882 uint64_t ScaledDist = Distance / TypeByteSize;
1900 return ScaledDist % Stride;
1908 return any_of(UnderlyingObjects, [&SE, L](
const Value *UO) {
1914 MemoryDepChecker::DepDistanceStrideAndSizeInfo>
1915MemoryDepChecker::getDependenceDistanceStrideAndSize(
1919 &UnderlyingObjects) {
1921 auto &SE = *PSE.
getSE();
1922 auto [APtr, AIsWrite] =
A;
1923 auto [BPtr, BIsWrite] =
B;
1926 if (!AIsWrite && !BIsWrite)
1933 if (APtr->getType()->getPointerAddressSpace() !=
1934 BPtr->getType()->getPointerAddressSpace())
1937 int64_t StrideAPtr =
1938 getPtrStride(PSE, ATy, APtr, InnermostLoop, SymbolicStrides,
true)
1940 int64_t StrideBPtr =
1941 getPtrStride(PSE, BTy, BPtr, InnermostLoop, SymbolicStrides,
true)
1950 if (StrideAPtr < 0) {
1957 LLVM_DEBUG(
dbgs() <<
"LAA: Src Scev: " << *Src <<
"Sink Scev: " << *Sink
1958 <<
"(Induction step: " << StrideAPtr <<
")\n");
1959 LLVM_DEBUG(
dbgs() <<
"LAA: Distance for " << *AInst <<
" to " << *BInst
1960 <<
": " << *Dist <<
"\n");
1976 const auto &[SrcStart, SrcEnd] =
1978 const auto &[SinkStart, SinkEnd] =
1980 if (!isa<SCEVCouldNotCompute>(SrcStart) &&
1981 !isa<SCEVCouldNotCompute>(SrcEnd) &&
1982 !isa<SCEVCouldNotCompute>(SinkStart) &&
1983 !isa<SCEVCouldNotCompute>(SinkEnd)) {
1994 if (!StrideAPtr || !StrideBPtr || (StrideAPtr > 0 && StrideBPtr < 0) ||
1995 (StrideAPtr < 0 && StrideBPtr > 0)) {
1996 LLVM_DEBUG(
dbgs() <<
"Pointer access with non-constant stride\n");
2000 uint64_t TypeByteSize =
DL.getTypeAllocSize(ATy);
2002 DL.getTypeStoreSizeInBits(ATy) ==
DL.getTypeStoreSizeInBits(BTy);
2005 return DepDistanceStrideAndSizeInfo(Dist, std::abs(StrideAPtr),
2006 std::abs(StrideBPtr), TypeByteSize,
2007 AIsWrite, BIsWrite);
2014 &UnderlyingObjects) {
2015 assert(AIdx < BIdx &&
"Must pass arguments in program order");
2019 auto Res = getDependenceDistanceStrideAndSize(
2020 A, InstMap[AIdx],
B, InstMap[BIdx], UnderlyingObjects);
2021 if (std::holds_alternative<Dependence::DepType>(Res))
2022 return std::get<Dependence::DepType>(Res);
2024 auto &[Dist, StrideA, StrideB, TypeByteSize, AIsWrite, BIsWrite] =
2025 std::get<DepDistanceStrideAndSizeInfo>(Res);
2026 bool HasSameSize = TypeByteSize > 0;
2028 std::optional<uint64_t> CommonStride =
2029 StrideA == StrideB ? std::make_optional(StrideA) :
std::nullopt;
2030 if (isa<SCEVCouldNotCompute>(Dist)) {
2033 FoundNonConstantDistanceDependence |= CommonStride.has_value();
2034 LLVM_DEBUG(
dbgs() <<
"LAA: Dependence because of uncomputable distance.\n");
2040 uint64_t MaxStride = std::max(StrideA, StrideB);
2049 *Dist, MaxStride, TypeByteSize))
2056 const APInt &Val =
C->getAPInt();
2061 if (std::abs(Distance) > 0 && CommonStride && *CommonStride > 1 &&
2078 LLVM_DEBUG(
dbgs() <<
"LAA: possibly zero dependence difference but "
2079 "different type sizes\n");
2083 bool IsTrueDataDependence = (AIsWrite && !BIsWrite);
2098 FoundNonConstantDistanceDependence |= CommonStride.has_value();
2102 couldPreventStoreLoadForward(
C->getAPInt().abs().getZExtValue(),
2105 dbgs() <<
"LAA: Forward but may prevent st->ld forwarding\n");
2116 if (MinDistance <= 0) {
2117 FoundNonConstantDistanceDependence |= CommonStride.has_value();
2121 if (!isa<SCEVConstant>(Dist)) {
2130 FoundNonConstantDistanceDependence |= CommonStride.has_value();
2134 LLVM_DEBUG(
dbgs() <<
"LAA: ReadWrite-Write positive dependency with "
2135 "different type sizes\n");
2148 unsigned MinNumIter = std::max(ForcedFactor * ForcedUnroll, 2U);
2181 TypeByteSize * *CommonStride * (MinNumIter - 1) + TypeByteSize;
2182 if (MinDistanceNeeded >
static_cast<uint64_t>(MinDistance)) {
2183 if (!isa<SCEVConstant>(Dist)) {
2190 LLVM_DEBUG(
dbgs() <<
"LAA: Failure because of positive minimum distance "
2191 << MinDistance <<
'\n');
2197 if (MinDistanceNeeded > MinDepDistBytes) {
2199 << MinDistanceNeeded <<
" size in bytes\n");
2220 std::min(
static_cast<uint64_t>(MinDistance), MinDepDistBytes);
2222 bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
2223 uint64_t MinDepDistBytesOld = MinDepDistBytes;
2225 isa<SCEVConstant>(Dist) &&
2226 couldPreventStoreLoadForward(MinDistance, TypeByteSize)) {
2229 assert(MinDepDistBytes == MinDepDistBytesOld &&
2230 "An update to MinDepDistBytes requires an update to "
2231 "MaxSafeVectorWidthInBits");
2232 (void)MinDepDistBytesOld;
2238 uint64_t MaxVF = MinDepDistBytes / (TypeByteSize * *CommonStride);
2239 LLVM_DEBUG(
dbgs() <<
"LAA: Positive min distance " << MinDistance
2240 <<
" with max VF = " << MaxVF <<
'\n');
2242 uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8;
2243 if (!isa<SCEVConstant>(Dist) && MaxVFInBits < MaxTargetVectorWidthInBits) {
2250 MaxSafeVectorWidthInBits = std::min(MaxSafeVectorWidthInBits, MaxVFInBits);
2257 &UnderlyingObjects) {
2259 MinDepDistBytes = -1;
2262 if (Visited.
count(CurAccess))
2278 bool AIIsWrite = AI->getInt();
2282 (AIIsWrite ? AI : std::next(AI));
2285 for (std::vector<unsigned>::iterator I1 = Accesses[*AI].begin(),
2286 I1E = Accesses[*AI].
end(); I1 != I1E; ++I1)
2289 for (std::vector<unsigned>::iterator
2290 I2 = (OI == AI ? std::next(I1) : Accesses[*OI].begin()),
2291 I2E = (OI == AI ? I1E : Accesses[*OI].end());
2293 auto A = std::make_pair(&*AI, *I1);
2294 auto B = std::make_pair(&*OI, *I2);
2301 B.second, UnderlyingObjects);
2308 if (RecordDependences) {
2313 RecordDependences =
false;
2314 Dependences.clear();
2316 <<
"Too many dependences, stopped recording\n");
2328 LLVM_DEBUG(
dbgs() <<
"Total Dependences: " << Dependences.size() <<
"\n");
2335 auto &IndexVector = Accesses.find(Access)->second;
2339 std::back_inserter(Insts),
2340 [&](
unsigned Idx) {
return this->InstMap[
Idx]; });
2349 "ForwardButPreventsForwarding",
2351 "BackwardVectorizable",
2352 "BackwardVectorizableButPreventsForwarding"};
2362bool LoopAccessInfo::canAnalyzeLoop() {
2371 recordAnalysis(
"NotInnerMostLoop") <<
"loop is not the innermost loop";
2378 dbgs() <<
"LAA: loop control flow is not understood by analyzer\n");
2379 recordAnalysis(
"CFGNotUnderstood")
2380 <<
"loop control flow is not understood by analyzer";
2388 if (isa<SCEVCouldNotCompute>(ExitCount)) {
2389 recordAnalysis(
"CantComputeNumberOfIterations")
2390 <<
"could not determine number of loop iterations";
2391 LLVM_DEBUG(
dbgs() <<
"LAA: SCEV could not compute the loop exit count.\n");
2409 unsigned NumReads = 0;
2410 unsigned NumReadWrites = 0;
2412 bool HasComplexMemInst =
false;
2415 HasConvergentOp =
false;
2417 PtrRtChecking->Pointers.
clear();
2418 PtrRtChecking->Need =
false;
2422 const bool EnableMemAccessVersioningOfLoop =
2434 if (
auto *Call = dyn_cast<CallBase>(&
I)) {
2435 if (
Call->isConvergent())
2436 HasConvergentOp =
true;
2441 if (HasComplexMemInst && HasConvergentOp)
2445 if (HasComplexMemInst)
2449 if (
auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&
I))
2450 for (
Metadata *
Op : Decl->getScopeList()->operands())
2451 LoopAliasScopes.
insert(cast<MDNode>(
Op));
2456 auto *
Call = dyn_cast<CallInst>(&
I);
2463 if (
I.mayReadFromMemory()) {
2466 if (Call && !
Call->isNoBuiltin() &&
Call->getCalledFunction() &&
2470 auto *Ld = dyn_cast<LoadInst>(&
I);
2472 recordAnalysis(
"CantVectorizeInstruction", Ld)
2473 <<
"instruction cannot be vectorized";
2474 HasComplexMemInst =
true;
2477 if (!Ld->isSimple() && !IsAnnotatedParallel) {
2478 recordAnalysis(
"NonSimpleLoad", Ld)
2479 <<
"read with atomic ordering or volatile read";
2481 HasComplexMemInst =
true;
2487 if (EnableMemAccessVersioningOfLoop)
2488 collectStridedAccess(Ld);
2493 if (
I.mayWriteToMemory()) {
2494 auto *St = dyn_cast<StoreInst>(&
I);
2496 recordAnalysis(
"CantVectorizeInstruction", St)
2497 <<
"instruction cannot be vectorized";
2498 HasComplexMemInst =
true;
2501 if (!St->isSimple() && !IsAnnotatedParallel) {
2502 recordAnalysis(
"NonSimpleStore", St)
2503 <<
"write with atomic ordering or volatile write";
2505 HasComplexMemInst =
true;
2511 if (EnableMemAccessVersioningOfLoop)
2512 collectStridedAccess(St);
2517 if (HasComplexMemInst)
2525 if (!Stores.
size()) {
2531 AccessAnalysis Accesses(TheLoop, AA, LI, DependentAccesses, *PSE,
2548 if (isInvariant(
Ptr)) {
2550 StoresToInvariantAddresses.push_back(ST);
2551 HasStoreStoreDependenceInvolvingLoopInvariantAddress |=
2558 if (Seen.
insert({Ptr, AccessTy}).second) {
2565 if (blockNeedsPredication(
ST->getParent(), TheLoop, DT))
2569 [&Accesses, AccessTy, Loc](
Value *
Ptr) {
2570 MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr);
2571 Accesses.addStore(NewLoc, AccessTy);
2576 if (IsAnnotatedParallel) {
2578 dbgs() <<
"LAA: A loop annotated parallel, ignore memory dependency "
2593 bool IsReadOnlyPtr =
false;
2595 if (Seen.
insert({Ptr, AccessTy}).second ||
2596 !
getPtrStride(*PSE,
LD->getType(),
Ptr, TheLoop, SymbolicStrides).value_or(0)) {
2598 IsReadOnlyPtr =
true;
2604 LLVM_DEBUG(
dbgs() <<
"LAA: Found an unsafe dependency between a uniform "
2605 "load and uniform store to the same address!\n");
2606 HasLoadStoreDependenceInvolvingLoopInvariantAddress =
true;
2613 if (blockNeedsPredication(
LD->getParent(), TheLoop, DT))
2617 [&Accesses, AccessTy, Loc, IsReadOnlyPtr](
Value *
Ptr) {
2618 MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr);
2619 Accesses.addLoad(NewLoc, AccessTy, IsReadOnlyPtr);
2625 if (NumReadWrites == 1 && NumReads == 0) {
2632 Accesses.buildDependenceSets();
2636 Value *UncomputablePtr =
nullptr;
2637 bool CanDoRTIfNeeded =
2638 Accesses.canCheckPtrAtRT(*PtrRtChecking, PSE->
getSE(), TheLoop,
2639 SymbolicStrides, UncomputablePtr,
false);
2640 if (!CanDoRTIfNeeded) {
2641 auto *
I = dyn_cast_or_null<Instruction>(UncomputablePtr);
2642 recordAnalysis(
"CantIdentifyArrayBounds",
I)
2643 <<
"cannot identify array bounds";
2644 LLVM_DEBUG(
dbgs() <<
"LAA: We can't vectorize because we can't find "
2645 <<
"the array bounds.\n");
2650 dbgs() <<
"LAA: May be able to perform a memory runtime check if needed.\n");
2652 bool DepsAreSafe =
true;
2653 if (Accesses.isDependencyCheckNeeded()) {
2655 DepsAreSafe = DepChecker->
areDepsSafe(DependentAccesses,
2656 Accesses.getDependenciesToCheck(),
2657 Accesses.getUnderlyingObjects());
2663 Accesses.resetDepChecks(*DepChecker);
2665 PtrRtChecking->reset();
2666 PtrRtChecking->Need =
true;
2668 auto *SE = PSE->
getSE();
2669 UncomputablePtr =
nullptr;
2670 CanDoRTIfNeeded = Accesses.canCheckPtrAtRT(
2671 *PtrRtChecking, SE, TheLoop, SymbolicStrides, UncomputablePtr,
true);
2674 if (!CanDoRTIfNeeded) {
2675 auto *
I = dyn_cast_or_null<Instruction>(UncomputablePtr);
2676 recordAnalysis(
"CantCheckMemDepsAtRunTime",
I)
2677 <<
"cannot check memory dependencies at runtime";
2678 LLVM_DEBUG(
dbgs() <<
"LAA: Can't vectorize with memory checks\n");
2685 if (HasConvergentOp) {
2686 recordAnalysis(
"CantInsertRuntimeCheckWithConvergent")
2687 <<
"cannot add control dependency to convergent operation";
2688 LLVM_DEBUG(
dbgs() <<
"LAA: We can't vectorize because a runtime check "
2689 "would be needed with a convergent operation\n");
2695 dbgs() <<
"LAA: No unsafe dependent memory operations in loop. We"
2696 << (PtrRtChecking->Need ?
"" :
" don't")
2697 <<
" need runtime memory checks.\n");
2701 emitUnsafeDependenceRemark();
2705void LoopAccessInfo::emitUnsafeDependenceRemark() {
2706 const auto *Deps = getDepChecker().getDependences();
2714 if (Found == Deps->end())
2718 LLVM_DEBUG(
dbgs() <<
"LAA: unsafe dependent memory operations in loop\n");
2721 bool HasForcedDistribution =
false;
2722 std::optional<const MDOperand *>
Value =
2726 assert(
Op && mdconst::hasa<ConstantInt>(*
Op) &&
"invalid metadata");
2727 HasForcedDistribution = mdconst::extract<ConstantInt>(*Op)->getZExtValue();
2730 const std::string
Info =
2731 HasForcedDistribution
2732 ?
"unsafe dependent memory operations in loop."
2733 :
"unsafe dependent memory operations in loop. Use "
2734 "#pragma clang loop distribute(enable) to allow loop distribution "
2735 "to attempt to isolate the offending operations into a separate "
2746 R <<
"\nBackward loop carried data dependence.";
2749 R <<
"\nForward loop carried data dependence that prevents "
2750 "store-to-load forwarding.";
2753 R <<
"\nBackward loop carried data dependence that prevents "
2754 "store-to-load forwarding.";
2757 R <<
"\nUnsafe indirect dependence.";
2760 R <<
"\nUnknown data dependence.";
2767 SourceLoc = DD->getDebugLoc();
2769 R <<
" Memory location is the same as accessed at "
2770 <<
ore::NV(
"Location", SourceLoc);
2785 assert(!Report &&
"Multiple reports generated");
2791 CodeRegion =
I->getParent();
2794 if (
I->getDebugLoc())
2795 DL =
I->getDebugLoc();
2798 Report = std::make_unique<OptimizationRemarkAnalysis>(
DEBUG_TYPE, RemarkName,
DL,
2804 auto *SE = PSE->
getSE();
2825 std::advance(GEPTI, LastOperand - 2);
2832 if (ElemSize != GEPAllocSize)
2852 for (
unsigned I = 0, E =
GEP->getNumOperands();
I != E; ++
I)
2853 if (
I != InductionOperand &&
2856 return GEP->getOperand(InductionOperand);
2862 auto *PtrTy = dyn_cast<PointerType>(
Ptr->getType());
2863 if (!PtrTy || PtrTy->isAggregateType())
2872 int64_t PtrAccessSize = 1;
2880 V =
C->getOperand();
2897 if (OrigPtr ==
Ptr) {
2898 if (
const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(V)) {
2899 if (M->getOperand(0)->getSCEVType() !=
scConstant)
2902 const APInt &APStepVal = cast<SCEVConstant>(M->getOperand(0))->getAPInt();
2909 if (PtrAccessSize != StepVal)
2911 V = M->getOperand(1);
2921 if (isa<SCEVUnknown>(V))
2924 if (
const auto *
C = dyn_cast<SCEVIntegralCastExpr>(V))
2925 if (isa<SCEVUnknown>(
C->getOperand()))
2931void LoopAccessInfo::collectStridedAccess(
Value *MemAccess) {
2946 LLVM_DEBUG(
dbgs() <<
"LAA: Found a strided access that is a candidate for "
2951 LLVM_DEBUG(
dbgs() <<
" Chose not to due to -laa-speculate-unit-stride\n");
2976 const SCEV *CastedStride = StrideExpr;
2977 const SCEV *CastedBECount = MaxBTC;
2979 if (BETypeSizeBits >= StrideTypeSizeBits)
2983 const SCEV *StrideMinusBETaken = SE->
getMinusSCEV(CastedStride, CastedBECount);
2989 dbgs() <<
"LAA: Stride>=TripCount; No point in versioning as the "
2990 "Stride==1 predicate will imply that the loop executes "
2994 LLVM_DEBUG(
dbgs() <<
"LAA: Found a strided access that we can version.\n");
2998 const SCEV *StrideBase = StrideExpr;
2999 if (
const auto *
C = dyn_cast<SCEVIntegralCastExpr>(StrideBase))
3000 StrideBase =
C->getOperand();
3001 SymbolicStrides[
Ptr] = cast<SCEVUnknown>(StrideBase);
3009 PtrRtChecking(nullptr), TheLoop(L) {
3010 unsigned MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
3017 MaxTargetVectorWidthInBits = FixedWidth.
getFixedValue() * 2;
3023 MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
3025 DepChecker = std::make_unique<MemoryDepChecker>(*PSE, L, SymbolicStrides,
3026 MaxTargetVectorWidthInBits);
3027 PtrRtChecking = std::make_unique<RuntimePointerChecking>(*DepChecker, SE);
3028 if (canAnalyzeLoop())
3029 CanVecMem = analyzeLoop(AA, LI, TLI, DT);
3037 OS <<
" with a maximum safe vector width of "
3039 if (PtrRtChecking->Need)
3040 OS <<
" with run-time checks";
3044 if (HasConvergentOp)
3052 for (
const auto &Dep : *Dependences) {
3060 PtrRtChecking->print(
OS,
Depth);
3064 <<
"Non vectorizable stores to invariant address were "
3065 << (HasStoreStoreDependenceInvolvingLoopInvariantAddress ||
3066 HasLoadStoreDependenceInvolvingLoopInvariantAddress
3069 <<
"found in loop.\n";
3081 auto [It, Inserted] = LoopAccessInfoMap.insert({&L,
nullptr});
3085 std::make_unique<LoopAccessInfo>(&L, &SE,
TTI, TLI, &AA, &DT, &LI);
3095 for (
const auto &[L, LAI] : LoopAccessInfoMap) {
3096 if (LAI->getRuntimePointerChecking()->getChecks().empty() &&
3097 LAI->getPSE().getPredicate().isAlwaysTrue())
3103 LoopAccessInfoMap.erase(L);
This file implements a class to represent arbitrary precision integral constant values and operations...
ReachingDefAnalysis InstSet & ToRemove
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
Generic implementation of equivalence classes through the use Tarjan's efficient union-find algorithm...
static std::pair< const SCEV *, const SCEV * > getStartAndEndForAccess(const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy, PredicatedScalarEvolution &PSE, DenseMap< std::pair< const SCEV *, Type * >, std::pair< const SCEV *, const SCEV * > > &PointerBounds)
Calculate Start and End points of memory access.
static cl::opt< unsigned > MaxDependences("max-dependences", cl::Hidden, cl::desc("Maximum number of dependences collected by " "loop-access analysis (default = 100)"), cl::init(100))
We collect dependences up to this threshold.
static cl::opt< bool > EnableForwardingConflictDetection("store-to-load-forwarding-conflict-detection", cl::Hidden, cl::desc("Enable conflict detection in loop-access analysis"), cl::init(true))
Enable store-to-load forwarding conflict detection.
static void findForkedSCEVs(ScalarEvolution *SE, const Loop *L, Value *Ptr, SmallVectorImpl< PointerIntPair< const SCEV *, 1, bool > > &ScevList, unsigned Depth)
static bool hasComputableBounds(PredicatedScalarEvolution &PSE, Value *Ptr, const SCEV *PtrScev, Loop *L, bool Assume)
Check whether a pointer can participate in a runtime bounds check.
static cl::opt< unsigned > MemoryCheckMergeThreshold("memory-check-merge-threshold", cl::Hidden, cl::desc("Maximum number of comparisons done when trying to merge " "runtime memory checks. (default = 100)"), cl::init(100))
The maximum iterations used to merge memory checks.
static bool isNoWrap(PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &Strides, Value *Ptr, Type *AccessTy, Loop *L)
Check whether a pointer address cannot wrap.
static const SCEV * getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp)
Get the stride of a pointer access in a loop.
static unsigned getGEPInductionOperand(const GetElementPtrInst *Gep)
Find the operand of the GEP that should be checked for consecutive stores.
static cl::opt< unsigned, true > VectorizationInterleave("force-vector-interleave", cl::Hidden, cl::desc("Sets the vectorization interleave count. " "Zero is autoselect."), cl::location(VectorizerParams::VectorizationInterleave))
static bool isLoopVariantIndirectAddress(ArrayRef< const Value * > UnderlyingObjects, ScalarEvolution &SE, const Loop *L)
Returns true if any of the underlying objects has a loop varying address, i.e.
static cl::opt< bool, true > HoistRuntimeChecks("hoist-runtime-checks", cl::Hidden, cl::desc("Hoist inner loop runtime memory checks to outer loop if possible"), cl::location(VectorizerParams::HoistRuntimeChecks), cl::init(true))
static cl::opt< unsigned, true > VectorizationFactor("force-vector-width", cl::Hidden, cl::desc("Sets the SIMD width. Zero is autoselect."), cl::location(VectorizerParams::VectorizationFactor))
static bool isSafeDependenceDistance(const DataLayout &DL, ScalarEvolution &SE, const SCEV &MaxBTC, const SCEV &Dist, uint64_t MaxStride, uint64_t TypeByteSize)
Given a dependence-distance Dist between two memory accesses, that have strides in the same direction...
static cl::opt< unsigned, true > RuntimeMemoryCheckThreshold("runtime-memory-check-threshold", cl::Hidden, cl::desc("When performing memory disambiguation checks at runtime do not " "generate more than this number of comparisons (default = 8)."), cl::location(VectorizerParams::RuntimeMemoryCheckThreshold), cl::init(8))
static void visitPointers(Value *StartPtr, const Loop &InnermostLoop, function_ref< void(Value *)> AddPointer)
static bool isNoWrapAddRec(Value *Ptr, const SCEVAddRecExpr *AR, PredicatedScalarEvolution &PSE, const Loop *L)
Return true if an AddRec pointer Ptr is unsigned non-wrapping, i.e.
static Value * stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp)
If the argument is a GEP, then returns the operand identified by getGEPInductionOperand.
static bool areStridedAccessesIndependent(uint64_t Distance, uint64_t Stride, uint64_t TypeByteSize)
Check the dependence for two accesses with the same stride Stride.
static const SCEV * getMinFromExprs(const SCEV *I, const SCEV *J, ScalarEvolution *SE)
Compare I and J and return the minimum.
static cl::opt< unsigned > MaxForkedSCEVDepth("max-forked-scev-depth", cl::Hidden, cl::desc("Maximum recursion depth when finding forked SCEVs (default = 5)"), cl::init(5))
static cl::opt< bool > SpeculateUnitStride("laa-speculate-unit-stride", cl::Hidden, cl::desc("Speculate that non-constant strides are unit in LAA"), cl::init(true))
static SmallVector< PointerIntPair< const SCEV *, 1, bool > > findForkedPointer(PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &StridesMap, Value *Ptr, const Loop *L)
static cl::opt< bool > EnableMemAccessVersioning("enable-mem-access-versioning", cl::init(true), cl::Hidden, cl::desc("Enable symbolic stride memory access versioning"))
This enables versioning on the strides of symbolically striding memory accesses in code like the foll...
This header provides classes for managing per-loop analyses.
This file provides utility analysis objects describing memory locations.
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
This file defines the PointerIntPair class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
static const X86InstrFMA3Group Groups[]
A manager for alias analyses.
Class for arbitrary precision integers.
unsigned getBitWidth() const
Return the number of bits in the APInt.
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
int64_t getSExtValue() const
Get sign extended value.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
API to communicate dependencies between analyses during invalidation.
bool invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Trigger the invalidation of some other analysis pass if not already handled and return whether it was...
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
@ ICMP_ULE
unsigned less or equal
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
iterator findValue(const ElemTy &V) const
findValue - Return an iterator to the specified value.
iterator insert(const ElemTy &Data)
insert - Insert a new value into the union/find set, ignoring the request if the value already exists...
member_iterator member_end() const
typename std::set< ECValue, ECValueComparator >::const_iterator iterator
iterator* - Provides a way to iterate over all values in the set.
member_iterator member_begin(iterator I) const
member_iterator unionSets(const ElemTy &V1, const ElemTy &V2)
union - Merge the two equivalence sets for the specified values, inserting them if they do not alread...
const ElemTy & getLeaderValue(const ElemTy &V) const
getLeaderValue - Return the leader for the specified value that is in the set.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Type * getResultElementType() const
PointerType * getType() const
Global values are always pointers.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
An instruction for reading from memory.
Value * getPointerOperand()
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
This analysis provides dependence information for the memory accesses of a loop.
Result run(Function &F, FunctionAnalysisManager &AM)
bool invalidate(Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &Inv)
const LoopAccessInfo & getInfo(Loop &L)
Drive the analysis of memory accesses in the loop.
const MemoryDepChecker & getDepChecker() const
the Memory Dependence Checker which can determine the loop-independent and loop-carried dependences b...
bool isInvariant(Value *V) const
Returns true if value V is loop invariant.
void print(raw_ostream &OS, unsigned Depth=0) const
Print the information about the memory accesses in the loop.
static bool blockNeedsPredication(BasicBlock *BB, Loop *TheLoop, DominatorTree *DT)
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetTransformInfo *TTI, const TargetLibraryInfo *TLI, AAResults *AA, DominatorTree *DT, LoopInfo *LI)
Analysis pass that exposes the LoopInfo for a function.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
unsigned getNumBackEdges() const
Calculate the number of back edges to the loop header.
BlockT * getHeader() const
LoopT * getParentLoop() const
Return the parent loop if it exists or nullptr for top level loops.
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
Represents a single loop in the control flow graph.
std::string getLocStr() const
Return a string containing the debug location of the loop (file name + line number if present,...
bool isAnnotatedParallel() const
Returns true if the loop is annotated parallel.
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
ArrayRef< MDOperand > operands() const
Tracking metadata reference owned by Metadata.
This class implements a map that also provides access to all stored values in a deterministic order.
Checks memory dependences among accesses to the same underlying object to determine whether there vec...
ArrayRef< unsigned > getOrderForAccess(Value *Ptr, bool IsWrite) const
Return the program order indices for the access location (Ptr, IsWrite).
bool areDepsSafe(DepCandidates &AccessSets, MemAccessInfoList &CheckDeps, const DenseMap< Value *, SmallVector< const Value *, 16 > > &UnderlyingObjects)
Check whether the dependencies between the accesses are safe.
bool isSafeForAnyVectorWidth() const
Return true if the number of elements that are safe to operate on simultaneously is not bounded.
const SmallVectorImpl< Instruction * > & getMemoryInstructions() const
The vector of memory access instructions.
const Loop * getInnermostLoop() const
uint64_t getMaxSafeVectorWidthInBits() const
Return the number of elements that are safe to operate on simultaneously, multiplied by the size of t...
bool isSafeForVectorization() const
No memory dependence was encountered that would inhibit vectorization.
const SmallVectorImpl< Dependence > * getDependences() const
Returns the memory dependences.
DenseMap< std::pair< const SCEV *, Type * >, std::pair< const SCEV *, const SCEV * > > & getPointerBounds()
SmallVector< Instruction *, 4 > getInstructionsForAccess(Value *Ptr, bool isWrite) const
Find the set of instructions that read or write via Ptr.
VectorizationSafetyStatus
Type to keep track of the status of the dependence check.
@ PossiblySafeWithRtChecks
bool shouldRetryWithRuntimeCheck() const
In same cases when the dependency check fails we can still vectorize the loop with a dynamic array ac...
void addAccess(StoreInst *SI)
Register the location (instructions are given increasing numbers) of a write access.
PointerIntPair< Value *, 1, bool > MemAccessInfo
Representation for a specific memory location.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
LocationSize Size
The maximum size of the location, in address-units, or UnknownSize if the size is not known.
AAMDNodes AATags
The metadata nodes which describes the aliasing of the location (each member is null if that kind of ...
const Value * Ptr
The address of the start of the location.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
void addPredicate(const SCEVPredicate &Pred)
Adds a new predicate.
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
const SCEVPredicate & getPredicate() const
bool hasNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Returns true if we've proved that V doesn't wrap by means of a SCEV predicate.
void setNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Proves that V doesn't overflow by adding SCEV predicate.
void print(raw_ostream &OS, unsigned Depth) const
Print the SCEV mappings done by the Predicated Scalar Evolution.
const SCEVAddRecExpr * getAsAddRec(Value *V)
Attempts to produce an AddRecExpr for V by adding additional SCEV predicates.
const SCEV * getSymbolicMaxBackedgeTakenCount()
Get the (predicated) symbolic max backedge count for the analyzed loop.
const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
PreservedAnalysisChecker getChecker() const
Build a checker for this PreservedAnalyses and the specified analysis type.
Holds information about the memory runtime legality checks to verify that a group of pointers do not ...
bool Need
This flag indicates if we need to add the runtime check.
void reset()
Reset the state of the pointer runtime information.
unsigned getNumberOfChecks() const
Returns the number of run-time checks required according to needsChecking.
void printChecks(raw_ostream &OS, const SmallVectorImpl< RuntimePointerCheck > &Checks, unsigned Depth=0) const
Print Checks.
bool needsChecking(const RuntimeCheckingPtrGroup &M, const RuntimeCheckingPtrGroup &N) const
Decide if we need to add a check between two groups of pointers, according to needsChecking.
void print(raw_ostream &OS, unsigned Depth=0) const
Print the list run-time memory checks necessary.
SmallVector< RuntimeCheckingPtrGroup, 2 > CheckingGroups
Holds a partitioning of pointers into "check groups".
void generateChecks(MemoryDepChecker::DepCandidates &DepCands, bool UseDependencies)
Generate the checks and store it.
friend struct RuntimeCheckingPtrGroup
static bool arePointersInSamePartition(const SmallVectorImpl< int > &PtrToPartition, unsigned PtrIdx1, unsigned PtrIdx2)
Check if pointers are in the same partition.
SmallVector< PointerInfo, 2 > Pointers
Information about the pointers that may require checking.
void insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr, Type *AccessTy, bool WritePtr, unsigned DepSetId, unsigned ASId, PredicatedScalarEvolution &PSE, bool NeedsFreeze)
Insert a pointer and calculate the start and end SCEVs.
This node represents a polynomial recurrence on the trip count of the specified loop.
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values.
const Loop * getLoop() const
This class represents a constant integer value.
This is the base class for unary integral cast operator classes.
This node represents multiplication of some number of SCEVs.
NoWrapFlags getNoWrapFlags(NoWrapFlags Mask=NoWrapMask) const
virtual void print(raw_ostream &OS, unsigned Depth=0) const =0
Prints a textual representation of this predicate with an indentation of Depth.
This class represents an analyzed expression in the program.
Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
bool isKnownNonNegative(const SCEV *S)
Test if the given expression is known to be non-negative.
const SCEV * getNegativeSCEV(const SCEV *V, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return the SCEV object corresponding to -V.
bool isKnownNonPositive(const SCEV *S)
Test if the given expression is known to be non-positive.
const SCEV * getUMaxExpr(const SCEV *LHS, const SCEV *RHS)
const SCEVPredicate * getEqualPredicate(const SCEV *LHS, const SCEV *RHS)
const SCEV * getConstant(ConstantInt *V)
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
const SCEV * getNoopOrSignExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
const SCEV * getPtrToIntExpr(const SCEV *Op, Type *Ty)
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
bool isKnownPositive(const SCEV *S)
Test if the given expression is known to be positive.
bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
const SCEV * getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth=0)
bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
Type * getEffectiveSCEVType(Type *Ty) const
Return a type with the same bitwidth as the given type and which represents how SCEV will treat the g...
const SCEV * getUMinExpr(const SCEV *LHS, const SCEV *RHS, bool Sequential=false)
APInt getSignedRangeMin(const SCEV *S)
Determine the min of the signed range for a particular SCEV.
const SCEV * getStoreSizeOfExpr(Type *IntTy, Type *StoreTy)
Return an expression for the store size of StoreTy that is type IntTy.
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Return LHS-RHS.
const SCEV * getCouldNotCompute()
const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
const SCEV * getSizeOfExpr(Type *IntTy, TypeSize Size)
Return an expression for a TypeSize.
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
const SCEV * getTruncateOrSignExtend(const SCEV *V, Type *Ty, unsigned Depth=0)
Return a SCEV corresponding to a conversion of the input value to the specified type.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
const Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset) const
This is a wrapper around stripAndAccumulateConstantOffsets with the in-bounds requirement set to fals...
StringRef getName() const
Return a constant reference to the value's name.
constexpr ScalarTy getFixedValue() const
constexpr bool isNonZero() const
An efficient, type-erasing, non-owning reference to a callable.
TypeSize getSequentialElementStride(const DataLayout &DL) const
Type * getIndexedType() const
This class implements an extremely fast bulk output stream that can only output to a stream.
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
friend const_iterator end(StringRef path)
Get end iterator over path.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
bool match(Val *V, const Pattern &P)
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
std::optional< int > getPointersDiff(Type *ElemTyA, Value *PtrA, Type *ElemTyB, Value *PtrB, const DataLayout &DL, ScalarEvolution &SE, bool StrictCheck=false, bool CheckType=true)
Returns the distance between the pointers PtrA and PtrB iff they are compatible and it is possible to...
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
unsigned getPointerAddressSpace(const Type *T)
std::optional< const MDOperand * > findStringMetadataForLoop(const Loop *TheLoop, StringRef Name)
Find string metadata for loop.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
std::optional< int64_t > getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp, const DenseMap< Value *, const SCEV * > &StridesMap=DenseMap< Value *, const SCEV * >(), bool Assume=false, bool ShouldCheckWrap=true)
If the pointer has a constant stride return it in units of the access type size.
bool sortPtrAccesses(ArrayRef< Value * > VL, Type *ElemTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< unsigned > &SortedIndices)
Attempt to sort the pointers in VL and return the sorted indices in SortedIndices,...
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
const SCEV * replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &PtrToStride, Value *Ptr)
Return the SCEV corresponding to a pointer with the symbolic stride replaced with constant one,...
bool isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL, ScalarEvolution &SE, bool CheckType=true)
Returns true if the memory operations A and B are consecutive.
bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
OutputIt copy(R &&Range, OutputIt Out)
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
gep_type_iterator gep_type_begin(const User *GEP)
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, const LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
Type * getLoadStoreType(Value *I)
A helper function that returns the type of a load or store instruction.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
IR Values for the lower and upper bounds of a pointer evolution.
MDNode * Scope
The tag for alias scope specification (used with noalias).
MDNode * TBAA
The tag for type-based alias analysis.
MDNode * NoAlias
The tag specifying the noalias scope.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Dependece between memory access instructions.
Instruction * getDestination(const MemoryDepChecker &DepChecker) const
Return the destination instruction of the dependence.
DepType Type
The type of the dependence.
bool isPossiblyBackward() const
May be a lexically backward dependence type (includes Unknown).
Instruction * getSource(const MemoryDepChecker &DepChecker) const
Return the source instruction of the dependence.
bool isForward() const
Lexically forward dependence.
bool isBackward() const
Lexically backward dependence.
void print(raw_ostream &OS, unsigned Depth, const SmallVectorImpl< Instruction * > &Instrs) const
Print the dependence.
DepType
The type of the dependence.
@ BackwardVectorizableButPreventsForwarding
@ ForwardButPreventsForwarding
static const char * DepName[]
String version of the types.
static VectorizationSafetyStatus isSafeForVectorization(DepType Type)
Dependence types that don't prevent vectorization.
unsigned AddressSpace
Address space of the involved pointers.
bool addPointer(unsigned Index, RuntimePointerChecking &RtCheck)
Tries to add the pointer recorded in RtCheck at index Index to this pointer checking group.
bool NeedsFreeze
Whether the pointer needs to be frozen after expansion, e.g.
const SCEV * High
The SCEV expression which represents the upper bound of all the pointers in this group.
SmallVector< unsigned, 2 > Members
Indices of all the pointers that constitute this grouping.
RuntimeCheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck)
Create a new pointer checking group containing a single pointer, with index Index in RtCheck.
const SCEV * Low
The SCEV expression which represents the lower bound of all the pointers in this group.
bool IsWritePtr
Holds the information if this pointer is used for writing to memory.
unsigned DependencySetId
Holds the id of the set of pointers that could be dependent because of a shared underlying object.
unsigned AliasSetId
Holds the id of the disjoint alias set to which this pointer belongs.
static const unsigned MaxVectorWidth
Maximum SIMD width.
static unsigned VectorizationFactor
VF as overridden by the user.
static unsigned RuntimeMemoryCheckThreshold
\When performing memory disambiguation checks at runtime do not make more than this number of compari...
static bool isInterleaveForced()
True if force-vector-interleave was specified by the user.
static unsigned VectorizationInterleave
Interleave factor as overridden by the user.
static bool HoistRuntimeChecks
Function object to check whether the first component of a container supported by std::get (like std::...