71#define DEBUG_TYPE "loop-accesses"
75 cl::desc(
"Sets the SIMD width. Zero is autoselect."),
81 cl::desc(
"Sets the vectorization interleave count. "
82 "Zero is autoselect."),
89 cl::desc(
"When performing memory disambiguation checks at runtime do not "
90 "generate more than this number of comparisons (default = 8)."),
97 cl::desc(
"Maximum number of comparisons done when trying to merge "
98 "runtime memory checks. (default = 100)"),
107 cl::desc(
"Maximum number of dependences collected by "
108 "loop-access analysis (default = 100)"),
124 cl::desc(
"Enable symbolic stride memory access versioning"));
129 "store-to-load-forwarding-conflict-detection",
cl::Hidden,
130 cl::desc(
"Enable conflict detection in loop-access analysis"),
135 cl::desc(
"Maximum recursion depth when finding forked SCEVs (default = 5)"),
140 cl::desc(
"Speculate that non-constant strides are unit in LAA"),
146 "Hoist inner loop runtime memory checks to outer loop if possible"),
151 return ::VectorizationInterleave.getNumOccurrences() > 0;
162 if (SI == PtrToStride.
end())
166 const SCEV *StrideSCEV = SI->second;
171 assert(isa<SCEVUnknown>(StrideSCEV) &&
"shouldn't be in map");
179 <<
" by: " << *Expr <<
"\n");
189 NeedsFreeze(RtCheck.Pointers[
Index].NeedsFreeze) {
206static std::pair<const SCEV *, const SCEV *>
215 ScStart = ScEnd = PtrExpr;
216 }
else if (
auto *AR = dyn_cast<SCEVAddRecExpr>(PtrExpr)) {
219 ScStart = AR->getStart();
220 ScEnd = AR->evaluateAtIteration(Ex, *SE);
221 const SCEV *Step = AR->getStepRecurrence(*SE);
225 if (
const auto *CStep = dyn_cast<SCEVConstant>(Step)) {
226 if (CStep->getValue()->isNegative())
247 return {ScStart, ScEnd};
253 Type *AccessTy,
bool WritePtr,
254 unsigned DepSetId,
unsigned ASId,
257 const auto &[ScStart, ScEnd] =
259 assert(!isa<SCEVCouldNotCompute>(ScStart) &&
260 !isa<SCEVCouldNotCompute>(ScEnd) &&
261 "must be able to compute both start and end expressions");
262 Pointers.emplace_back(
Ptr, ScStart, ScEnd, WritePtr, DepSetId, ASId, PtrExpr,
266bool RuntimePointerChecking::tryToCreateDiffCheck(
289 if (AccSrc.
size() != 1 || AccSink.
size() != 1)
293 if (AccSink[0] < AccSrc[0])
296 auto *SrcAR = dyn_cast<SCEVAddRecExpr>(Src->Expr);
297 auto *SinkAR = dyn_cast<SCEVAddRecExpr>(Sink->Expr);
308 if (isa<ScalableVectorType>(SrcTy) || isa<ScalableVectorType>(DstTy))
312 SinkAR->getLoop()->getHeader()->getDataLayout();
314 std::max(
DL.getTypeAllocSize(SrcTy),
DL.getTypeAllocSize(DstTy));
319 auto *Step = dyn_cast<SCEVConstant>(SinkAR->getStepRecurrence(*SE));
320 if (!Step || Step != SrcAR->getStepRecurrence(*SE) ||
321 Step->getAPInt().abs() != AllocSize)
329 if (Step->getValue()->isNegative())
334 if (isa<SCEVCouldNotCompute>(SinkStartInt) ||
335 isa<SCEVCouldNotCompute>(SrcStartInt))
338 const Loop *InnerLoop = SrcAR->getLoop();
344 isa<SCEVAddRecExpr>(SinkStartInt) && isa<SCEVAddRecExpr>(SrcStartInt)) {
345 auto *SrcStartAR = cast<SCEVAddRecExpr>(SrcStartInt);
346 auto *SinkStartAR = cast<SCEVAddRecExpr>(SinkStartInt);
347 const Loop *StartARLoop = SrcStartAR->getLoop();
348 if (StartARLoop == SinkStartAR->getLoop() &&
353 SrcStartAR->getStepRecurrence(*SE) !=
354 SinkStartAR->getStepRecurrence(*SE)) {
355 LLVM_DEBUG(
dbgs() <<
"LAA: Not creating diff runtime check, since these "
356 "cannot be hoisted out of the outer loop\n");
362 <<
"SrcStart: " << *SrcStartInt <<
'\n'
363 <<
"SinkStartInt: " << *SinkStartInt <<
'\n');
364 DiffChecks.emplace_back(SrcStartInt, SinkStartInt, AllocSize,
365 Src->NeedsFreeze ||
Sink->NeedsFreeze);
378 CanUseDiffCheck = CanUseDiffCheck && tryToCreateDiffCheck(CGI, CGJ);
379 Checks.
push_back(std::make_pair(&CGI, &CGJ));
386void RuntimePointerChecking::generateChecks(
389 groupChecks(DepCands, UseDependencies);
395 for (
const auto &
I : M.Members)
396 for (
const auto &J :
N.Members)
411 return C->getValue()->isNegative() ? J :
I;
418 RtCheck.
Pointers[
Index].PointerValue->getType()->getPointerAddressSpace(),
427 "all pointers in a checking group must be in the same address space");
453void RuntimePointerChecking::groupChecks(
499 if (!UseDependencies) {
505 unsigned TotalComparisons = 0;
510 It->second.push_back(
Index);
539 auto PointerI = PositionMap.
find(
MI->getPointer());
541 "pointer in equivalence class not found in PositionMap");
542 for (
unsigned Pointer : PointerI->second) {
559 if (Group.addPointer(Pointer, *
this)) {
582 return (PtrToPartition[PtrIdx1] != -1 &&
583 PtrToPartition[PtrIdx1] == PtrToPartition[PtrIdx2]);
607 unsigned Depth)
const {
609 for (
const auto &[Check1, Check2] : Checks) {
610 const auto &
First = Check1->Members, &Second = Check2->Members;
614 OS.
indent(
Depth + 2) <<
"Comparing group (" << Check1 <<
"):\n";
615 for (
unsigned K :
First)
618 OS.
indent(
Depth + 2) <<
"Against group (" << Check2 <<
"):\n";
619 for (
unsigned K : Second)
632 OS.
indent(
Depth + 4) <<
"(Low: " << *CG.Low <<
" High: " << *CG.High
634 for (
unsigned Member : CG.Members) {
646class AccessAnalysis {
656 : TheLoop(TheLoop), BAA(*AA), AST(BAA), LI(LI), DepCands(DA), PSE(PSE),
657 LoopAliasScopes(LoopAliasScopes) {
659 BAA.enableCrossIterationMode();
665 AST.add(adjustLoc(Loc));
666 Accesses[MemAccessInfo(
Ptr,
false)].insert(AccessTy);
668 ReadOnlyPtr.insert(
Ptr);
674 AST.add(adjustLoc(Loc));
675 Accesses[MemAccessInfo(
Ptr,
true)].insert(AccessTy);
686 MemAccessInfo Access,
Type *AccessTy,
689 Loop *TheLoop,
unsigned &RunningDepId,
690 unsigned ASId,
bool ShouldCheckStride,
bool Assume);
699 Value *&UncomputablePtr,
bool ShouldCheckWrap =
false);
703 void buildDependenceSets() {
704 processMemAccesses();
712 bool isDependencyCheckNeeded() {
return !CheckDeps.empty(); }
720 MemAccessInfoList &getDependenciesToCheck() {
return CheckDeps; }
724 return UnderlyingObjects;
749 return LoopAliasScopes.contains(cast<MDNode>(Scope));
758 void processMemAccesses();
762 PtrAccessMap Accesses;
768 MemAccessInfoList CheckDeps;
794 bool IsRTCheckAnalysisNeeded =
false;
812 const SCEV *PtrScev,
Loop *L,
bool Assume) {
836 int64_t Stride =
getPtrStride(PSE, AccessTy,
Ptr, L, Strides).value_or(0);
849 while (!WorkList.
empty()) {
853 auto *PN = dyn_cast<PHINode>(
Ptr);
857 if (PN && InnermostLoop.
contains(PN->getParent()) &&
858 PN->getParent() != InnermostLoop.
getHeader()) {
859 for (
const Use &Inc : PN->incoming_values())
892 if (isa<SCEVAddRecExpr>(Scev) || L->isLoopInvariant(
Ptr) ||
893 !isa<Instruction>(
Ptr) ||
Depth == 0) {
904 auto GetBinOpExpr = [&SE](
unsigned Opcode,
const SCEV *L,
const SCEV *R) {
906 case Instruction::Add:
908 case Instruction::Sub:
916 unsigned Opcode =
I->getOpcode();
918 case Instruction::GetElementPtr: {
920 Type *SourceTy =
GEP->getSourceElementType();
923 if (
I->getNumOperands() != 2 || SourceTy->
isVectorTy()) {
933 bool NeedsFreeze =
any_of(BaseScevs, UndefPoisonCheck) ||
934 any_of(OffsetScevs, UndefPoisonCheck);
939 if (OffsetScevs.
size() == 2 && BaseScevs.
size() == 1)
941 else if (BaseScevs.
size() == 2 && OffsetScevs.
size() == 1)
944 ScevList.emplace_back(Scev, NeedsFreeze);
962 ScevList.emplace_back(SE->
getAddExpr(get<0>(BaseScevs[0]), Scaled1),
964 ScevList.emplace_back(SE->
getAddExpr(get<0>(BaseScevs[1]), Scaled2),
968 case Instruction::Select: {
975 if (ChildScevs.
size() == 2) {
976 ScevList.push_back(ChildScevs[0]);
977 ScevList.push_back(ChildScevs[1]);
982 case Instruction::PHI: {
987 if (
I->getNumOperands() == 2) {
991 if (ChildScevs.
size() == 2) {
992 ScevList.push_back(ChildScevs[0]);
993 ScevList.push_back(ChildScevs[1]);
998 case Instruction::Add:
999 case Instruction::Sub: {
1007 any_of(LScevs, UndefPoisonCheck) ||
any_of(RScevs, UndefPoisonCheck);
1012 if (LScevs.
size() == 2 && RScevs.
size() == 1)
1014 else if (RScevs.
size() == 2 && LScevs.
size() == 1)
1017 ScevList.emplace_back(Scev, NeedsFreeze);
1021 ScevList.emplace_back(
1022 GetBinOpExpr(Opcode, get<0>(LScevs[0]), get<0>(RScevs[0])),
1024 ScevList.emplace_back(
1025 GetBinOpExpr(Opcode, get<0>(LScevs[1]), get<0>(RScevs[1])),
1031 LLVM_DEBUG(
dbgs() <<
"ForkedPtr unhandled instruction: " << *
I <<
"\n");
1048 if (Scevs.
size() == 2 &&
1049 (isa<SCEVAddRecExpr>(get<0>(Scevs[0])) ||
1051 (isa<SCEVAddRecExpr>(get<0>(Scevs[1])) ||
1063 MemAccessInfo Access,
Type *AccessTy,
1066 Loop *TheLoop,
unsigned &RunningDepId,
1067 unsigned ASId,
bool ShouldCheckWrap,
1074 for (
auto &
P : TranslatedPtrs) {
1075 const SCEV *PtrExpr = get<0>(
P);
1081 if (ShouldCheckWrap) {
1083 if (TranslatedPtrs.size() > 1)
1086 if (!
isNoWrap(PSE, StridesMap,
Ptr, AccessTy, TheLoop)) {
1088 if (!Assume || !isa<SCEVAddRecExpr>(Expr))
1095 if (TranslatedPtrs.size() == 1)
1100 for (
auto [PtrExpr, NeedsFreeze] : TranslatedPtrs) {
1104 if (isDependencyCheckNeeded()) {
1106 unsigned &LeaderId = DepSetId[Leader];
1108 LeaderId = RunningDepId++;
1112 DepId = RunningDepId++;
1114 bool IsWrite = Access.getInt();
1115 RtCheck.
insert(TheLoop,
Ptr, PtrExpr, AccessTy, IsWrite, DepId, ASId, PSE,
1126 Value *&UncomputablePtr,
bool ShouldCheckWrap) {
1129 bool CanDoRT =
true;
1131 bool MayNeedRTCheck =
false;
1132 if (!IsRTCheckAnalysisNeeded)
return true;
1134 bool IsDepCheckNeeded = isDependencyCheckNeeded();
1139 for (
auto &AS : AST) {
1140 int NumReadPtrChecks = 0;
1141 int NumWritePtrChecks = 0;
1142 bool CanDoAliasSetRT =
true;
1144 auto ASPointers = AS.getPointers();
1148 unsigned RunningDepId = 1;
1156 for (
const Value *ConstPtr : ASPointers) {
1158 bool IsWrite = Accesses.count(MemAccessInfo(
Ptr,
true));
1160 ++NumWritePtrChecks;
1168 if (NumWritePtrChecks == 0 ||
1169 (NumWritePtrChecks == 1 && NumReadPtrChecks == 0)) {
1170 assert((ASPointers.size() <= 1 ||
1173 MemAccessInfo AccessWrite(
const_cast<Value *
>(
Ptr),
1175 return DepCands.
findValue(AccessWrite) == DepCands.
end();
1177 "Can only skip updating CanDoRT below, if all entries in AS "
1178 "are reads or there is at most 1 entry");
1182 for (
auto &Access : AccessInfos) {
1183 for (
const auto &AccessTy : Accesses[Access]) {
1184 if (!createCheckForAccess(RtCheck, Access, AccessTy, StridesMap,
1185 DepSetId, TheLoop, RunningDepId, ASId,
1186 ShouldCheckWrap,
false)) {
1188 << *Access.getPointer() <<
'\n');
1190 CanDoAliasSetRT =
false;
1204 bool NeedsAliasSetRTCheck = RunningDepId > 2 || !Retries.
empty();
1208 if (NeedsAliasSetRTCheck && !CanDoAliasSetRT) {
1212 CanDoAliasSetRT =
true;
1213 for (
const auto &[Access, AccessTy] : Retries) {
1214 if (!createCheckForAccess(RtCheck, Access, AccessTy, StridesMap,
1215 DepSetId, TheLoop, RunningDepId, ASId,
1216 ShouldCheckWrap,
true)) {
1217 CanDoAliasSetRT =
false;
1218 UncomputablePtr = Access.getPointer();
1224 CanDoRT &= CanDoAliasSetRT;
1225 MayNeedRTCheck |= NeedsAliasSetRTCheck;
1234 unsigned NumPointers = RtCheck.
Pointers.size();
1235 for (
unsigned i = 0; i < NumPointers; ++i) {
1236 for (
unsigned j = i + 1;
j < NumPointers; ++
j) {
1238 if (RtCheck.
Pointers[i].DependencySetId ==
1239 RtCheck.
Pointers[j].DependencySetId)
1252 dbgs() <<
"LAA: Runtime check would require comparison between"
1253 " different address spaces\n");
1259 if (MayNeedRTCheck && CanDoRT)
1263 <<
" pointer comparisons.\n");
1270 bool CanDoRTIfNeeded = !RtCheck.
Need || CanDoRT;
1271 if (!CanDoRTIfNeeded)
1273 return CanDoRTIfNeeded;
1276void AccessAnalysis::processMemAccesses() {
1283 LLVM_DEBUG(
dbgs() <<
"LAA: Accesses(" << Accesses.size() <<
"):\n");
1285 for (
const auto &[
A,
_] : Accesses)
1286 dbgs() <<
"\t" << *
A.getPointer() <<
" ("
1287 << (
A.getInt() ?
"write"
1288 : (ReadOnlyPtr.count(
A.getPointer()) ?
"read-only"
1297 for (
const auto &AS : AST) {
1301 auto ASPointers = AS.getPointers();
1303 bool SetHasWrite =
false;
1307 UnderlyingObjToAccessMap ObjToLastAccess;
1310 PtrAccessMap DeferredAccesses;
1314 for (
int SetIteration = 0; SetIteration < 2; ++SetIteration) {
1315 bool UseDeferred = SetIteration > 0;
1316 PtrAccessMap &S = UseDeferred ? DeferredAccesses : Accesses;
1318 for (
const Value *ConstPtr : ASPointers) {
1323 for (
const auto &[AC,
_] : S) {
1324 if (AC.getPointer() !=
Ptr)
1327 bool IsWrite = AC.getInt();
1331 bool IsReadOnlyPtr = ReadOnlyPtr.count(
Ptr) && !IsWrite;
1332 if (UseDeferred && !IsReadOnlyPtr)
1336 assert(((IsReadOnlyPtr && UseDeferred) || IsWrite ||
1337 S.count(MemAccessInfo(
Ptr,
false))) &&
1338 "Alias-set pointer not in the access set?");
1340 MemAccessInfo Access(
Ptr, IsWrite);
1348 if (!UseDeferred && IsReadOnlyPtr) {
1351 DeferredAccesses.insert({Access, {}});
1359 if ((IsWrite || IsReadOnlyPtr) && SetHasWrite) {
1360 CheckDeps.push_back(Access);
1361 IsRTCheckAnalysisNeeded =
true;
1370 ValueVector TempObjects;
1372 UnderlyingObjects[
Ptr] = {};
1376 <<
"Underlying objects for pointer " << *
Ptr <<
"\n");
1377 for (
const Value *UnderlyingObj : UOs) {
1380 if (isa<ConstantPointerNull>(UnderlyingObj) &&
1386 UnderlyingObjToAccessMap::iterator Prev =
1387 ObjToLastAccess.find(UnderlyingObj);
1388 if (Prev != ObjToLastAccess.end())
1389 DepCands.
unionSets(Access, Prev->second);
1391 ObjToLastAccess[UnderlyingObj] = Access;
1420 auto *
GEP = dyn_cast<GetElementPtrInst>(
Ptr);
1421 if (!
GEP || !
GEP->isInBounds())
1425 Value *NonConstIndex =
nullptr;
1427 if (!isa<ConstantInt>(
Index)) {
1430 NonConstIndex =
Index;
1438 if (
auto *OBO = dyn_cast<OverflowingBinaryOperator>(NonConstIndex))
1439 if (OBO->hasNoSignedWrap() &&
1442 isa<ConstantInt>(OBO->getOperand(1))) {
1443 auto *OpScev = PSE.
getSCEV(OBO->getOperand(0));
1445 if (
auto *OpAR = dyn_cast<SCEVAddRecExpr>(OpScev))
1446 return OpAR->getLoop() == L && OpAR->getNoWrapFlags(
SCEV::FlagNSW);
1457 bool Assume,
bool ShouldCheckWrap) {
1461 if (isa<ScalableVectorType>(AccessTy)) {
1462 LLVM_DEBUG(
dbgs() <<
"LAA: Bad stride - Scalable object: " << *AccessTy
1464 return std::nullopt;
1475 <<
" SCEV: " << *PtrScev <<
"\n");
1476 return std::nullopt;
1481 LLVM_DEBUG(
dbgs() <<
"LAA: Bad stride - Not striding over innermost loop "
1482 << *
Ptr <<
" SCEV: " << *AR <<
"\n");
1483 return std::nullopt;
1493 <<
" SCEV: " << *AR <<
"\n");
1494 return std::nullopt;
1498 TypeSize AllocSize =
DL.getTypeAllocSize(AccessTy);
1500 const APInt &APStepVal =
C->getAPInt();
1504 return std::nullopt;
1509 int64_t Stride = StepVal /
Size;
1510 int64_t Rem = StepVal %
Size;
1512 return std::nullopt;
1514 if (!ShouldCheckWrap)
1526 if (
auto *
GEP = dyn_cast<GetElementPtrInst>(
Ptr);
1527 GEP &&
GEP->isInBounds() && (Stride == 1 || Stride == -1))
1535 (Stride == 1 || Stride == -1))
1541 <<
"LAA: Pointer: " << *
Ptr <<
"\n"
1542 <<
"LAA: SCEV: " << *AR <<
"\n"
1543 <<
"LAA: Added an overflow assumption\n");
1547 dbgs() <<
"LAA: Bad stride - Pointer may wrap in the address space "
1548 << *
Ptr <<
" SCEV: " << *AR <<
"\n");
1549 return std::nullopt;
1557 assert(PtrA && PtrB &&
"Expected non-nullptr pointers.");
1565 return std::nullopt;
1572 return std::nullopt;
1573 unsigned IdxWidth =
DL.getIndexSizeInBits(ASA);
1575 APInt OffsetA(IdxWidth, 0), OffsetB(IdxWidth, 0);
1580 if (PtrA1 == PtrB1) {
1583 ASA = cast<PointerType>(PtrA1->
getType())->getAddressSpace();
1584 ASB = cast<PointerType>(PtrB1->
getType())->getAddressSpace();
1587 return std::nullopt;
1589 IdxWidth =
DL.getIndexSizeInBits(ASA);
1590 OffsetA = OffsetA.sextOrTrunc(IdxWidth);
1600 dyn_cast<SCEVConstant>(SE.
getMinusSCEV(PtrSCEVB, PtrSCEVA));
1602 return std::nullopt;
1603 Val = Diff->getAPInt().getSExtValue();
1605 int Size =
DL.getTypeStoreSize(ElemTyA);
1606 int Dist = Val /
Size;
1610 if (!StrictCheck || Dist *
Size == Val)
1612 return std::nullopt;
1620 "Expected list of pointer operands.");
1623 Value *Ptr0 = VL[0];
1625 using DistOrdPair = std::pair<int64_t, int>;
1627 std::set<DistOrdPair,
decltype(Compare)> Offsets(Compare);
1628 Offsets.emplace(0, 0);
1629 bool IsConsecutive =
true;
1638 auto [It, IsInserted] = Offsets.emplace(
Offset,
Idx);
1642 IsConsecutive &= std::next(It) == Offsets.end();
1644 SortedIndices.
clear();
1645 if (!IsConsecutive) {
1649 SortedIndices[
Idx] = Off.second;
1663 std::optional<int> Diff =
1666 return Diff && *Diff == 1;
1672 Accesses[MemAccessInfo(Ptr, true)].push_back(AccessIdx);
1673 InstMap.push_back(SI);
1681 Accesses[MemAccessInfo(Ptr, false)].push_back(AccessIdx);
1682 InstMap.push_back(LI);
1710 case ForwardButPreventsForwarding:
1712 case IndirectUnsafe:
1715 case BackwardVectorizable:
1717 case BackwardVectorizableButPreventsForwarding:
1730 case ForwardButPreventsForwarding:
1735 case BackwardVectorizable:
1737 case BackwardVectorizableButPreventsForwarding:
1738 case IndirectUnsafe:
1744bool MemoryDepChecker::couldPreventStoreLoadForward(
uint64_t Distance,
1758 const uint64_t NumItersForStoreLoadThroughMemory = 8 * TypeByteSize;
1760 uint64_t MaxVFWithoutSLForwardIssues = std::min(
1764 for (
uint64_t VF = 2 * TypeByteSize; VF <= MaxVFWithoutSLForwardIssues;
1768 if (Distance % VF && Distance / VF < NumItersForStoreLoadThroughMemory) {
1769 MaxVFWithoutSLForwardIssues = (VF >> 1);
1774 if (MaxVFWithoutSLForwardIssues < 2 * TypeByteSize) {
1776 dbgs() <<
"LAA: Distance " << Distance
1777 <<
" that could cause a store-load forwarding conflict\n");
1781 if (MaxVFWithoutSLForwardIssues < MinDepDistBytes &&
1782 MaxVFWithoutSLForwardIssues !=
1784 MinDepDistBytes = MaxVFWithoutSLForwardIssues;
1806 const SCEV &MaxBTC,
const SCEV &Dist,
1827 const uint64_t ByteStride = MaxStride * TypeByteSize;
1831 const SCEV *CastedDist = &Dist;
1832 const SCEV *CastedProduct = Product;
1839 if (DistTypeSizeBits > ProductTypeSizeBits)
1864 assert(Stride > 1 &&
"The stride must be greater than 1");
1865 assert(TypeByteSize > 0 &&
"The type size in byte must be non-zero");
1866 assert(Distance > 0 &&
"The distance must be non-zero");
1869 if (Distance % TypeByteSize)
1872 uint64_t ScaledDist = Distance / TypeByteSize;
1890 return ScaledDist % Stride;
1898 return any_of(UnderlyingObjects, [&SE, L](
const Value *UO) {
1904 MemoryDepChecker::DepDistanceStrideAndSizeInfo>
1905MemoryDepChecker::getDependenceDistanceStrideAndSize(
1909 &UnderlyingObjects) {
1911 auto &SE = *PSE.
getSE();
1912 auto [APtr, AIsWrite] =
A;
1913 auto [BPtr, BIsWrite] =
B;
1916 if (!AIsWrite && !BIsWrite)
1923 if (APtr->getType()->getPointerAddressSpace() !=
1924 BPtr->getType()->getPointerAddressSpace())
1927 int64_t StrideAPtr =
1928 getPtrStride(PSE, ATy, APtr, InnermostLoop, SymbolicStrides,
true)
1930 int64_t StrideBPtr =
1931 getPtrStride(PSE, BTy, BPtr, InnermostLoop, SymbolicStrides,
true)
1940 if (StrideAPtr < 0) {
1947 LLVM_DEBUG(
dbgs() <<
"LAA: Src Scev: " << *Src <<
"Sink Scev: " << *Sink
1948 <<
"(Induction step: " << StrideAPtr <<
")\n");
1949 LLVM_DEBUG(
dbgs() <<
"LAA: Distance for " << *AInst <<
" to " << *BInst
1950 <<
": " << *Dist <<
"\n");
1966 const auto &[SrcStart, SrcEnd] =
1968 const auto &[SinkStart, SinkEnd] =
1971 if (!isa<SCEVCouldNotCompute>(SrcStart) &&
1972 !isa<SCEVCouldNotCompute>(SrcEnd) &&
1973 !isa<SCEVCouldNotCompute>(SinkStart) &&
1974 !isa<SCEVCouldNotCompute>(SinkEnd)) {
1985 if (!StrideAPtr || !StrideBPtr || (StrideAPtr > 0 && StrideBPtr < 0) ||
1986 (StrideAPtr < 0 && StrideBPtr > 0)) {
1987 LLVM_DEBUG(
dbgs() <<
"Pointer access with non-constant stride\n");
1991 uint64_t TypeByteSize =
DL.getTypeAllocSize(ATy);
1993 DL.getTypeStoreSizeInBits(ATy) ==
DL.getTypeStoreSizeInBits(BTy);
1996 return DepDistanceStrideAndSizeInfo(Dist, std::abs(StrideAPtr),
1997 std::abs(StrideBPtr), TypeByteSize,
1998 AIsWrite, BIsWrite);
2005 &UnderlyingObjects) {
2006 assert(AIdx < BIdx &&
"Must pass arguments in program order");
2010 auto Res = getDependenceDistanceStrideAndSize(
2011 A, InstMap[AIdx],
B, InstMap[BIdx], UnderlyingObjects);
2012 if (std::holds_alternative<Dependence::DepType>(Res))
2013 return std::get<Dependence::DepType>(Res);
2015 auto &[Dist, StrideA, StrideB, TypeByteSize, AIsWrite, BIsWrite] =
2016 std::get<DepDistanceStrideAndSizeInfo>(Res);
2017 bool HasSameSize = TypeByteSize > 0;
2019 std::optional<uint64_t> CommonStride =
2020 StrideA == StrideB ? std::make_optional(StrideA) :
std::nullopt;
2021 if (isa<SCEVCouldNotCompute>(Dist)) {
2024 FoundNonConstantDistanceDependence |= CommonStride.has_value();
2025 LLVM_DEBUG(
dbgs() <<
"LAA: Dependence because of uncomputable distance.\n");
2031 uint64_t MaxStride = std::max(StrideA, StrideB);
2040 *Dist, MaxStride, TypeByteSize))
2047 const APInt &Val =
C->getAPInt();
2052 if (std::abs(Distance) > 0 && CommonStride && *CommonStride > 1 &&
2069 LLVM_DEBUG(
dbgs() <<
"LAA: possibly zero dependence difference but "
2070 "different type sizes\n");
2074 bool IsTrueDataDependence = (AIsWrite && !BIsWrite);
2089 FoundNonConstantDistanceDependence |= CommonStride.has_value();
2093 couldPreventStoreLoadForward(
C->getAPInt().abs().getZExtValue(),
2096 dbgs() <<
"LAA: Forward but may prevent st->ld forwarding\n");
2107 if (MinDistance <= 0) {
2108 FoundNonConstantDistanceDependence |= CommonStride.has_value();
2112 if (!isa<SCEVConstant>(Dist)) {
2121 FoundNonConstantDistanceDependence |= CommonStride.has_value();
2125 LLVM_DEBUG(
dbgs() <<
"LAA: ReadWrite-Write positive dependency with "
2126 "different type sizes\n");
2139 unsigned MinNumIter = std::max(ForcedFactor * ForcedUnroll, 2U);
2172 TypeByteSize * *CommonStride * (MinNumIter - 1) + TypeByteSize;
2173 if (MinDistanceNeeded >
static_cast<uint64_t>(MinDistance)) {
2174 if (!isa<SCEVConstant>(Dist)) {
2181 LLVM_DEBUG(
dbgs() <<
"LAA: Failure because of positive minimum distance "
2182 << MinDistance <<
'\n');
2188 if (MinDistanceNeeded > MinDepDistBytes) {
2190 << MinDistanceNeeded <<
" size in bytes\n");
2211 std::min(
static_cast<uint64_t>(MinDistance), MinDepDistBytes);
2213 bool IsTrueDataDependence = (!AIsWrite && BIsWrite);
2214 uint64_t MinDepDistBytesOld = MinDepDistBytes;
2216 isa<SCEVConstant>(Dist) &&
2217 couldPreventStoreLoadForward(MinDistance, TypeByteSize)) {
2220 assert(MinDepDistBytes == MinDepDistBytesOld &&
2221 "An update to MinDepDistBytes requires an update to "
2222 "MaxSafeVectorWidthInBits");
2223 (void)MinDepDistBytesOld;
2229 uint64_t MaxVF = MinDepDistBytes / (TypeByteSize * *CommonStride);
2230 LLVM_DEBUG(
dbgs() <<
"LAA: Positive min distance " << MinDistance
2231 <<
" with max VF = " << MaxVF <<
'\n');
2233 uint64_t MaxVFInBits = MaxVF * TypeByteSize * 8;
2234 if (!isa<SCEVConstant>(Dist) && MaxVFInBits < MaxTargetVectorWidthInBits) {
2241 MaxSafeVectorWidthInBits = std::min(MaxSafeVectorWidthInBits, MaxVFInBits);
2248 &UnderlyingObjects) {
2250 MinDepDistBytes = -1;
2253 if (Visited.
count(CurAccess))
2269 bool AIIsWrite = AI->getInt();
2273 (AIIsWrite ? AI : std::next(AI));
2276 for (std::vector<unsigned>::iterator I1 = Accesses[*AI].begin(),
2277 I1E = Accesses[*AI].
end(); I1 != I1E; ++I1)
2280 for (std::vector<unsigned>::iterator
2281 I2 = (OI == AI ? std::next(I1) : Accesses[*OI].begin()),
2282 I2E = (OI == AI ? I1E : Accesses[*OI].end());
2284 auto A = std::make_pair(&*AI, *I1);
2285 auto B = std::make_pair(&*OI, *I2);
2292 B.second, UnderlyingObjects);
2299 if (RecordDependences) {
2304 RecordDependences =
false;
2305 Dependences.clear();
2307 <<
"Too many dependences, stopped recording\n");
2319 LLVM_DEBUG(
dbgs() <<
"Total Dependences: " << Dependences.size() <<
"\n");
2326 auto &IndexVector = Accesses.find(Access)->second;
2330 std::back_inserter(Insts),
2331 [&](
unsigned Idx) {
return this->InstMap[
Idx]; });
2340 "ForwardButPreventsForwarding",
2342 "BackwardVectorizable",
2343 "BackwardVectorizableButPreventsForwarding"};
2353bool LoopAccessInfo::canAnalyzeLoop() {
2362 recordAnalysis(
"NotInnerMostLoop") <<
"loop is not the innermost loop";
2369 dbgs() <<
"LAA: loop control flow is not understood by analyzer\n");
2370 recordAnalysis(
"CFGNotUnderstood")
2371 <<
"loop control flow is not understood by analyzer";
2379 if (isa<SCEVCouldNotCompute>(ExitCount)) {
2380 recordAnalysis(
"CantComputeNumberOfIterations")
2381 <<
"could not determine number of loop iterations";
2382 LLVM_DEBUG(
dbgs() <<
"LAA: SCEV could not compute the loop exit count.\n");
2400 unsigned NumReads = 0;
2401 unsigned NumReadWrites = 0;
2403 bool HasComplexMemInst =
false;
2406 HasConvergentOp =
false;
2408 PtrRtChecking->Pointers.
clear();
2409 PtrRtChecking->Need =
false;
2413 const bool EnableMemAccessVersioningOfLoop =
2425 if (
auto *Call = dyn_cast<CallBase>(&
I)) {
2426 if (
Call->isConvergent())
2427 HasConvergentOp =
true;
2432 if (HasComplexMemInst && HasConvergentOp)
2436 if (HasComplexMemInst)
2440 if (
auto *Decl = dyn_cast<NoAliasScopeDeclInst>(&
I))
2441 for (
Metadata *
Op : Decl->getScopeList()->operands())
2442 LoopAliasScopes.
insert(cast<MDNode>(
Op));
2447 auto *
Call = dyn_cast<CallInst>(&
I);
2454 if (
I.mayReadFromMemory()) {
2457 if (Call && !
Call->isNoBuiltin() &&
Call->getCalledFunction() &&
2461 auto *Ld = dyn_cast<LoadInst>(&
I);
2463 recordAnalysis(
"CantVectorizeInstruction", Ld)
2464 <<
"instruction cannot be vectorized";
2465 HasComplexMemInst =
true;
2468 if (!Ld->isSimple() && !IsAnnotatedParallel) {
2469 recordAnalysis(
"NonSimpleLoad", Ld)
2470 <<
"read with atomic ordering or volatile read";
2472 HasComplexMemInst =
true;
2478 if (EnableMemAccessVersioningOfLoop)
2479 collectStridedAccess(Ld);
2484 if (
I.mayWriteToMemory()) {
2485 auto *St = dyn_cast<StoreInst>(&
I);
2487 recordAnalysis(
"CantVectorizeInstruction", St)
2488 <<
"instruction cannot be vectorized";
2489 HasComplexMemInst =
true;
2492 if (!St->isSimple() && !IsAnnotatedParallel) {
2493 recordAnalysis(
"NonSimpleStore", St)
2494 <<
"write with atomic ordering or volatile write";
2496 HasComplexMemInst =
true;
2502 if (EnableMemAccessVersioningOfLoop)
2503 collectStridedAccess(St);
2508 if (HasComplexMemInst)
2516 if (!Stores.
size()) {
2522 AccessAnalysis Accesses(TheLoop, AA, LI, DependentAccesses, *PSE,
2539 if (isInvariant(
Ptr)) {
2541 StoresToInvariantAddresses.push_back(ST);
2542 HasStoreStoreDependenceInvolvingLoopInvariantAddress |=
2549 if (Seen.
insert({Ptr, AccessTy}).second) {
2556 if (blockNeedsPredication(
ST->getParent(), TheLoop, DT))
2560 [&Accesses, AccessTy, Loc](
Value *
Ptr) {
2561 MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr);
2562 Accesses.addStore(NewLoc, AccessTy);
2567 if (IsAnnotatedParallel) {
2569 dbgs() <<
"LAA: A loop annotated parallel, ignore memory dependency "
2584 bool IsReadOnlyPtr =
false;
2586 if (Seen.
insert({Ptr, AccessTy}).second ||
2587 !
getPtrStride(*PSE,
LD->getType(),
Ptr, TheLoop, SymbolicStrides).value_or(0)) {
2589 IsReadOnlyPtr =
true;
2595 LLVM_DEBUG(
dbgs() <<
"LAA: Found an unsafe dependency between a uniform "
2596 "load and uniform store to the same address!\n");
2597 HasLoadStoreDependenceInvolvingLoopInvariantAddress =
true;
2604 if (blockNeedsPredication(
LD->getParent(), TheLoop, DT))
2608 [&Accesses, AccessTy, Loc, IsReadOnlyPtr](
Value *
Ptr) {
2609 MemoryLocation NewLoc = Loc.getWithNewPtr(Ptr);
2610 Accesses.addLoad(NewLoc, AccessTy, IsReadOnlyPtr);
2616 if (NumReadWrites == 1 && NumReads == 0) {
2623 Accesses.buildDependenceSets();
2627 Value *UncomputablePtr =
nullptr;
2628 bool CanDoRTIfNeeded =
2629 Accesses.canCheckPtrAtRT(*PtrRtChecking, PSE->
getSE(), TheLoop,
2630 SymbolicStrides, UncomputablePtr,
false);
2631 if (!CanDoRTIfNeeded) {
2632 auto *
I = dyn_cast_or_null<Instruction>(UncomputablePtr);
2633 recordAnalysis(
"CantIdentifyArrayBounds",
I)
2634 <<
"cannot identify array bounds";
2635 LLVM_DEBUG(
dbgs() <<
"LAA: We can't vectorize because we can't find "
2636 <<
"the array bounds.\n");
2641 dbgs() <<
"LAA: May be able to perform a memory runtime check if needed.\n");
2643 bool DepsAreSafe =
true;
2644 if (Accesses.isDependencyCheckNeeded()) {
2646 DepsAreSafe = DepChecker->
areDepsSafe(DependentAccesses,
2647 Accesses.getDependenciesToCheck(),
2648 Accesses.getUnderlyingObjects());
2654 Accesses.resetDepChecks(*DepChecker);
2656 PtrRtChecking->reset();
2657 PtrRtChecking->Need =
true;
2659 auto *SE = PSE->
getSE();
2660 UncomputablePtr =
nullptr;
2661 CanDoRTIfNeeded = Accesses.canCheckPtrAtRT(
2662 *PtrRtChecking, SE, TheLoop, SymbolicStrides, UncomputablePtr,
true);
2665 if (!CanDoRTIfNeeded) {
2666 auto *
I = dyn_cast_or_null<Instruction>(UncomputablePtr);
2667 recordAnalysis(
"CantCheckMemDepsAtRunTime",
I)
2668 <<
"cannot check memory dependencies at runtime";
2669 LLVM_DEBUG(
dbgs() <<
"LAA: Can't vectorize with memory checks\n");
2676 if (HasConvergentOp) {
2677 recordAnalysis(
"CantInsertRuntimeCheckWithConvergent")
2678 <<
"cannot add control dependency to convergent operation";
2679 LLVM_DEBUG(
dbgs() <<
"LAA: We can't vectorize because a runtime check "
2680 "would be needed with a convergent operation\n");
2686 dbgs() <<
"LAA: No unsafe dependent memory operations in loop. We"
2687 << (PtrRtChecking->Need ?
"" :
" don't")
2688 <<
" need runtime memory checks.\n");
2692 emitUnsafeDependenceRemark();
2696void LoopAccessInfo::emitUnsafeDependenceRemark() {
2697 const auto *Deps = getDepChecker().getDependences();
2705 if (Found == Deps->end())
2709 LLVM_DEBUG(
dbgs() <<
"LAA: unsafe dependent memory operations in loop\n");
2712 bool HasForcedDistribution =
false;
2713 std::optional<const MDOperand *>
Value =
2717 assert(
Op && mdconst::hasa<ConstantInt>(*
Op) &&
"invalid metadata");
2718 HasForcedDistribution = mdconst::extract<ConstantInt>(*Op)->getZExtValue();
2721 const std::string
Info =
2722 HasForcedDistribution
2723 ?
"unsafe dependent memory operations in loop."
2724 :
"unsafe dependent memory operations in loop. Use "
2725 "#pragma clang loop distribute(enable) to allow loop distribution "
2726 "to attempt to isolate the offending operations into a separate "
2737 R <<
"\nBackward loop carried data dependence.";
2740 R <<
"\nForward loop carried data dependence that prevents "
2741 "store-to-load forwarding.";
2744 R <<
"\nBackward loop carried data dependence that prevents "
2745 "store-to-load forwarding.";
2748 R <<
"\nUnsafe indirect dependence.";
2751 R <<
"\nUnknown data dependence.";
2758 SourceLoc = DD->getDebugLoc();
2760 R <<
" Memory location is the same as accessed at "
2761 <<
ore::NV(
"Location", SourceLoc);
2776 assert(!Report &&
"Multiple reports generated");
2782 CodeRegion =
I->getParent();
2785 if (
I->getDebugLoc())
2786 DL =
I->getDebugLoc();
2789 Report = std::make_unique<OptimizationRemarkAnalysis>(
DEBUG_TYPE, RemarkName,
DL,
2795 auto *SE = PSE->
getSE();
2816 std::advance(GEPTI, LastOperand - 2);
2823 if (ElemSize != GEPAllocSize)
2843 for (
unsigned I = 0, E =
GEP->getNumOperands();
I != E; ++
I)
2844 if (
I != InductionOperand &&
2847 return GEP->getOperand(InductionOperand);
2853 auto *PtrTy = dyn_cast<PointerType>(
Ptr->getType());
2854 if (!PtrTy || PtrTy->isAggregateType())
2863 int64_t PtrAccessSize = 1;
2871 V =
C->getOperand();
2888 if (OrigPtr ==
Ptr) {
2889 if (
const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(V)) {
2890 if (M->getOperand(0)->getSCEVType() !=
scConstant)
2893 const APInt &APStepVal = cast<SCEVConstant>(M->getOperand(0))->getAPInt();
2900 if (PtrAccessSize != StepVal)
2902 V = M->getOperand(1);
2912 if (isa<SCEVUnknown>(V))
2915 if (
const auto *
C = dyn_cast<SCEVIntegralCastExpr>(V))
2916 if (isa<SCEVUnknown>(
C->getOperand()))
2922void LoopAccessInfo::collectStridedAccess(
Value *MemAccess) {
2937 LLVM_DEBUG(
dbgs() <<
"LAA: Found a strided access that is a candidate for "
2942 LLVM_DEBUG(
dbgs() <<
" Chose not to due to -laa-speculate-unit-stride\n");
2967 const SCEV *CastedStride = StrideExpr;
2968 const SCEV *CastedBECount = MaxBTC;
2970 if (BETypeSizeBits >= StrideTypeSizeBits)
2974 const SCEV *StrideMinusBETaken = SE->
getMinusSCEV(CastedStride, CastedBECount);
2980 dbgs() <<
"LAA: Stride>=TripCount; No point in versioning as the "
2981 "Stride==1 predicate will imply that the loop executes "
2985 LLVM_DEBUG(
dbgs() <<
"LAA: Found a strided access that we can version.\n");
2989 const SCEV *StrideBase = StrideExpr;
2990 if (
const auto *
C = dyn_cast<SCEVIntegralCastExpr>(StrideBase))
2991 StrideBase =
C->getOperand();
2992 SymbolicStrides[
Ptr] = cast<SCEVUnknown>(StrideBase);
3000 PtrRtChecking(nullptr), TheLoop(L) {
3001 unsigned MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
3008 MaxTargetVectorWidthInBits = FixedWidth.
getFixedValue() * 2;
3014 MaxTargetVectorWidthInBits = std::numeric_limits<unsigned>::max();
3016 DepChecker = std::make_unique<MemoryDepChecker>(*PSE, L, SymbolicStrides,
3017 MaxTargetVectorWidthInBits);
3018 PtrRtChecking = std::make_unique<RuntimePointerChecking>(*DepChecker, SE);
3019 if (canAnalyzeLoop())
3020 CanVecMem = analyzeLoop(AA, LI, TLI, DT);
3028 OS <<
" with a maximum safe vector width of "
3030 if (PtrRtChecking->Need)
3031 OS <<
" with run-time checks";
3035 if (HasConvergentOp)
3043 for (
const auto &Dep : *Dependences) {
3051 PtrRtChecking->print(
OS,
Depth);
3055 <<
"Non vectorizable stores to invariant address were "
3056 << (HasStoreStoreDependenceInvolvingLoopInvariantAddress ||
3057 HasLoadStoreDependenceInvolvingLoopInvariantAddress
3060 <<
"found in loop.\n";
3072 auto [It, Inserted] = LoopAccessInfoMap.insert({&L,
nullptr});
3076 std::make_unique<LoopAccessInfo>(&L, &SE,
TTI, TLI, &AA, &DT, &LI);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
Generic implementation of equivalence classes through the use Tarjan's efficient union-find algorithm...
static cl::opt< unsigned > MaxDependences("max-dependences", cl::Hidden, cl::desc("Maximum number of dependences collected by " "loop-access analysis (default = 100)"), cl::init(100))
We collect dependences up to this threshold.
static cl::opt< bool > EnableForwardingConflictDetection("store-to-load-forwarding-conflict-detection", cl::Hidden, cl::desc("Enable conflict detection in loop-access analysis"), cl::init(true))
Enable store-to-load forwarding conflict detection.
static void findForkedSCEVs(ScalarEvolution *SE, const Loop *L, Value *Ptr, SmallVectorImpl< PointerIntPair< const SCEV *, 1, bool > > &ScevList, unsigned Depth)
static bool hasComputableBounds(PredicatedScalarEvolution &PSE, Value *Ptr, const SCEV *PtrScev, Loop *L, bool Assume)
Check whether a pointer can participate in a runtime bounds check.
static cl::opt< unsigned > MemoryCheckMergeThreshold("memory-check-merge-threshold", cl::Hidden, cl::desc("Maximum number of comparisons done when trying to merge " "runtime memory checks. (default = 100)"), cl::init(100))
The maximum iterations used to merge memory checks.
static bool isNoWrap(PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &Strides, Value *Ptr, Type *AccessTy, Loop *L)
Check whether a pointer address cannot wrap.
static const SCEV * getStrideFromPointer(Value *Ptr, ScalarEvolution *SE, Loop *Lp)
Get the stride of a pointer access in a loop.
static unsigned getGEPInductionOperand(const GetElementPtrInst *Gep)
Find the operand of the GEP that should be checked for consecutive stores.
static cl::opt< unsigned, true > VectorizationInterleave("force-vector-interleave", cl::Hidden, cl::desc("Sets the vectorization interleave count. " "Zero is autoselect."), cl::location(VectorizerParams::VectorizationInterleave))
static bool isLoopVariantIndirectAddress(ArrayRef< const Value * > UnderlyingObjects, ScalarEvolution &SE, const Loop *L)
Returns true if any of the underlying objects has a loop varying address, i.e.
static cl::opt< bool, true > HoistRuntimeChecks("hoist-runtime-checks", cl::Hidden, cl::desc("Hoist inner loop runtime memory checks to outer loop if possible"), cl::location(VectorizerParams::HoistRuntimeChecks), cl::init(true))
static cl::opt< unsigned, true > VectorizationFactor("force-vector-width", cl::Hidden, cl::desc("Sets the SIMD width. Zero is autoselect."), cl::location(VectorizerParams::VectorizationFactor))
static bool isSafeDependenceDistance(const DataLayout &DL, ScalarEvolution &SE, const SCEV &MaxBTC, const SCEV &Dist, uint64_t MaxStride, uint64_t TypeByteSize)
Given a dependence-distance Dist between two memory accesses, that have strides in the same direction...
static cl::opt< unsigned, true > RuntimeMemoryCheckThreshold("runtime-memory-check-threshold", cl::Hidden, cl::desc("When performing memory disambiguation checks at runtime do not " "generate more than this number of comparisons (default = 8)."), cl::location(VectorizerParams::RuntimeMemoryCheckThreshold), cl::init(8))
static void visitPointers(Value *StartPtr, const Loop &InnermostLoop, function_ref< void(Value *)> AddPointer)
static bool isNoWrapAddRec(Value *Ptr, const SCEVAddRecExpr *AR, PredicatedScalarEvolution &PSE, const Loop *L)
Return true if an AddRec pointer Ptr is unsigned non-wrapping, i.e.
static std::pair< const SCEV *, const SCEV * > getStartAndEndForAccess(const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy, PredicatedScalarEvolution &PSE)
Calculate Start and End points of memory access.
static Value * stripGetElementPtr(Value *Ptr, ScalarEvolution *SE, Loop *Lp)
If the argument is a GEP, then returns the operand identified by getGEPInductionOperand.
static bool areStridedAccessesIndependent(uint64_t Distance, uint64_t Stride, uint64_t TypeByteSize)
Check the dependence for two accesses with the same stride Stride.
static const SCEV * getMinFromExprs(const SCEV *I, const SCEV *J, ScalarEvolution *SE)
Compare I and J and return the minimum.
static cl::opt< unsigned > MaxForkedSCEVDepth("max-forked-scev-depth", cl::Hidden, cl::desc("Maximum recursion depth when finding forked SCEVs (default = 5)"), cl::init(5))
static cl::opt< bool > SpeculateUnitStride("laa-speculate-unit-stride", cl::Hidden, cl::desc("Speculate that non-constant strides are unit in LAA"), cl::init(true))
static SmallVector< PointerIntPair< const SCEV *, 1, bool > > findForkedPointer(PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &StridesMap, Value *Ptr, const Loop *L)
static cl::opt< bool > EnableMemAccessVersioning("enable-mem-access-versioning", cl::init(true), cl::Hidden, cl::desc("Enable symbolic stride memory access versioning"))
This enables versioning on the strides of symbolically striding memory accesses in code like the foll...
This header provides classes for managing per-loop analyses.
This file provides utility analysis objects describing memory locations.
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
This file defines the PointerIntPair class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static LLVM_ATTRIBUTE_ALWAYS_INLINE bool CheckType(MVT::SimpleValueType VT, SDValue N, const TargetLowering *TLI, const DataLayout &DL)
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
static const X86InstrFMA3Group Groups[]
A manager for alias analyses.
Class for arbitrary precision integers.
unsigned getBitWidth() const
Return the number of bits in the APInt.
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
int64_t getSExtValue() const
Get sign extended value.
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
API to communicate dependencies between analyses during invalidation.
bool invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Trigger the invalidation of some other analysis pass if not already handled and return whether it was...
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
const DataLayout & getDataLayout() const
Get the data layout of the module this basic block belongs to.
This class is a wrapper over an AAResults, and it is intended to be used only when there are no IR ch...
@ ICMP_ULE
unsigned less or equal
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
iterator findValue(const ElemTy &V) const
findValue - Return an iterator to the specified value.
iterator insert(const ElemTy &Data)
insert - Insert a new value into the union/find set, ignoring the request if the value already exists...
member_iterator member_end() const
typename std::set< ECValue, ECValueComparator >::const_iterator iterator
iterator* - Provides a way to iterate over all values in the set.
member_iterator member_begin(iterator I) const
member_iterator unionSets(const ElemTy &V1, const ElemTy &V2)
union - Merge the two equivalence sets for the specified values, inserting them if they do not alread...
const ElemTy & getLeaderValue(const ElemTy &V) const
getLeaderValue - Return the leader for the specified value that is in the set.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Type * getResultElementType() const
PointerType * getType() const
Global values are always pointers.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
An instruction for reading from memory.
Value * getPointerOperand()
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
This analysis provides dependence information for the memory accesses of a loop.
Result run(Function &F, FunctionAnalysisManager &AM)
bool invalidate(Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &Inv)
const LoopAccessInfo & getInfo(Loop &L)
Drive the analysis of memory accesses in the loop.
const MemoryDepChecker & getDepChecker() const
the Memory Dependence Checker which can determine the loop-independent and loop-carried dependences b...
bool isInvariant(Value *V) const
Returns true if value V is loop invariant.
void print(raw_ostream &OS, unsigned Depth=0) const
Print the information about the memory accesses in the loop.
static bool blockNeedsPredication(BasicBlock *BB, Loop *TheLoop, DominatorTree *DT)
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetTransformInfo *TTI, const TargetLibraryInfo *TLI, AAResults *AA, DominatorTree *DT, LoopInfo *LI)
Analysis pass that exposes the LoopInfo for a function.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
unsigned getNumBackEdges() const
Calculate the number of back edges to the loop header.
BlockT * getHeader() const
LoopT * getParentLoop() const
Return the parent loop if it exists or nullptr for top level loops.
Wrapper class to LoopBlocksDFS that provides a standard begin()/end() interface for the DFS reverse p...
Represents a single loop in the control flow graph.
std::string getLocStr() const
Return a string containing the debug location of the loop (file name + line number if present,...
bool isAnnotatedParallel() const
Returns true if the loop is annotated parallel.
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
ArrayRef< MDOperand > operands() const
Tracking metadata reference owned by Metadata.
This class implements a map that also provides access to all stored values in a deterministic order.
Checks memory dependences among accesses to the same underlying object to determine whether there vec...
ArrayRef< unsigned > getOrderForAccess(Value *Ptr, bool IsWrite) const
Return the program order indices for the access location (Ptr, IsWrite).
bool areDepsSafe(DepCandidates &AccessSets, MemAccessInfoList &CheckDeps, const DenseMap< Value *, SmallVector< const Value *, 16 > > &UnderlyingObjects)
Check whether the dependencies between the accesses are safe.
bool isSafeForAnyVectorWidth() const
Return true if the number of elements that are safe to operate on simultaneously is not bounded.
const SmallVectorImpl< Instruction * > & getMemoryInstructions() const
The vector of memory access instructions.
const Loop * getInnermostLoop() const
uint64_t getMaxSafeVectorWidthInBits() const
Return the number of elements that are safe to operate on simultaneously, multiplied by the size of t...
bool isSafeForVectorization() const
No memory dependence was encountered that would inhibit vectorization.
const SmallVectorImpl< Dependence > * getDependences() const
Returns the memory dependences.
SmallVector< Instruction *, 4 > getInstructionsForAccess(Value *Ptr, bool isWrite) const
Find the set of instructions that read or write via Ptr.
VectorizationSafetyStatus
Type to keep track of the status of the dependence check.
@ PossiblySafeWithRtChecks
bool shouldRetryWithRuntimeCheck() const
In same cases when the dependency check fails we can still vectorize the loop with a dynamic array ac...
void addAccess(StoreInst *SI)
Register the location (instructions are given increasing numbers) of a write access.
PointerIntPair< Value *, 1, bool > MemAccessInfo
Representation for a specific memory location.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
LocationSize Size
The maximum size of the location, in address-units, or UnknownSize if the size is not known.
AAMDNodes AATags
The metadata nodes which describes the aliasing of the location (each member is null if that kind of ...
const Value * Ptr
The address of the start of the location.
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
void addPredicate(const SCEVPredicate &Pred)
Adds a new predicate.
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
const SCEVPredicate & getPredicate() const
bool hasNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Returns true if we've proved that V doesn't wrap by means of a SCEV predicate.
void setNoOverflow(Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags)
Proves that V doesn't overflow by adding SCEV predicate.
void print(raw_ostream &OS, unsigned Depth) const
Print the SCEV mappings done by the Predicated Scalar Evolution.
const SCEVAddRecExpr * getAsAddRec(Value *V)
Attempts to produce an AddRecExpr for V by adding additional SCEV predicates.
const SCEV * getSymbolicMaxBackedgeTakenCount()
Get the (predicated) symbolic max backedge count for the analyzed loop.
const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
A set of analyses that are preserved following a run of a transformation pass.
PreservedAnalysisChecker getChecker() const
Build a checker for this PreservedAnalyses and the specified analysis type.
Holds information about the memory runtime legality checks to verify that a group of pointers do not ...
bool Need
This flag indicates if we need to add the runtime check.
void reset()
Reset the state of the pointer runtime information.
unsigned getNumberOfChecks() const
Returns the number of run-time checks required according to needsChecking.
void printChecks(raw_ostream &OS, const SmallVectorImpl< RuntimePointerCheck > &Checks, unsigned Depth=0) const
Print Checks.
bool needsChecking(const RuntimeCheckingPtrGroup &M, const RuntimeCheckingPtrGroup &N) const
Decide if we need to add a check between two groups of pointers, according to needsChecking.
void print(raw_ostream &OS, unsigned Depth=0) const
Print the list run-time memory checks necessary.
SmallVector< RuntimeCheckingPtrGroup, 2 > CheckingGroups
Holds a partitioning of pointers into "check groups".
void generateChecks(MemoryDepChecker::DepCandidates &DepCands, bool UseDependencies)
Generate the checks and store it.
friend struct RuntimeCheckingPtrGroup
static bool arePointersInSamePartition(const SmallVectorImpl< int > &PtrToPartition, unsigned PtrIdx1, unsigned PtrIdx2)
Check if pointers are in the same partition.
SmallVector< PointerInfo, 2 > Pointers
Information about the pointers that may require checking.
void insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr, Type *AccessTy, bool WritePtr, unsigned DepSetId, unsigned ASId, PredicatedScalarEvolution &PSE, bool NeedsFreeze)
Insert a pointer and calculate the start and end SCEVs.
This node represents a polynomial recurrence on the trip count of the specified loop.
const SCEV * getStepRecurrence(ScalarEvolution &SE) const
Constructs and returns the recurrence indicating how much this expression steps by.
bool isAffine() const
Return true if this represents an expression A + B*x where A and B are loop invariant values.
const Loop * getLoop() const
This class represents a constant integer value.
This is the base class for unary integral cast operator classes.
This node represents multiplication of some number of SCEVs.
NoWrapFlags getNoWrapFlags(NoWrapFlags Mask=NoWrapMask) const
virtual void print(raw_ostream &OS, unsigned Depth=0) const =0
Prints a textual representation of this predicate with an indentation of Depth.
This class represents an analyzed expression in the program.
Type * getType() const
Return the LLVM type of this SCEV expression.
Analysis pass that exposes the ScalarEvolution for a function.
The main scalar evolution driver.
bool isKnownNonNegative(const SCEV *S)
Test if the given expression is known to be non-negative.
const SCEV * getNegativeSCEV(const SCEV *V, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Return the SCEV object corresponding to -V.
bool isKnownNonPositive(const SCEV *S)
Test if the given expression is known to be non-positive.
const SCEV * getUMaxExpr(const SCEV *LHS, const SCEV *RHS)
const SCEVPredicate * getEqualPredicate(const SCEV *LHS, const SCEV *RHS)
const SCEV * getConstant(ConstantInt *V)
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
const SCEV * getNoopOrSignExtend(const SCEV *V, Type *Ty)
Return a SCEV corresponding to a conversion of the input value to the specified type.
const SCEV * getOne(Type *Ty)
Return a SCEV for the constant 1 of a specific type.
const SCEV * getPtrToIntExpr(const SCEV *Op, Type *Ty)
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
bool isKnownPositive(const SCEV *S)
Test if the given expression is known to be positive.
bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
const SCEV * getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth=0)
bool isSCEVable(Type *Ty) const
Test if values of the given type are analyzable within the SCEV framework.
Type * getEffectiveSCEVType(Type *Ty) const
Return a type with the same bitwidth as the given type and which represents how SCEV will treat the g...
const SCEV * getUMinExpr(const SCEV *LHS, const SCEV *RHS, bool Sequential=false)
APInt getSignedRangeMin(const SCEV *S)
Determine the min of the signed range for a particular SCEV.
const SCEV * getStoreSizeOfExpr(Type *IntTy, Type *StoreTy)
Return an expression for the store size of StoreTy that is type IntTy.
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Return LHS-RHS.
const SCEV * getCouldNotCompute()
const SCEV * applyLoopGuards(const SCEV *Expr, const Loop *L)
Try to apply information from loop guards for L to Expr.
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical multiply expression, or something simpler if possible.
const SCEV * getSizeOfExpr(Type *IntTy, TypeSize Size)
Return an expression for a TypeSize.
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Get a canonical add expression, or something simpler if possible.
const SCEV * getTruncateOrSignExtend(const SCEV *V, Type *Ty, unsigned Depth=0)
Return a SCEV corresponding to a conversion of the input value to the specified type.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
const Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset) const
This is a wrapper around stripAndAccumulateConstantOffsets with the in-bounds requirement set to fals...
StringRef getName() const
Return a constant reference to the value's name.
constexpr ScalarTy getFixedValue() const
constexpr bool isNonZero() const
An efficient, type-erasing, non-owning reference to a callable.
TypeSize getSequentialElementStride(const DataLayout &DL) const
Type * getIndexedType() const
This class implements an extremely fast bulk output stream that can only output to a stream.
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
friend const_iterator end(StringRef path)
Get end iterator over path.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
bool match(Val *V, const Pattern &P)
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
initializer< Ty > init(const Ty &Val)
LocationClass< Ty > location(Ty &L)
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
std::optional< int > getPointersDiff(Type *ElemTyA, Value *PtrA, Type *ElemTyB, Value *PtrB, const DataLayout &DL, ScalarEvolution &SE, bool StrictCheck=false, bool CheckType=true)
Returns the distance between the pointers PtrA and PtrB iff they are compatible and it is possible to...
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
unsigned getPointerAddressSpace(const Type *T)
std::optional< const MDOperand * > findStringMetadataForLoop(const Loop *TheLoop, StringRef Name)
Find string metadata for loop.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
std::optional< int64_t > getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp, const DenseMap< Value *, const SCEV * > &StridesMap=DenseMap< Value *, const SCEV * >(), bool Assume=false, bool ShouldCheckWrap=true)
If the pointer has a constant stride return it in units of the access type size.
bool sortPtrAccesses(ArrayRef< Value * > VL, Type *ElemTy, const DataLayout &DL, ScalarEvolution &SE, SmallVectorImpl< unsigned > &SortedIndices)
Attempt to sort the pointers in VL and return the sorted indices in SortedIndices,...
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
const SCEV * replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &PtrToStride, Value *Ptr)
Return the SCEV corresponding to a pointer with the symbolic stride replaced with constant one,...
bool isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL, ScalarEvolution &SE, bool CheckType=true)
Returns true if the memory operations A and B are consecutive.
bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
OutputIt copy(R &&Range, OutputIt Out)
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
gep_type_iterator gep_type_begin(const User *GEP)
Type * getLoadStoreType(Value *I)
A helper function that returns the type of a load or store instruction.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
MDNode * Scope
The tag for alias scope specification (used with noalias).
MDNode * TBAA
The tag for type-based alias analysis.
MDNode * NoAlias
The tag specifying the noalias scope.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Dependece between memory access instructions.
Instruction * getDestination(const MemoryDepChecker &DepChecker) const
Return the destination instruction of the dependence.
DepType Type
The type of the dependence.
bool isPossiblyBackward() const
May be a lexically backward dependence type (includes Unknown).
Instruction * getSource(const MemoryDepChecker &DepChecker) const
Return the source instruction of the dependence.
bool isForward() const
Lexically forward dependence.
bool isBackward() const
Lexically backward dependence.
void print(raw_ostream &OS, unsigned Depth, const SmallVectorImpl< Instruction * > &Instrs) const
Print the dependence.
DepType
The type of the dependence.
@ BackwardVectorizableButPreventsForwarding
@ ForwardButPreventsForwarding
static const char * DepName[]
String version of the types.
static VectorizationSafetyStatus isSafeForVectorization(DepType Type)
Dependence types that don't prevent vectorization.
unsigned AddressSpace
Address space of the involved pointers.
bool addPointer(unsigned Index, RuntimePointerChecking &RtCheck)
Tries to add the pointer recorded in RtCheck at index Index to this pointer checking group.
bool NeedsFreeze
Whether the pointer needs to be frozen after expansion, e.g.
const SCEV * High
The SCEV expression which represents the upper bound of all the pointers in this group.
SmallVector< unsigned, 2 > Members
Indices of all the pointers that constitute this grouping.
RuntimeCheckingPtrGroup(unsigned Index, RuntimePointerChecking &RtCheck)
Create a new pointer checking group containing a single pointer, with index Index in RtCheck.
const SCEV * Low
The SCEV expression which represents the lower bound of all the pointers in this group.
bool IsWritePtr
Holds the information if this pointer is used for writing to memory.
unsigned DependencySetId
Holds the id of the set of pointers that could be dependent because of a shared underlying object.
unsigned AliasSetId
Holds the id of the disjoint alias set to which this pointer belongs.
static const unsigned MaxVectorWidth
Maximum SIMD width.
static unsigned VectorizationFactor
VF as overridden by the user.
static unsigned RuntimeMemoryCheckThreshold
\When performing memory disambiguation checks at runtime do not make more than this number of compari...
static bool isInterleaveForced()
True if force-vector-interleave was specified by the user.
static unsigned VectorizationInterleave
Interleave factor as overridden by the user.
static bool HoistRuntimeChecks
Function object to check whether the first component of a container supported by std::get (like std::...