58#include "llvm/IR/IntrinsicsAMDGPU.h"
59#include "llvm/IR/IntrinsicsNVPTX.h"
85#define DEBUG_TYPE "attributor"
87STATISTIC(NumAAs,
"Number of abstract attributes created");
88STATISTIC(NumIndirectCallsPromoted,
"Number of indirect calls promoted");
103#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
104 ("Number of " #TYPE " marked '" #NAME "'")
105#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
106#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
107#define STATS_DECL(NAME, TYPE, MSG) \
108 STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
109#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
110#define STATS_DECLTRACK(NAME, TYPE, MSG) \
111 {STATS_DECL(NAME, TYPE, MSG) STATS_TRACK(NAME, TYPE)}
112#define STATS_DECLTRACK_ARG_ATTR(NAME) \
113 STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
114#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
115 STATS_DECLTRACK(NAME, CSArguments, \
116 BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
117#define STATS_DECLTRACK_FN_ATTR(NAME) \
118 STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
119#define STATS_DECLTRACK_CS_ATTR(NAME) \
120 STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
121#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
122 STATS_DECLTRACK(NAME, FunctionReturn, \
123 BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
124#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
125 STATS_DECLTRACK(NAME, CSReturn, \
126 BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
127#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
128 STATS_DECLTRACK(NAME, Floating, \
129 ("Number of floating values known to be '" #NAME "'"))
134#define PIPE_OPERATOR(CLASS) \
135 raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
136 return OS << static_cast<const AbstractAttribute &>(AA); \
193 bool HeaderOnly,
CycleRef *CPtr =
nullptr) {
196 auto *BB =
I->getParent();
213 if (
DL.getTypeSizeInBits(Ty) !=
DL.getTypeAllocSizeInBits(Ty))
238 StartPos +=
DL.getTypeAllocSizeInBits(ElTy);
249 bool AllowVolatile) {
250 if (!AllowVolatile &&
I->isVolatile())
254 return LI->getPointerOperand();
258 return SI->getPointerOperand();
262 return CXI->getPointerOperand();
266 return RMWI->getPointerOperand();
288 bool GetMinOffset,
bool AllowNonInbounds,
289 bool UseAssumed =
false) {
291 auto AttributorAnalysis = [&](
Value &
V,
APInt &ROffset) ->
bool {
298 if (!ValueConstantRangeAA)
302 if (
Range.isFullSet())
308 ROffset =
Range.getSignedMin();
310 ROffset =
Range.getSignedMax();
321 const Value *Ptr, int64_t &BytesOffset,
326 true, AllowNonInbounds);
334template <
typename AAType,
typename StateType =
typename AAType::StateType,
336 bool RecurseForSelectAndPHI =
true>
338 Attributor &
A,
const AAType &QueryingAA, StateType &S,
340 LLVM_DEBUG(
dbgs() <<
"[Attributor] Clamp return value states for "
341 << QueryingAA <<
" into " << S <<
"\n");
343 assert((QueryingAA.getIRPosition().getPositionKind() ==
345 QueryingAA.getIRPosition().getPositionKind() ==
347 "Can only clamp returned value states for a function returned or call "
348 "site returned position!");
352 std::optional<StateType>
T;
355 auto CheckReturnValue = [&](
Value &RV) ->
bool {
369 <<
" AA: " <<
AA->getAsStr(&
A) <<
" @ " << RVPos <<
"\n");
370 const StateType &AAS =
AA->getState();
372 T = StateType::getBestState(AAS);
374 LLVM_DEBUG(
dbgs() <<
"[Attributor] AA State: " << AAS <<
" RV State: " <<
T
376 return T->isValidState();
379 if (!
A.checkForAllReturnedValues(CheckReturnValue, QueryingAA,
381 RecurseForSelectAndPHI))
382 S.indicatePessimisticFixpoint();
389template <
typename AAType,
typename BaseType,
390 typename StateType =
typename BaseType::StateType,
391 bool PropagateCallBaseContext =
false,
393 bool RecurseForSelectAndPHI =
true>
394struct AAReturnedFromReturnedValues :
public BaseType {
395 AAReturnedFromReturnedValues(
const IRPosition &IRP, Attributor &
A)
399 StateType S(StateType::getBestState(this->getState()));
401 RecurseForSelectAndPHI>(
403 PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
412template <
typename AAType,
typename StateType =
typename AAType::StateType,
414static void clampCallSiteArgumentStates(
Attributor &
A,
const AAType &QueryingAA,
416 LLVM_DEBUG(
dbgs() <<
"[Attributor] Clamp call site argument states for "
417 << QueryingAA <<
" into " << S <<
"\n");
419 assert(QueryingAA.getIRPosition().getPositionKind() ==
421 "Can only clamp call site argument states for an argument position!");
425 std::optional<StateType>
T;
428 unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
448 LLVM_DEBUG(
dbgs() <<
"[Attributor] ACS: " << *ACS.getInstruction()
449 <<
" AA: " <<
AA->getAsStr(&
A) <<
" @" << ACSArgPos
451 const StateType &AAS =
AA->getState();
453 T = StateType::getBestState(AAS);
455 LLVM_DEBUG(
dbgs() <<
"[Attributor] AA State: " << AAS <<
" CSA State: " <<
T
457 return T->isValidState();
460 bool UsedAssumedInformation =
false;
461 if (!
A.checkForAllCallSites(CallSiteCheck, QueryingAA,
true,
462 UsedAssumedInformation))
463 S.indicatePessimisticFixpoint();
470template <
typename AAType,
typename BaseType,
471 typename StateType =
typename AAType::StateType,
473bool getArgumentStateFromCallBaseContext(
Attributor &
A,
477 "Expected an 'argument' position !");
483 assert(ArgNo >= 0 &&
"Invalid Arg No!");
497 const StateType &CBArgumentState =
498 static_cast<const StateType &
>(
AA->getState());
500 LLVM_DEBUG(
dbgs() <<
"[Attributor] Briding Call site context to argument"
501 <<
"Position:" << Pos <<
"CB Arg state:" << CBArgumentState
505 State ^= CBArgumentState;
510template <
typename AAType,
typename BaseType,
511 typename StateType =
typename AAType::StateType,
512 bool BridgeCallBaseContext =
false,
514struct AAArgumentFromCallSiteArguments :
public BaseType {
515 AAArgumentFromCallSiteArguments(
const IRPosition &IRP, Attributor &
A)
519 StateType S = StateType::getBestState(this->getState());
521 if (BridgeCallBaseContext) {
523 getArgumentStateFromCallBaseContext<AAType,
BaseType, StateType,
525 A, *
this, this->getIRPosition(), S);
529 clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(
A, *
this,
539template <
typename AAType,
typename BaseType,
540 typename StateType =
typename BaseType::StateType,
541 bool IntroduceCallBaseContext =
false,
543struct AACalleeToCallSite :
public BaseType {
544 AACalleeToCallSite(
const IRPosition &IRP, Attributor &
A) :
BaseType(IRP,
A) {}
547 auto IRPKind = this->getIRPosition().getPositionKind();
550 "Can only wrap function returned positions for call site "
551 "returned positions!");
552 auto &S = this->getState();
555 if (IntroduceCallBaseContext)
556 LLVM_DEBUG(
dbgs() <<
"[Attributor] Introducing call base context:" << CB
561 for (
const Function *Callee : Callees) {
570 IntroduceCallBaseContext ? &CB :
nullptr)
572 *
Callee, IntroduceCallBaseContext ? &CB : nullptr);
574 if (Attribute::isEnumAttrKind(IRAttributeKind)) {
577 A,
this, FnPos, DepClassTy::REQUIRED, IsKnown))
583 A.getAAFor<AAType>(*
this, FnPos, DepClassTy::REQUIRED);
587 if (S.isAtFixpoint())
588 return S.isValidState();
592 if (!
A.checkForAllCallees(CalleePred, *
this, CB))
593 return S.indicatePessimisticFixpoint();
599template <
class AAType,
typename StateType =
typename AAType::StateType>
605 for (
unsigned u = 0;
u <
Uses.size(); ++
u) {
609 if (Found &&
AA.followUseInMBEC(
A, U, UserI, State))
624template <
class AAType,
typename StateType =
typename AAType::StateType>
625static void followUsesInMBEC(AAType &
AA,
Attributor &
A, StateType &S,
627 if (!
A.canUseBodyFor(
AA.getIRPosition()))
630 const Value &Val =
AA.getIRPosition().getAssociatedValue();
635 A.getInfoCache().getMustBeExecutedContextExplorer();
641 for (
const Use &U : Val.
uses())
644 followUsesInContext<AAType>(
AA,
A, *Explorer, &CtxI,
Uses, S);
646 if (S.isAtFixpoint())
690 StateType ParentState;
694 ParentState.indicateOptimisticFixpoint();
696 for (
const BasicBlock *BB : Br->successors()) {
697 StateType ChildState;
699 size_t BeforeSize =
Uses.size();
700 followUsesInContext(
AA,
A, *Explorer, &BB->front(),
Uses, ChildState);
703 for (
auto It =
Uses.begin() + BeforeSize; It !=
Uses.end();)
706 ParentState &= ChildState;
757 R.indicatePessimisticFixpoint();
771 BS.indicateOptimisticFixpoint();
776 BS.indicatePessimisticFixpoint();
846 template <
typename F>
853 if (!
Range.mayOverlap(ItRange))
855 bool IsExact =
Range == ItRange && !
Range.offsetOrSizeAreUnknown();
856 for (
auto Index : It.getSecond()) {
866 template <
typename F>
877 for (
unsigned Index : LocalList->getSecond()) {
880 if (
Range.offsetAndSizeAreUnknown())
896 RemoteI = RemoteI ? RemoteI : &
I;
900 bool AccExists =
false;
902 for (
auto Index : LocalList) {
904 if (
A.getLocalInst() == &
I) {
913 <<
"[AAPointerInfo] Inserting access in new offset bins\n";);
915 for (
auto Key : ToAdd) {
922 AccessList.emplace_back(&
I, RemoteI, Ranges, Content, Kind, Ty);
924 "New Access should have been at AccIndex");
925 LocalList.push_back(AccIndex);
934 auto Before = Current;
936 if (Current == Before)
939 auto &ExistingRanges = Before.getRanges();
940 auto &NewRanges = Current.getRanges();
947 <<
"[AAPointerInfo] Removing access from old offset bins\n";);
954 "Expected bin to actually contain the Access.");
976struct AAPointerInfoImpl
977 :
public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
981 const std::string getAsStr(
Attributor *
A)
const override {
982 return std::string(
"PointerInfo ") +
983 (isValidState() ? (std::string(
"#") +
984 std::to_string(OffsetBins.size()) +
" bins")
989 [](int64_t O) {
return std::to_string(O); }),
996 return AAPointerInfo::manifest(
A);
999 const_bin_iterator
begin()
const override {
return State::begin(); }
1000 const_bin_iterator
end()
const override {
return State::end(); }
1001 int64_t numOffsetBins()
const override {
return State::numOffsetBins(); }
1002 bool reachesReturn()
const override {
1003 return !ReturnedOffsets.isUnassigned();
1005 void addReturnedOffsetsTo(OffsetInfo &OI)
const override {
1006 if (ReturnedOffsets.isUnknown()) {
1011 OffsetInfo MergedOI;
1012 for (
auto Offset : ReturnedOffsets) {
1013 OffsetInfo TmpOI = OI;
1015 MergedOI.merge(TmpOI);
1017 OI = std::move(MergedOI);
1020 ChangeStatus setReachesReturn(
const OffsetInfo &ReachedReturnedOffsets) {
1021 if (ReturnedOffsets.isUnknown())
1022 return ChangeStatus::UNCHANGED;
1023 if (ReachedReturnedOffsets.isUnknown()) {
1024 ReturnedOffsets.setUnknown();
1025 return ChangeStatus::CHANGED;
1027 if (ReturnedOffsets.merge(ReachedReturnedOffsets))
1028 return ChangeStatus::CHANGED;
1029 return ChangeStatus::UNCHANGED;
1032 bool forallInterferingAccesses(
1034 function_ref<
bool(
const AAPointerInfo::Access &,
bool)> CB)
1036 return State::forallInterferingAccesses(
Range, CB);
1039 bool forallInterferingAccesses(
1040 Attributor &
A,
const AbstractAttribute &QueryingAA, Instruction &
I,
1041 bool FindInterferingWrites,
bool FindInterferingReads,
1042 function_ref<
bool(
const Access &,
bool)> UserCB,
bool &HasBeenWrittenTo,
1044 function_ref<
bool(
const Access &)> SkipCB)
const override {
1045 HasBeenWrittenTo =
false;
1047 SmallPtrSet<const Access *, 8> DominatingWrites;
1055 const auto *ExecDomainAA =
A.lookupAAFor<AAExecutionDomain>(
1057 bool AllInSameNoSyncFn = IsAssumedNoSync;
1058 bool InstIsExecutedByInitialThreadOnly =
1059 ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(
I);
1066 bool InstIsExecutedInAlignedRegion =
1067 FindInterferingReads && ExecDomainAA &&
1068 ExecDomainAA->isExecutedInAlignedRegion(
A,
I);
1070 if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
1071 A.recordDependence(*ExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1073 InformationCache &InfoCache =
A.getInfoCache();
1074 bool IsThreadLocalObj =
1083 auto CanIgnoreThreadingForInst = [&](
const Instruction &
I) ->
bool {
1084 if (IsThreadLocalObj || AllInSameNoSyncFn)
1086 const auto *FnExecDomainAA =
1087 I.getFunction() == &
Scope
1089 :
A.lookupAAFor<AAExecutionDomain>(
1092 if (!FnExecDomainAA)
1094 if (InstIsExecutedInAlignedRegion ||
1095 (FindInterferingWrites &&
1096 FnExecDomainAA->isExecutedInAlignedRegion(
A,
I))) {
1097 A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1100 if (InstIsExecutedByInitialThreadOnly &&
1101 FnExecDomainAA->isExecutedByInitialThreadOnly(
I)) {
1102 A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
1111 auto CanIgnoreThreading = [&](
const Access &Acc) ->
bool {
1112 return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
1113 (Acc.getRemoteInst() != Acc.getLocalInst() &&
1114 CanIgnoreThreadingForInst(*Acc.getLocalInst()));
1118 bool IsKnownNoRecurse;
1126 bool InstInKernel =
A.getInfoCache().isKernel(Scope);
1127 bool ObjHasKernelLifetime =
false;
1128 const bool UseDominanceReasoning =
1129 FindInterferingWrites && IsKnownNoRecurse;
1130 const DominatorTree *DT =
1139 unsigned VAS =
V->getType()->getPointerAddressSpace();
1150 std::function<bool(
const Function &)> IsLiveInCalleeCB;
1155 const Function *AIFn = AI->getFunction();
1156 ObjHasKernelLifetime =
A.getInfoCache().isKernel(*AIFn);
1157 bool IsKnownNoRecurse;
1160 IsKnownNoRecurse)) {
1161 IsLiveInCalleeCB = [AIFn](
const Function &Fn) {
return AIFn != &Fn; };
1166 ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
1167 if (ObjHasKernelLifetime)
1168 IsLiveInCalleeCB = [&
A](
const Function &Fn) {
1169 return !
A.getInfoCache().isKernel(Fn);
1177 auto AccessCB = [&](
const Access &Acc,
bool Exact) {
1178 Function *AccScope = Acc.getRemoteInst()->getFunction();
1179 bool AccInSameScope = AccScope == &
Scope;
1183 if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
1184 A.getInfoCache().isKernel(*AccScope))
1187 if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &
I) {
1188 if (Acc.isWrite() || (
isa<LoadInst>(
I) && Acc.isWriteOrAssumption()))
1189 ExclusionSet.
insert(Acc.getRemoteInst());
1192 if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
1193 (!FindInterferingReads || !Acc.isRead()))
1196 bool Dominates = FindInterferingWrites && DT &&
Exact &&
1197 Acc.isMustAccess() && AccInSameScope &&
1200 DominatingWrites.
insert(&Acc);
1204 AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &
Scope;
1209 if (!State::forallInterferingAccesses(
I, AccessCB,
Range))
1212 HasBeenWrittenTo = !DominatingWrites.
empty();
1216 for (
const Access *Acc : DominatingWrites) {
1217 if (!LeastDominatingWriteInst) {
1218 LeastDominatingWriteInst = Acc->getRemoteInst();
1219 }
else if (DT->
dominates(LeastDominatingWriteInst,
1220 Acc->getRemoteInst())) {
1221 LeastDominatingWriteInst = Acc->getRemoteInst();
1226 auto CanSkipAccess = [&](
const Access &Acc,
bool Exact) {
1227 if (SkipCB && SkipCB(Acc))
1229 if (!CanIgnoreThreading(Acc))
1235 bool ReadChecked = !FindInterferingReads;
1236 bool WriteChecked = !FindInterferingWrites;
1242 &ExclusionSet, IsLiveInCalleeCB))
1247 if (!WriteChecked) {
1249 &ExclusionSet, IsLiveInCalleeCB))
1250 WriteChecked =
true;
1264 if (!WriteChecked && HasBeenWrittenTo &&
1265 Acc.getRemoteInst()->getFunction() != &Scope) {
1267 const auto *FnReachabilityAA =
A.getAAFor<AAInterFnReachability>(
1269 if (FnReachabilityAA) {
1275 if (!FnReachabilityAA->instructionCanReach(
1276 A, *LeastDominatingWriteInst,
1277 *Acc.getRemoteInst()->getFunction(), &ExclusionSet))
1278 WriteChecked =
true;
1285 if (ReadChecked && WriteChecked)
1288 if (!DT || !UseDominanceReasoning)
1290 if (!DominatingWrites.count(&Acc))
1292 return LeastDominatingWriteInst != Acc.getRemoteInst();
1297 for (
auto &It : InterferingAccesses) {
1298 if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
1299 !CanSkipAccess(*It.first, It.second)) {
1300 if (!UserCB(*It.first, It.second))
1308 const AAPointerInfo &OtherAA,
1310 using namespace AA::PointerInfo;
1312 return indicatePessimisticFixpoint();
1315 const auto &OtherAAImpl =
static_cast<const AAPointerInfoImpl &
>(OtherAA);
1316 bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
1317 Changed |= setReachesReturn(OtherAAImpl.ReturnedOffsets);
1320 const auto &State = OtherAAImpl.getState();
1321 for (
const auto &It : State) {
1322 for (
auto Index : It.getSecond()) {
1323 const auto &RAcc = State.getAccess(Index);
1324 if (IsByval && !RAcc.isRead())
1326 bool UsedAssumedInformation =
false;
1328 auto Content =
A.translateArgumentToCallSiteContent(
1329 RAcc.getContent(), CB, *
this, UsedAssumedInformation);
1330 AK =
AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
1331 AK =
AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
1333 Changed |= addAccess(
A, RAcc.getRanges(), CB, Content, AK,
1334 RAcc.getType(), RAcc.getRemoteInst());
1340 ChangeStatus translateAndAddState(Attributor &
A,
const AAPointerInfo &OtherAA,
1341 const OffsetInfo &Offsets, CallBase &CB,
1343 using namespace AA::PointerInfo;
1345 return indicatePessimisticFixpoint();
1347 const auto &OtherAAImpl =
static_cast<const AAPointerInfoImpl &
>(OtherAA);
1351 const auto &State = OtherAAImpl.getState();
1352 for (
const auto &It : State) {
1353 for (
auto Index : It.getSecond()) {
1354 const auto &RAcc = State.getAccess(Index);
1355 if (!IsMustAcc && RAcc.isAssumption())
1357 for (
auto Offset : Offsets) {
1361 if (!NewRanges.isUnknown()) {
1362 NewRanges.addToAllOffsets(Offset);
1367 Changed |= addAccess(
A, NewRanges, CB, RAcc.getContent(), AK,
1368 RAcc.getType(), RAcc.getRemoteInst());
1376 void trackPointerInfoStatistics(
const IRPosition &IRP)
const {}
1379 void dumpState(raw_ostream &O) {
1380 for (
auto &It : OffsetBins) {
1381 O <<
"[" << It.first.Offset <<
"-" << It.first.Offset + It.first.Size
1382 <<
"] : " << It.getSecond().size() <<
"\n";
1383 for (
auto AccIndex : It.getSecond()) {
1384 auto &Acc = AccessList[AccIndex];
1385 O <<
" - " << Acc.getKind() <<
" - " << *Acc.getLocalInst() <<
"\n";
1386 if (Acc.getLocalInst() != Acc.getRemoteInst())
1387 O <<
" --> " << *Acc.getRemoteInst()
1389 if (!Acc.isWrittenValueYetUndetermined()) {
1391 O <<
" - c: func " << Acc.getWrittenValue()->getName()
1393 else if (Acc.getWrittenValue())
1394 O <<
" - c: " << *Acc.getWrittenValue() <<
"\n";
1396 O <<
" - c: <unknown>\n";
1403struct AAPointerInfoFloating :
public AAPointerInfoImpl {
1405 AAPointerInfoFloating(
const IRPosition &IRP, Attributor &
A)
1406 : AAPointerInfoImpl(IRP,
A) {}
1409 bool handleAccess(Attributor &
A, Instruction &
I,
1410 std::optional<Value *> Content,
AccessKind Kind,
1413 using namespace AA::PointerInfo;
1415 const DataLayout &
DL =
A.getDataLayout();
1416 TypeSize AccessSize =
DL.getTypeStoreSize(&Ty);
1425 if (!VT || VT->getElementCount().isScalable() ||
1427 (*Content)->getType() != VT ||
1428 DL.getTypeStoreSize(VT->getElementType()).isScalable()) {
1439 int64_t ElementSize =
DL.getTypeStoreSize(ElementType).getFixedValue();
1443 OffsetsSorted.end());
1445 for (
int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
1447 ConstContent, ConstantInt::get(Int32Ty, i));
1454 for (
auto &ElementOffset : ElementOffsets)
1455 ElementOffset += ElementSize;
1467 bool collectConstantsForGEP(Attributor &
A,
const DataLayout &
DL,
1468 OffsetInfo &UsrOI,
const OffsetInfo &PtrOI,
1469 const GEPOperator *
GEP);
1471 void trackStatistics()
const override {
1472 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1476bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &
A,
1477 const DataLayout &
DL,
1479 const OffsetInfo &PtrOI,
1480 const GEPOperator *
GEP) {
1481 unsigned BitWidth =
DL.getIndexTypeSizeInBits(
GEP->getType());
1482 SmallMapVector<Value *, APInt, 4> VariableOffsets;
1485 assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
1486 "Don't look for constant values if the offset has already been "
1487 "determined to be unknown.");
1489 if (!
GEP->collectOffset(
DL,
BitWidth, VariableOffsets, ConstantOffset)) {
1495 << (VariableOffsets.
empty() ?
"" :
"not") <<
" constant "
1499 Union.addToAll(ConstantOffset.getSExtValue());
1504 for (
const auto &VI : VariableOffsets) {
1505 auto *PotentialConstantsAA =
A.getAAFor<AAPotentialConstantValues>(
1507 if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
1513 if (PotentialConstantsAA->undefIsContained())
1520 auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
1521 if (AssumedSet.empty())
1525 for (
const auto &ConstOffset : AssumedSet) {
1526 auto CopyPerOffset =
Union;
1527 CopyPerOffset.addToAll(ConstOffset.getSExtValue() *
1528 VI.second.getZExtValue());
1529 Product.merge(CopyPerOffset);
1534 UsrOI = std::move(Union);
1538ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &
A) {
1539 using namespace AA::PointerInfo;
1541 const DataLayout &
DL =
A.getDataLayout();
1542 Value &AssociatedValue = getAssociatedValue();
1544 DenseMap<Value *, OffsetInfo> OffsetInfoMap;
1545 OffsetInfoMap[&AssociatedValue].
insert(0);
1547 auto HandlePassthroughUser = [&](
Value *Usr,
Value *CurPtr,
bool &Follow) {
1558 "CurPtr does not exist in the map!");
1560 auto &UsrOI = OffsetInfoMap[Usr];
1561 auto &PtrOI = OffsetInfoMap[CurPtr];
1562 assert(!PtrOI.isUnassigned() &&
1563 "Cannot pass through if the input Ptr was not visited!");
1569 auto UsePred = [&](
const Use &
U,
bool &Follow) ->
bool {
1571 User *Usr =
U.getUser();
1572 LLVM_DEBUG(
dbgs() <<
"[AAPointerInfo] Analyze " << *CurPtr <<
" in " << *Usr
1575 "The current pointer offset should have been seeded!");
1576 assert(!OffsetInfoMap[CurPtr].isUnassigned() &&
1577 "Current pointer should be assigned");
1583 return HandlePassthroughUser(Usr, CurPtr, Follow);
1588 auto &UsrOI = OffsetInfoMap[Usr];
1589 auto &PtrOI = OffsetInfoMap[CurPtr];
1591 if (UsrOI.isUnknown())
1594 if (PtrOI.isUnknown()) {
1600 Follow = collectConstantsForGEP(
A,
DL, UsrOI, PtrOI,
GEP);
1604 return HandlePassthroughUser(Usr, CurPtr, Follow);
1609 if (RI->getFunction() == getAssociatedFunction()) {
1610 auto &PtrOI = OffsetInfoMap[CurPtr];
1611 Changed |= setReachesReturn(PtrOI);
1624 auto &UsrOI = PhiIt->second;
1625 auto &PtrOI = OffsetInfoMap[CurPtr];
1629 if (PtrOI.isUnknown()) {
1630 LLVM_DEBUG(
dbgs() <<
"[AAPointerInfo] PHI operand offset unknown "
1631 << *CurPtr <<
" in " << *
PHI <<
"\n");
1632 Follow = !UsrOI.isUnknown();
1638 if (UsrOI == PtrOI) {
1639 assert(!PtrOI.isUnassigned() &&
1640 "Cannot assign if the current Ptr was not visited!");
1641 LLVM_DEBUG(
dbgs() <<
"[AAPointerInfo] PHI is invariant (so far)");
1651 auto It = OffsetInfoMap.
find(CurPtrBase);
1652 if (It == OffsetInfoMap.
end()) {
1653 LLVM_DEBUG(
dbgs() <<
"[AAPointerInfo] PHI operand is too complex "
1654 << *CurPtr <<
" in " << *
PHI
1655 <<
" (base: " << *CurPtrBase <<
")\n");
1669 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
1670 *
PHI->getFunction());
1672 auto BaseOI = It->getSecond();
1673 BaseOI.addToAll(
Offset.getZExtValue());
1674 if (IsFirstPHIUser || BaseOI == UsrOI) {
1675 LLVM_DEBUG(
dbgs() <<
"[AAPointerInfo] PHI is invariant " << *CurPtr
1676 <<
" in " << *Usr <<
"\n");
1677 return HandlePassthroughUser(Usr, CurPtr, Follow);
1681 dbgs() <<
"[AAPointerInfo] PHI operand pointer offset mismatch "
1682 << *CurPtr <<
" in " << *
PHI <<
"\n");
1701 if (!handleAccess(
A, *LoadI,
nullptr, AK,
1702 OffsetInfoMap[CurPtr].Offsets,
Changed,
1708 return II->isAssumeLikeIntrinsic();
1719 }
while (FromI && FromI != ToI);
1724 auto IsValidAssume = [&](IntrinsicInst &IntrI) {
1725 if (IntrI.getIntrinsicID() != Intrinsic::assume)
1728 if (IntrI.getParent() == BB) {
1729 if (IsImpactedInRange(LoadI->getNextNode(), &IntrI))
1735 if ((*PredIt) != BB)
1740 if (SuccBB == IntrBB)
1746 if (IsImpactedInRange(LoadI->getNextNode(), BB->
getTerminator()))
1748 if (IsImpactedInRange(&IntrBB->
front(), &IntrI))
1754 std::pair<Value *, IntrinsicInst *>
Assumption;
1755 for (
const Use &LoadU : LoadI->uses()) {
1757 if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
1759 for (
const Use &CmpU : CmpI->uses()) {
1761 if (!IsValidAssume(*IntrI))
1763 int Idx = CmpI->getOperandUse(0) == LoadU;
1780 bool UsedAssumedInformation =
false;
1781 std::optional<Value *> Content =
nullptr;
1786 return handleAccess(
1787 A, *
Assumption.second, Content, AccessKind::AK_ASSUMPTION,
1788 OffsetInfoMap[CurPtr].Offsets,
Changed, *LoadI->getType());
1793 for (
auto *OtherOp : OtherOps) {
1794 if (OtherOp == CurPtr) {
1797 <<
"[AAPointerInfo] Escaping use in store like instruction " <<
I
1809 bool UsedAssumedInformation =
false;
1810 std::optional<Value *> Content =
nullptr;
1812 Content =
A.getAssumedSimplified(
1814 return handleAccess(
A,
I, Content, AK, OffsetInfoMap[CurPtr].Offsets,
1819 return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
1820 *StoreI->getValueOperand()->getType(),
1821 {StoreI->getValueOperand()}, AccessKind::AK_W);
1823 return HandleStoreLike(*RMWI,
nullptr, *RMWI->getValOperand()->getType(),
1824 {RMWI->getValOperand()}, AccessKind::AK_RW);
1826 return HandleStoreLike(
1827 *CXI,
nullptr, *CXI->getNewValOperand()->getType(),
1828 {CXI->getCompareOperand(), CXI->getNewValOperand()},
1835 A.getInfoCache().getTargetLibraryInfoForFunction(*CB->
getFunction());
1840 const auto *CSArgPI =
A.getAAFor<AAPointerInfo>(
1846 Changed = translateAndAddState(
A, *CSArgPI, OffsetInfoMap[CurPtr], *CB,
1849 if (!CSArgPI->reachesReturn())
1850 return isValidState();
1853 if (!Callee ||
Callee->arg_size() <= ArgNo)
1855 bool UsedAssumedInformation =
false;
1856 auto ReturnedValue =
A.getAssumedSimplified(
1861 auto *Arg =
Callee->getArg(ArgNo);
1862 if (ReturnedArg && Arg != ReturnedArg)
1864 bool IsRetMustAcc = IsArgMustAcc && (ReturnedArg == Arg);
1865 const auto *CSRetPI =
A.getAAFor<AAPointerInfo>(
1869 OffsetInfo OI = OffsetInfoMap[CurPtr];
1870 CSArgPI->addReturnedOffsetsTo(OI);
1872 translateAndAddState(
A, *CSRetPI, OI, *CB, IsRetMustAcc) |
Changed;
1873 return isValidState();
1875 LLVM_DEBUG(
dbgs() <<
"[AAPointerInfo] Call user not handled " << *CB
1880 LLVM_DEBUG(
dbgs() <<
"[AAPointerInfo] User not handled " << *Usr <<
"\n");
1883 auto EquivalentUseCB = [&](
const Use &OldU,
const Use &NewU) {
1884 assert(OffsetInfoMap.
count(OldU) &&
"Old use should be known already!");
1885 assert(!OffsetInfoMap[OldU].isUnassigned() &&
"Old use should be assinged");
1886 if (OffsetInfoMap.
count(NewU)) {
1888 if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
1889 dbgs() <<
"[AAPointerInfo] Equivalent use callback failed: "
1890 << OffsetInfoMap[NewU] <<
" vs " << OffsetInfoMap[OldU]
1894 return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
1897 return HandlePassthroughUser(NewU.get(), OldU.
get(), Unused);
1899 if (!
A.checkForAllUses(UsePred, *
this, AssociatedValue,
1901 true, EquivalentUseCB)) {
1902 LLVM_DEBUG(
dbgs() <<
"[AAPointerInfo] Check for all uses failed, abort!\n");
1903 return indicatePessimisticFixpoint();
1907 dbgs() <<
"Accesses by bin after update:\n";
1914struct AAPointerInfoReturned final : AAPointerInfoImpl {
1915 AAPointerInfoReturned(
const IRPosition &IRP, Attributor &
A)
1916 : AAPointerInfoImpl(IRP,
A) {}
1919 return indicatePessimisticFixpoint();
1922 void trackStatistics()
const override {
1923 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1927struct AAPointerInfoArgument final : AAPointerInfoFloating {
1928 AAPointerInfoArgument(
const IRPosition &IRP, Attributor &
A)
1929 : AAPointerInfoFloating(IRP,
A) {}
1931 void trackStatistics()
const override {
1932 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
1936struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
1937 AAPointerInfoCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
1938 : AAPointerInfoFloating(IRP,
A) {}
1941 using namespace AA::PointerInfo;
1947 if (
auto Length =
MI->getLengthInBytes())
1948 LengthVal =
Length->getSExtValue();
1949 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
1952 LLVM_DEBUG(
dbgs() <<
"[AAPointerInfo] Unhandled memory intrinsic "
1954 return indicatePessimisticFixpoint();
1957 ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
1959 Changed | addAccess(
A, {0, LengthVal}, *
MI,
nullptr,
Kind,
nullptr);
1962 dbgs() <<
"Accesses by bin after update:\n";
1973 Argument *Arg = getAssociatedArgument();
1977 A.getAAFor<AAPointerInfo>(*
this, ArgPos, DepClassTy::REQUIRED);
1978 if (ArgAA && ArgAA->getState().isValidState())
1979 return translateAndAddStateFromCallee(
A, *ArgAA,
1982 return indicatePessimisticFixpoint();
1985 bool IsKnownNoCapture;
1987 A,
this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture))
1988 return indicatePessimisticFixpoint();
1990 bool IsKnown =
false;
1992 return ChangeStatus::UNCHANGED;
1995 ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
2000 void trackStatistics()
const override {
2001 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
2005struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
2006 AAPointerInfoCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
2007 : AAPointerInfoFloating(IRP,
A) {}
2009 void trackStatistics()
const override {
2010 AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
2018struct AANoUnwindImpl : AANoUnwind {
2019 AANoUnwindImpl(
const IRPosition &IRP, Attributor &
A) : AANoUnwind(IRP,
A) {}
2024 A,
nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2028 const std::string getAsStr(Attributor *
A)
const override {
2029 return getAssumed() ?
"nounwind" :
"may-unwind";
2034 (unsigned)Instruction::Invoke, (
unsigned)Instruction::CallBr,
2035 (unsigned)Instruction::Call, (
unsigned)Instruction::CleanupRet,
2036 (unsigned)Instruction::CatchSwitch, (
unsigned)Instruction::Resume};
2039 if (!
I.mayThrow(
true))
2043 bool IsKnownNoUnwind;
2051 bool UsedAssumedInformation =
false;
2052 if (!
A.checkForAllInstructions(CheckForNoUnwind, *
this, Opcodes,
2053 UsedAssumedInformation))
2054 return indicatePessimisticFixpoint();
2056 return ChangeStatus::UNCHANGED;
2060struct AANoUnwindFunction final :
public AANoUnwindImpl {
2061 AANoUnwindFunction(
const IRPosition &IRP, Attributor &
A)
2062 : AANoUnwindImpl(IRP,
A) {}
2068struct AANoUnwindCallSite final
2069 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
2070 AANoUnwindCallSite(
const IRPosition &IRP, Attributor &
A)
2071 : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP,
A) {}
2081 case Intrinsic::nvvm_barrier_cta_sync_aligned_all:
2082 case Intrinsic::nvvm_barrier_cta_sync_aligned_count:
2083 case Intrinsic::nvvm_barrier_cta_red_and_aligned_all:
2084 case Intrinsic::nvvm_barrier_cta_red_and_aligned_count:
2085 case Intrinsic::nvvm_barrier_cta_red_or_aligned_all:
2086 case Intrinsic::nvvm_barrier_cta_red_or_aligned_count:
2087 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_all:
2088 case Intrinsic::nvvm_barrier_cta_red_popc_aligned_count:
2090 case Intrinsic::amdgcn_s_barrier:
2091 if (ExecutedAligned)
2114 switch (
I->getOpcode()) {
2115 case Instruction::AtomicRMW:
2118 case Instruction::Store:
2121 case Instruction::Load:
2126 "New atomic operations need to be known in the attributor.");
2144 const std::string getAsStr(Attributor *
A)
const override {
2145 return getAssumed() ?
"nosync" :
"may-sync";
2160 if (
I.mayReadOrWriteMemory())
2174 bool UsedAssumedInformation =
false;
2175 if (!
A.checkForAllReadWriteInstructions(CheckRWInstForNoSync, *
this,
2176 UsedAssumedInformation) ||
2177 !
A.checkForAllCallLikeInstructions(CheckForNoSync, *
this,
2178 UsedAssumedInformation))
2179 return indicatePessimisticFixpoint();
2184struct AANoSyncFunction final :
public AANoSyncImpl {
2185 AANoSyncFunction(
const IRPosition &IRP, Attributor &
A)
2186 : AANoSyncImpl(IRP,
A) {}
2192struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
2193 AANoSyncCallSite(
const IRPosition &IRP, Attributor &
A)
2194 : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP,
A) {}
2203struct AANoFreeImpl :
public AANoFree {
2204 AANoFreeImpl(
const IRPosition &IRP, Attributor &
A) : AANoFree(IRP,
A) {}
2209 DepClassTy::NONE, IsKnown));
2226 bool UsedAssumedInformation =
false;
2227 if (!
A.checkForAllReadWriteInstructions(CheckForNoFree, *
this,
2228 UsedAssumedInformation) ||
2229 !
A.checkForAllCallLikeInstructions(CheckForNoFree, *
this,
2230 UsedAssumedInformation))
2231 return indicatePessimisticFixpoint();
2233 return ChangeStatus::UNCHANGED;
2236 const std::string getAsStr(Attributor *
A)
const override {
2237 return getAssumed() ?
"nofree" :
"may-free";
2241struct AANoFreeFunction final :
public AANoFreeImpl {
2242 AANoFreeFunction(
const IRPosition &IRP, Attributor &
A)
2243 : AANoFreeImpl(IRP,
A) {}
2249struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
2250 AANoFreeCallSite(
const IRPosition &IRP, Attributor &
A)
2251 : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP,
A) {}
2257struct AANoFreeFloating : AANoFreeImpl {
2258 AANoFreeFloating(
const IRPosition &IRP, Attributor &
A)
2259 : AANoFreeImpl(IRP,
A) {}
2265 const IRPosition &IRP = getIRPosition();
2270 DepClassTy::OPTIONAL, IsKnown))
2271 return ChangeStatus::UNCHANGED;
2273 Value &AssociatedValue = getIRPosition().getAssociatedValue();
2274 auto Pred = [&](
const Use &
U,
bool &Follow) ->
bool {
2289 DepClassTy::REQUIRED, IsKnown))
2292 const AANoCapture *NoCaptureAA =
nullptr;
2295 DepClassTy::REQUIRED, IsKnown,
2296 false, &NoCaptureAA)) {
2321 if (!
A.checkForAllUses(
Pred, *
this, AssociatedValue))
2322 return indicatePessimisticFixpoint();
2324 return ChangeStatus::UNCHANGED;
2329struct AANoFreeArgument final : AANoFreeFloating {
2330 AANoFreeArgument(
const IRPosition &IRP, Attributor &
A)
2331 : AANoFreeFloating(IRP,
A) {}
2337struct AANoFreeCallSiteArgument final : AANoFreeFloating {
2338 AANoFreeCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
2339 : AANoFreeFloating(IRP,
A) {}
2346 Argument *Arg = getAssociatedArgument();
2348 return indicatePessimisticFixpoint();
2352 DepClassTy::REQUIRED, IsKnown))
2353 return ChangeStatus::UNCHANGED;
2354 return indicatePessimisticFixpoint();
2361struct AANoFreeReturned final : AANoFreeFloating {
2362 AANoFreeReturned(
const IRPosition &IRP, Attributor &
A)
2363 : AANoFreeFloating(IRP,
A) {
2375 void trackStatistics()
const override {}
2379struct AANoFreeCallSiteReturned final : AANoFreeFloating {
2380 AANoFreeCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
2381 : AANoFreeFloating(IRP,
A) {}
2384 return ChangeStatus::UNCHANGED;
2394 bool IgnoreSubsumingPositions) {
2396 AttrKinds.
push_back(Attribute::NonNull);
2399 AttrKinds.
push_back(Attribute::Dereferenceable);
2400 if (
A.hasAttr(IRP, AttrKinds, IgnoreSubsumingPositions, Attribute::NonNull))
2403 if (!
A.canUseBodyFor(IRP))
2410 if (!Fn->isDeclaration()) {
2420 bool UsedAssumedInformation =
false;
2421 if (!
A.checkForAllInstructions(
2423 Worklist.push_back({*cast<ReturnInst>(I).getReturnValue(), &I});
2427 UsedAssumedInformation,
false,
true))
2439 Attribute::NonNull)});
2444static int64_t getKnownNonNullAndDerefBytesForUse(
2445 Attributor &
A,
const AbstractAttribute &QueryingAA,
Value &AssociatedValue,
2446 const Use *U,
const Instruction *
I,
bool &IsNonNull,
bool &TrackUse) {
2449 const Value *UseV =
U->get();
2470 const DataLayout &
DL =
A.getInfoCache().getDL();
2475 case BundleAttr::NonNull:
2478 case BundleAttr::Dereferenceable: {
2482 return CI->getZExtValue();
2499 bool IsKnownNonNull;
2502 IsNonNull |= IsKnownNonNull;
2505 return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
2509 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
2510 Loc->Size.isScalable() ||
I->isVolatile())
2516 if (
Base &&
Base == &AssociatedValue) {
2517 int64_t DerefBytes = Loc->Size.getValue() +
Offset;
2519 return std::max(int64_t(0), DerefBytes);
2526 int64_t DerefBytes = Loc->Size.getValue();
2528 return std::max(int64_t(0), DerefBytes);
2534struct AANonNullImpl : AANonNull {
2535 AANonNullImpl(
const IRPosition &IRP, Attributor &
A) : AANonNull(IRP,
A) {}
2538 Value &
V = *getAssociatedValue().stripPointerCasts();
2540 indicatePessimisticFixpoint();
2544 if (Instruction *CtxI = getCtxI())
2545 followUsesInMBEC(*
this,
A, getState(), *CtxI);
2549 bool followUseInMBEC(Attributor &
A,
const Use *U,
const Instruction *
I,
2550 AANonNull::StateType &State) {
2551 bool IsNonNull =
false;
2552 bool TrackUse =
false;
2553 getKnownNonNullAndDerefBytesForUse(
A, *
this, getAssociatedValue(), U,
I,
2554 IsNonNull, TrackUse);
2555 State.setKnown(IsNonNull);
2559 const std::string getAsStr(Attributor *
A)
const override {
2560 return getAssumed() ?
"nonnull" :
"may-null";
2565struct AANonNullFloating :
public AANonNullImpl {
2566 AANonNullFloating(
const IRPosition &IRP, Attributor &
A)
2567 : AANonNullImpl(IRP,
A) {}
2570 auto CheckIRP = [&](
const IRPosition &IRP) {
2571 bool IsKnownNonNull;
2573 A, *
this, IRP, DepClassTy::OPTIONAL, IsKnownNonNull);
2577 bool UsedAssumedInformation =
false;
2578 Value *AssociatedValue = &getAssociatedValue();
2580 if (!
A.getAssumedSimplifiedValues(getIRPosition(), *
this,
Values,
2585 Values.size() != 1 ||
Values.front().getValue() != AssociatedValue;
2591 return AA::hasAssumedIRAttr<Attribute::NonNull>(
2592 A, this, IRPosition::value(*Op), DepClassTy::OPTIONAL,
2595 return ChangeStatus::UNCHANGED;
2599 DepClassTy::OPTIONAL, IsKnown) &&
2602 DepClassTy::OPTIONAL, IsKnown))
2603 return ChangeStatus::UNCHANGED;
2610 if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
2611 return indicatePessimisticFixpoint();
2612 return ChangeStatus::UNCHANGED;
2615 for (
const auto &VAC :
Values)
2617 return indicatePessimisticFixpoint();
2619 return ChangeStatus::UNCHANGED;
2626struct AANonNullReturned final
2627 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2628 false, AANonNull::IRAttributeKind, false> {
2629 AANonNullReturned(
const IRPosition &IRP, Attributor &
A)
2630 : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
2634 const std::string getAsStr(Attributor *
A)
const override {
2635 return getAssumed() ?
"nonnull" :
"may-null";
2642struct AANonNullArgument final
2643 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
2644 AANonNullArgument(
const IRPosition &IRP, Attributor &
A)
2645 : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP,
A) {}
2650struct AANonNullCallSiteArgument final : AANonNullFloating {
2651 AANonNullCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
2652 : AANonNullFloating(IRP,
A) {}
2658struct AANonNullCallSiteReturned final
2659 : AACalleeToCallSite<AANonNull, AANonNullImpl> {
2660 AANonNullCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
2661 : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP,
A) {}
2669struct AAMustProgressImpl :
public AAMustProgress {
2670 AAMustProgressImpl(
const IRPosition &IRP, Attributor &
A)
2671 : AAMustProgress(IRP,
A) {}
2676 A,
nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2680 const std::string getAsStr(Attributor *
A)
const override {
2681 return getAssumed() ?
"mustprogress" :
"may-not-progress";
2685struct AAMustProgressFunction final : AAMustProgressImpl {
2686 AAMustProgressFunction(
const IRPosition &IRP, Attributor &
A)
2687 : AAMustProgressImpl(IRP,
A) {}
2692 A,
this, getIRPosition(), DepClassTy::OPTIONAL, IsKnown)) {
2694 return indicateOptimisticFixpoint();
2695 return ChangeStatus::UNCHANGED;
2698 auto CheckForMustProgress = [&](AbstractCallSite ACS) {
2700 bool IsKnownMustProgress;
2702 A,
this, IPos, DepClassTy::REQUIRED, IsKnownMustProgress,
2706 bool AllCallSitesKnown =
true;
2707 if (!
A.checkForAllCallSites(CheckForMustProgress, *
this,
2710 return indicatePessimisticFixpoint();
2712 return ChangeStatus::UNCHANGED;
2715 void trackStatistics()
const override {
2721struct AAMustProgressCallSite final : AAMustProgressImpl {
2722 AAMustProgressCallSite(
const IRPosition &IRP, Attributor &
A)
2723 : AAMustProgressImpl(IRP,
A) {}
2731 bool IsKnownMustProgress;
2733 A,
this, FnPos, DepClassTy::REQUIRED, IsKnownMustProgress))
2734 return indicatePessimisticFixpoint();
2735 return ChangeStatus::UNCHANGED;
2738 void trackStatistics()
const override {
2747struct AANoRecurseImpl :
public AANoRecurse {
2748 AANoRecurseImpl(
const IRPosition &IRP, Attributor &
A) : AANoRecurse(IRP,
A) {}
2753 A,
nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
2757 const std::string getAsStr(Attributor *
A)
const override {
2758 return getAssumed() ?
"norecurse" :
"may-recurse";
2762struct AANoRecurseFunction final : AANoRecurseImpl {
2763 AANoRecurseFunction(
const IRPosition &IRP, Attributor &
A)
2764 : AANoRecurseImpl(IRP,
A) {}
2769 auto CallSitePred = [&](AbstractCallSite ACS) {
2770 bool IsKnownNoRecurse;
2774 DepClassTy::NONE, IsKnownNoRecurse))
2776 return IsKnownNoRecurse;
2778 bool UsedAssumedInformation =
false;
2779 if (
A.checkForAllCallSites(CallSitePred, *
this,
true,
2780 UsedAssumedInformation)) {
2786 if (!UsedAssumedInformation)
2787 indicateOptimisticFixpoint();
2788 return ChangeStatus::UNCHANGED;
2791 const AAInterFnReachability *EdgeReachability =
2792 A.getAAFor<AAInterFnReachability>(*
this, getIRPosition(),
2793 DepClassTy::REQUIRED);
2794 if (EdgeReachability && EdgeReachability->
canReach(
A, *getAnchorScope()))
2795 return indicatePessimisticFixpoint();
2796 return ChangeStatus::UNCHANGED;
2803struct AANoRecurseCallSite final
2804 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
2805 AANoRecurseCallSite(
const IRPosition &IRP, Attributor &
A)
2806 : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP,
A) {}
2815struct AANonConvergentImpl :
public AANonConvergent {
2816 AANonConvergentImpl(
const IRPosition &IRP, Attributor &
A)
2817 : AANonConvergent(IRP,
A) {}
2819 const std::string getAsStr(Attributor *
A)
const override {
2820 return getAssumed() ?
"non-convergent" :
"may-be-convergent";
2824struct AANonConvergentFunction final : AANonConvergentImpl {
2825 AANonConvergentFunction(
const IRPosition &IRP, Attributor &
A)
2826 : AANonConvergentImpl(IRP,
A) {}
2831 auto CalleeIsNotConvergent = [&](
Instruction &Inst) {
2834 if (!Callee ||
Callee->isIntrinsic()) {
2837 if (
Callee->isDeclaration()) {
2838 return !
Callee->hasFnAttribute(Attribute::Convergent);
2840 const auto *ConvergentAA =
A.getAAFor<AANonConvergent>(
2842 return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
2845 bool UsedAssumedInformation =
false;
2846 if (!
A.checkForAllCallLikeInstructions(CalleeIsNotConvergent, *
this,
2847 UsedAssumedInformation)) {
2848 return indicatePessimisticFixpoint();
2850 return ChangeStatus::UNCHANGED;
2854 if (isKnownNotConvergent() &&
2855 A.hasAttr(getIRPosition(), Attribute::Convergent)) {
2856 A.removeAttrs(getIRPosition(), {Attribute::Convergent});
2857 return ChangeStatus::CHANGED;
2859 return ChangeStatus::UNCHANGED;
2869struct AAUndefinedBehaviorImpl :
public AAUndefinedBehavior {
2870 AAUndefinedBehaviorImpl(
const IRPosition &IRP, Attributor &
A)
2871 : AAUndefinedBehavior(IRP,
A) {}
2877 UndefBranchCondition,
2879 NullReturnViolatesNonNull,
2881 NullArgViolatesNonNull,
2885 std::optional<unsigned> ArgNo;
2887 UBInfo(Kind K) :
K(
K), ArgNo(std::nullopt) {}
2889 UBInfo(Kind K, std::optional<unsigned> ArgNo) :
K(
K), ArgNo(ArgNo) {}
2893 const size_t UBPrevSize = KnownUBInsts.size();
2894 const size_t NoUBPrevSize = AssumedNoUBInsts.size();
2902 if (AssumedNoUBInsts.count(&
I) || KnownUBInsts.count(&
I))
2911 "Expected pointer operand of memory accessing instruction");
2915 std::optional<Value *> SimplifiedPtrOp =
2916 stopOnUndefOrAssumed(
A, PtrOp, &
I, UBInfo::UndefPtrAccess);
2917 if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
2919 const Value *PtrOpVal = *SimplifiedPtrOp;
2925 AssumedNoUBInsts.insert(&
I);
2937 AssumedNoUBInsts.insert(&
I);
2939 KnownUBInsts.try_emplace(&
I, UBInfo::NullPtrAccess);
2948 if (AssumedNoUBInsts.count(&
I) || KnownUBInsts.count(&
I))
2956 std::optional<Value *> SimplifiedCond = stopOnUndefOrAssumed(
2957 A, BrInst->getCondition(), BrInst, UBInfo::UndefBranchCondition);
2958 if (!SimplifiedCond || !*SimplifiedCond)
2960 AssumedNoUBInsts.insert(&
I);
2968 if (AssumedNoUBInsts.count(&
I) || KnownUBInsts.count(&
I))
2977 for (
unsigned idx = 0; idx < CB.
arg_size(); idx++) {
2983 if (idx >=
Callee->arg_size())
2995 bool IsKnownNoUndef;
2997 A,
this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNoUndef);
2998 if (!IsKnownNoUndef)
3000 bool UsedAssumedInformation =
false;
3001 std::optional<Value *> SimplifiedVal =
3004 if (UsedAssumedInformation)
3006 if (SimplifiedVal && !*SimplifiedVal)
3009 KnownUBInsts.try_emplace(&
I, UBInfo(UBInfo::UndefCallArgument, idx));
3015 bool IsKnownNonNull;
3017 A,
this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNonNull);
3019 KnownUBInsts.try_emplace(&
I,
3020 UBInfo(UBInfo::NullArgViolatesNonNull, idx));
3029 std::optional<Value *> SimplifiedRetValue = stopOnUndefOrAssumed(
3030 A, RI.getReturnValue(), &
I, UBInfo::UndefReturnValue);
3031 if (!SimplifiedRetValue || !*SimplifiedRetValue)
3049 bool IsKnownNonNull;
3054 KnownUBInsts.try_emplace(&
I, UBInfo::NullReturnViolatesNonNull);
3060 bool UsedAssumedInformation =
false;
3061 A.checkForAllInstructions(InspectMemAccessInstForUB, *
this,
3062 {Instruction::Load, Instruction::Store,
3063 Instruction::AtomicCmpXchg,
3064 Instruction::AtomicRMW},
3065 UsedAssumedInformation,
3067 A.checkForAllInstructions(InspectBrInstForUB, *
this, {Instruction::CondBr},
3068 UsedAssumedInformation,
3070 A.checkForAllCallLikeInstructions(InspectCallSiteForUB, *
this,
3071 UsedAssumedInformation);
3075 if (!getAnchorScope()->getReturnType()->isVoidTy()) {
3077 if (!
A.isAssumedDead(ReturnIRP,
this,
nullptr, UsedAssumedInformation)) {
3078 bool IsKnownNoUndef;
3080 A,
this, ReturnIRP, DepClassTy::NONE, IsKnownNoUndef);
3082 A.checkForAllInstructions(InspectReturnInstForUB, *
this,
3083 {Instruction::Ret}, UsedAssumedInformation,
3088 if (NoUBPrevSize != AssumedNoUBInsts.size() ||
3089 UBPrevSize != KnownUBInsts.size())
3090 return ChangeStatus::CHANGED;
3091 return ChangeStatus::UNCHANGED;
3094 bool isKnownToCauseUB(Instruction *
I)
const override {
3095 return KnownUBInsts.count(
I);
3098 bool isAssumedToCauseUB(Instruction *
I)
const override {
3105 switch (
I->getOpcode()) {
3106 case Instruction::Load:
3107 case Instruction::Store:
3108 case Instruction::AtomicCmpXchg:
3109 case Instruction::AtomicRMW:
3110 case Instruction::CondBr:
3111 return !AssumedNoUBInsts.count(
I);
3120 static void emitUBRemark(Attributor &
A, Instruction *
I,
const UBInfo &Info) {
3121 auto Remark = [&](OptimizationRemark
OR) {
3123 case UBInfo::NullPtrAccess:
3124 case UBInfo::UndefPtrAccess: {
3125 return OR <<
"Memory access through a pointer known to be "
3128 <<
" is undefined behavior; replacing with 'unreachable'.";
3130 case UBInfo::UndefBranchCondition:
3131 return OR <<
"Branch condition known to be "
3133 <<
" is undefined behavior; replacing with 'unreachable'.";
3134 case UBInfo::UndefReturnValue:
3135 case UBInfo::NullReturnViolatesNonNull:
3136 return OR <<
"Value returned known to be "
3139 <<
" is undefined behavior; replacing with 'unreachable'.";
3140 case UBInfo::UndefCallArgument:
3141 case UBInfo::NullArgViolatesNonNull: {
3142 bool IsUndef =
Info.K == UBInfo::UndefCallArgument;
3145 <<
" passed to parameter of ";
3150 return OR <<
" known to be "
3151 <<
ore::NV(
"Argument", IsUndef ?
"undef" :
"null")
3152 <<
" is undefined behavior; replacing with 'unreachable'.";
3157 A.emitRemark<OptimizationRemark>(
I,
"UndefinedBehavior",
Remark);
3161 if (KnownUBInsts.empty())
3162 return ChangeStatus::UNCHANGED;
3163 for (
const auto &[
I, Info] : KnownUBInsts) {
3164 emitUBRemark(
A,
I, Info);
3165 A.changeToUnreachableAfterManifest(
I);
3167 return ChangeStatus::CHANGED;
3170 const std::string getAsStr(Attributor *
A)
const override {
3171 return getAssumed() ?
"undefined-behavior" :
"no-ub";
3200 MapVector<Instruction *, UBInfo> KnownUBInsts;
3204 SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
3215 std::optional<Value *> stopOnUndefOrAssumed(Attributor &
A,
Value *
V,
3216 Instruction *
I, UBInfo::Kind K) {
3217 bool UsedAssumedInformation =
false;
3218 std::optional<Value *> SimplifiedV =
3221 if (!UsedAssumedInformation) {
3226 KnownUBInsts.try_emplace(
I, K);
3227 return std::nullopt;
3234 KnownUBInsts.try_emplace(
I, K);
3235 return std::nullopt;
3241struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
3242 AAUndefinedBehaviorFunction(
const IRPosition &IRP, Attributor &
A)
3243 : AAUndefinedBehaviorImpl(IRP,
A) {}
3245 void trackStatistics()
const override {
3246 STATS_DECL(UndefinedBehaviorInstruction, Instruction,
3247 "Number of instructions known to have UB");
3249 KnownUBInsts.size();
3260static bool mayContainUnboundedCycle(
Function &
F, Attributor &
A) {
3261 ScalarEvolution *SE =
3262 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
F);
3263 LoopInfo *LI =
A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
F);
3269 for (scc_iterator<Function *> SCCI =
scc_begin(&
F); !SCCI.isAtEnd(); ++SCCI)
3270 if (SCCI.hasCycle())
3280 for (
auto *L : LI->getLoopsInPreorder()) {
3287struct AAWillReturnImpl :
public AAWillReturn {
3288 AAWillReturnImpl(
const IRPosition &IRP, Attributor &
A)
3289 : AAWillReturn(IRP,
A) {}
3294 A,
nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
3299 bool isImpliedByMustprogressAndReadonly(Attributor &
A,
bool KnownOnly) {
3300 if (!
A.hasAttr(getIRPosition(), {Attribute::MustProgress}))
3305 return IsKnown || !KnownOnly;
3310 if (isImpliedByMustprogressAndReadonly(
A,
false))
3311 return ChangeStatus::UNCHANGED;
3317 A,
this, IPos, DepClassTy::REQUIRED, IsKnown)) {
3323 bool IsKnownNoRecurse;
3325 A,
this, IPos, DepClassTy::REQUIRED, IsKnownNoRecurse);
3328 bool UsedAssumedInformation =
false;
3329 if (!
A.checkForAllCallLikeInstructions(CheckForWillReturn, *
this,
3330 UsedAssumedInformation))
3331 return indicatePessimisticFixpoint();
3335 return !
I.isVolatile();
3337 if (!
A.checkForAllInstructions(CheckForVolatile, *
this,
3338 {Instruction::Load, Instruction::Store,
3339 Instruction::AtomicCmpXchg,
3340 Instruction::AtomicRMW},
3341 UsedAssumedInformation))
3342 return indicatePessimisticFixpoint();
3344 return ChangeStatus::UNCHANGED;
3347 const std::string getAsStr(Attributor *
A)
const override {
3348 return getAssumed() ?
"willreturn" :
"may-noreturn";
3352struct AAWillReturnFunction final : AAWillReturnImpl {
3353 AAWillReturnFunction(
const IRPosition &IRP, Attributor &
A)
3354 : AAWillReturnImpl(IRP,
A) {}
3357 AAWillReturnImpl::initialize(
A);
3360 assert(
F &&
"Did expect an anchor function");
3361 if (
F->isDeclaration() || mayContainUnboundedCycle(*
F,
A))
3362 indicatePessimisticFixpoint();
3369struct AAWillReturnCallSite final
3370 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
3371 AAWillReturnCallSite(
const IRPosition &IRP, Attributor &
A)
3372 : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP,
A) {}
3375 if (isImpliedByMustprogressAndReadonly(
A,
false))
3376 return ChangeStatus::UNCHANGED;
3378 return AACalleeToCallSite::updateImpl(
A);
3399 const ToTy *
To =
nullptr;
3426 if (!ES || ES->
empty()) {
3427 ExclusionSet = nullptr;
3428 }
else if (MakeUnique) {
3429 ExclusionSet =
A.getInfoCache().getOrCreateUniqueBlockExecutionSet(ES);
3447 if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
3449 return InstSetDMI::isEqual(LHS->ExclusionSet, RHS->ExclusionSet);
3457template <
typename BaseTy,
typename ToTy>
3458struct CachedReachabilityAA :
public BaseTy {
3459 using RQITy = ReachabilityQueryInfo<ToTy>;
3461 CachedReachabilityAA(
const IRPosition &IRP, Attributor &
A) : BaseTy(IRP,
A) {}
3463 bool isQueryAA()
const override {
return true; }
3467 for (
unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
3468 RQITy *RQI = QueryVector[
u];
3469 if (RQI->Result == RQITy::Reachable::No &&
3471 Changed = ChangeStatus::CHANGED;
3477 bool IsTemporaryRQI) = 0;
3479 bool rememberResult(Attributor &
A,
typename RQITy::Reachable Result,
3480 RQITy &RQI,
bool UsedExclusionSet,
bool IsTemporaryRQI) {
3485 QueryCache.erase(&RQI);
3491 if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
3492 RQITy PlainRQI(RQI.From, RQI.To);
3493 if (!QueryCache.count(&PlainRQI)) {
3494 RQITy *RQIPtr =
new (
A.Allocator) RQITy(RQI.From, RQI.To);
3496 QueryVector.push_back(RQIPtr);
3497 QueryCache.insert(RQIPtr);
3502 if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
3503 assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
3504 "Did not expect empty set!");
3505 RQITy *RQIPtr =
new (
A.Allocator)
3506 RQITy(
A, *RQI.From, *RQI.To, RQI.ExclusionSet,
true);
3507 assert(RQIPtr->Result == RQITy::Reachable::No &&
"Already reachable?");
3509 assert(!QueryCache.count(RQIPtr));
3510 QueryVector.push_back(RQIPtr);
3511 QueryCache.insert(RQIPtr);
3514 if (Result == RQITy::Reachable::No && IsTemporaryRQI)
3515 A.registerForUpdate(*
this);
3516 return Result == RQITy::Reachable::Yes;
3519 const std::string getAsStr(Attributor *
A)
const override {
3521 return "#queries(" + std::to_string(QueryVector.size()) +
")";
3524 bool checkQueryCache(Attributor &
A, RQITy &StackRQI,
3525 typename RQITy::Reachable &Result) {
3526 if (!this->getState().isValidState()) {
3527 Result = RQITy::Reachable::Yes;
3533 if (StackRQI.ExclusionSet) {
3534 RQITy PlainRQI(StackRQI.From, StackRQI.To);
3535 auto It = QueryCache.find(&PlainRQI);
3536 if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
3537 Result = RQITy::Reachable::No;
3542 auto It = QueryCache.find(&StackRQI);
3543 if (It != QueryCache.end()) {
3550 QueryCache.insert(&StackRQI);
3556 DenseSet<RQITy *> QueryCache;
3559struct AAIntraFnReachabilityFunction final
3560 :
public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
3561 using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
3562 AAIntraFnReachabilityFunction(
const IRPosition &IRP, Attributor &
A)
3564 DT =
A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
3568 bool isAssumedReachable(
3569 Attributor &
A,
const Instruction &From,
const Instruction &To,
3571 auto *NonConstThis =
const_cast<AAIntraFnReachabilityFunction *
>(
this);
3575 RQITy StackRQI(
A, From, To, ExclusionSet,
false);
3577 if (!NonConstThis->checkQueryCache(
A, StackRQI, Result))
3578 return NonConstThis->isReachableImpl(
A, StackRQI,
3580 return Result == RQITy::Reachable::Yes;
3587 A.getAAFor<AAIsDead>(*
this, getIRPosition(), DepClassTy::OPTIONAL);
3590 [&](
const auto &DeadEdge) {
3591 return LivenessAA->isEdgeDead(DeadEdge.first,
3595 return LivenessAA->isAssumedDead(BB);
3597 return ChangeStatus::UNCHANGED;
3601 return Base::updateImpl(
A);
3605 bool IsTemporaryRQI)
override {
3607 bool UsedExclusionSet =
false;
3612 while (IP && IP != &To) {
3613 if (ExclusionSet && IP != Origin && ExclusionSet->
count(IP)) {
3614 UsedExclusionSet =
true;
3622 const BasicBlock *FromBB = RQI.From->getParent();
3623 const BasicBlock *ToBB = RQI.To->getParent();
3625 "Not an intra-procedural query!");
3629 if (FromBB == ToBB &&
3630 WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
3631 return rememberResult(
A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3636 if (!WillReachInBlock(ToBB->
front(), *RQI.To, RQI.ExclusionSet))
3637 return rememberResult(
A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3641 SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
3642 if (RQI.ExclusionSet)
3643 for (
auto *
I : *RQI.ExclusionSet)
3644 if (
I->getFunction() == Fn)
3645 ExclusionBlocks.
insert(
I->getParent());
3648 if (ExclusionBlocks.
count(FromBB) &&
3651 return rememberResult(
A, RQITy::Reachable::No, RQI,
true, IsTemporaryRQI);
3654 A.getAAFor<AAIsDead>(*
this, getIRPosition(), DepClassTy::OPTIONAL);
3655 if (LivenessAA && LivenessAA->isAssumedDead(ToBB)) {
3656 DeadBlocks.insert(ToBB);
3657 return rememberResult(
A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3661 SmallPtrSet<const BasicBlock *, 16> Visited;
3665 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
3666 while (!Worklist.
empty()) {
3668 if (!Visited.
insert(BB).second)
3670 for (
const BasicBlock *SuccBB :
successors(BB)) {
3671 if (LivenessAA && LivenessAA->isEdgeDead(BB, SuccBB)) {
3672 LocalDeadEdges.
insert({BB, SuccBB});
3677 return rememberResult(
A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3680 return rememberResult(
A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
3683 if (ExclusionBlocks.
count(SuccBB)) {
3684 UsedExclusionSet =
true;
3691 DeadEdges.insert_range(LocalDeadEdges);
3692 return rememberResult(
A, RQITy::Reachable::No, RQI, UsedExclusionSet,
3696 void trackStatistics()
const override {}
3701 DenseSet<const BasicBlock *> DeadBlocks;
3705 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
3708 const DominatorTree *DT =
nullptr;
3716 bool IgnoreSubsumingPositions) {
3717 assert(ImpliedAttributeKind == Attribute::NoAlias &&
3718 "Unexpected attribute kind");
3724 IgnoreSubsumingPositions =
true;
3735 if (
A.hasAttr(IRP, {Attribute::ByVal, Attribute::NoAlias},
3736 IgnoreSubsumingPositions, Attribute::NoAlias))
3746 "Noalias is a pointer attribute");
3749 const std::string getAsStr(
Attributor *
A)
const override {
3750 return getAssumed() ?
"noalias" :
"may-alias";
3755struct AANoAliasFloating final : AANoAliasImpl {
3756 AANoAliasFloating(
const IRPosition &IRP, Attributor &
A)
3757 : AANoAliasImpl(IRP,
A) {}
3761 return indicatePessimisticFixpoint();
3764 void trackStatistics()
const override {
3770struct AANoAliasArgument final
3771 : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
3772 using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
3773 AANoAliasArgument(
const IRPosition &IRP, Attributor &
A) :
Base(IRP,
A) {}
3785 DepClassTy::OPTIONAL, IsKnownNoSycn))
3786 return Base::updateImpl(
A);
3791 return Base::updateImpl(
A);
3795 bool UsedAssumedInformation =
false;
3796 if (
A.checkForAllCallSites(
3797 [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *
this,
3798 true, UsedAssumedInformation))
3799 return Base::updateImpl(
A);
3807 return indicatePessimisticFixpoint();
3813struct AANoAliasCallSiteArgument final : AANoAliasImpl {
3814 AANoAliasCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
3815 : AANoAliasImpl(IRP,
A) {}
3819 bool mayAliasWithArgument(Attributor &
A, AAResults *&AAR,
3820 const AAMemoryBehavior &MemBehaviorAA,
3821 const CallBase &CB,
unsigned OtherArgNo) {
3823 if (this->getCallSiteArgNo() == (
int)OtherArgNo)
3831 auto *CBArgMemBehaviorAA =
A.getAAFor<AAMemoryBehavior>(
3835 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
3836 A.recordDependence(*CBArgMemBehaviorAA, *
this, DepClassTy::OPTIONAL);
3843 if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
3845 A.recordDependence(MemBehaviorAA, *
this, DepClassTy::OPTIONAL);
3846 A.recordDependence(*CBArgMemBehaviorAA, *
this, DepClassTy::OPTIONAL);
3852 AAR =
A.getInfoCache().getAnalysisResultForFunction<AAManager>(
3856 bool IsAliasing = !AAR || !AAR->
isNoAlias(&getAssociatedValue(), ArgOp);
3858 "callsite arguments: "
3859 << getAssociatedValue() <<
" " << *ArgOp <<
" => "
3860 << (IsAliasing ?
"" :
"no-") <<
"alias \n");
3865 bool isKnownNoAliasDueToNoAliasPreservation(
3866 Attributor &
A, AAResults *&AAR,
const AAMemoryBehavior &MemBehaviorAA) {
3879 auto UsePred = [&](
const Use &
U,
bool &Follow) ->
bool {
3895 bool IsKnownNoCapture;
3898 DepClassTy::OPTIONAL, IsKnownNoCapture))
3904 A, *UserI, *getCtxI(), *
this,
nullptr,
3905 [ScopeFn](
const Function &Fn) {
return &Fn != ScopeFn; }))
3920 LLVM_DEBUG(
dbgs() <<
"[AANoAliasCSArg] Unknown user: " << *UserI <<
"\n");
3924 bool IsKnownNoCapture;
3925 const AANoCapture *NoCaptureAA =
nullptr;
3927 A,
this, VIRP, DepClassTy::NONE, IsKnownNoCapture,
false, &NoCaptureAA);
3928 if (!IsAssumedNoCapture &&
3930 if (!
A.checkForAllUses(UsePred, *
this, getAssociatedValue())) {
3932 dbgs() <<
"[AANoAliasCSArg] " << getAssociatedValue()
3933 <<
" cannot be noalias as it is potentially captured\n");
3938 A.recordDependence(*NoCaptureAA, *
this, DepClassTy::OPTIONAL);
3944 for (
unsigned OtherArgNo = 0; OtherArgNo < CB.
arg_size(); OtherArgNo++)
3945 if (mayAliasWithArgument(
A, AAR, MemBehaviorAA, CB, OtherArgNo))
3954 auto *MemBehaviorAA =
3955 A.getAAFor<AAMemoryBehavior>(*
this, getIRPosition(), DepClassTy::NONE);
3957 A.recordDependence(*MemBehaviorAA, *
this, DepClassTy::OPTIONAL);
3958 return ChangeStatus::UNCHANGED;
3961 bool IsKnownNoAlias;
3964 A,
this, VIRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
3966 <<
" is not no-alias at the definition\n");
3967 return indicatePessimisticFixpoint();
3970 AAResults *AAR =
nullptr;
3971 if (MemBehaviorAA &&
3972 isKnownNoAliasDueToNoAliasPreservation(
A, AAR, *MemBehaviorAA)) {
3974 dbgs() <<
"[AANoAlias] No-Alias deduced via no-alias preservation\n");
3975 return ChangeStatus::UNCHANGED;
3978 return indicatePessimisticFixpoint();
3985struct AANoAliasReturned final : AANoAliasImpl {
3986 AANoAliasReturned(
const IRPosition &IRP, Attributor &
A)
3987 : AANoAliasImpl(IRP,
A) {}
3991 auto CheckReturnValue = [&](
Value &RV) ->
bool {
4002 bool IsKnownNoAlias;
4004 A,
this, RVPos, DepClassTy::REQUIRED, IsKnownNoAlias))
4007 bool IsKnownNoCapture;
4008 const AANoCapture *NoCaptureAA =
nullptr;
4010 A,
this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture,
false,
4012 return IsAssumedNoCapture ||
4016 if (!
A.checkForAllReturnedValues(CheckReturnValue, *
this))
4017 return indicatePessimisticFixpoint();
4019 return ChangeStatus::UNCHANGED;
4026struct AANoAliasCallSiteReturned final
4027 : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
4028 AANoAliasCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
4029 : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP,
A) {}
4038struct AAIsDeadValueImpl :
public AAIsDead {
4039 AAIsDeadValueImpl(
const IRPosition &IRP, Attributor &
A) : AAIsDead(IRP,
A) {}
4041 bool isAssumedDead()
const override {
return isAssumed(IS_DEAD); }
4043 bool isKnownDead()
const override {
return isKnown(IS_DEAD); }
4045 bool isAssumedDead(
const BasicBlock *BB)
const override {
return false; }
4047 bool isKnownDead(
const BasicBlock *BB)
const override {
return false; }
4049 bool isAssumedDead(
const Instruction *
I)
const override {
4050 return I == getCtxI() && isAssumedDead();
4053 bool isKnownDead(
const Instruction *
I)
const override {
4054 return isAssumedDead(
I) && isKnownDead();
4057 const std::string getAsStr(Attributor *
A)
const override {
4058 return isAssumedDead() ?
"assumed-dead" :
"assumed-live";
4062 bool areAllUsesAssumedDead(Attributor &
A,
Value &
V) {
4064 if (
V.getType()->isVoidTy() ||
V.use_empty())
4070 if (!
A.isRunOn(*
I->getFunction()))
4072 bool UsedAssumedInformation =
false;
4073 std::optional<Constant *>
C =
4074 A.getAssumedConstant(
V, *
this, UsedAssumedInformation);
4079 auto UsePred = [&](
const Use &
U,
bool &Follow) {
return false; };
4084 return A.checkForAllUses(UsePred, *
this,
V,
false,
4085 DepClassTy::REQUIRED,
4090 bool isAssumedSideEffectFree(Attributor &
A, Instruction *
I) {
4094 if (!
I->isTerminator() && !
I->mayHaveSideEffects())
4103 bool IsKnownNoUnwind;
4105 A,
this, CallIRP, DepClassTy::OPTIONAL, IsKnownNoUnwind))
4113struct AAIsDeadFloating :
public AAIsDeadValueImpl {
4114 AAIsDeadFloating(
const IRPosition &IRP, Attributor &
A)
4115 : AAIsDeadValueImpl(IRP,
A) {}
4118 AAIsDeadValueImpl::initialize(
A);
4121 indicatePessimisticFixpoint();
4126 if (!isAssumedSideEffectFree(
A,
I)) {
4128 indicatePessimisticFixpoint();
4130 removeAssumedBits(HAS_NO_EFFECT);
4134 bool isDeadFence(Attributor &
A, FenceInst &FI) {
4135 const auto *ExecDomainAA =
A.lookupAAFor<AAExecutionDomain>(
4137 if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
4139 A.recordDependence(*ExecDomainAA, *
this, DepClassTy::OPTIONAL);
4143 bool isDeadStore(Attributor &
A, StoreInst &SI,
4144 SmallSetVector<Instruction *, 8> *AssumeOnlyInst =
nullptr) {
4146 if (
SI.isVolatile())
4152 bool UsedAssumedInformation =
false;
4153 if (!AssumeOnlyInst) {
4154 PotentialCopies.clear();
4156 UsedAssumedInformation)) {
4159 <<
"[AAIsDead] Could not determine potential copies of store!\n");
4163 LLVM_DEBUG(
dbgs() <<
"[AAIsDead] Store has " << PotentialCopies.size()
4164 <<
" potential copies.\n");
4166 InformationCache &InfoCache =
A.getInfoCache();
4169 UsedAssumedInformation))
4173 auto &UserI = cast<Instruction>(*U.getUser());
4174 if (InfoCache.isOnlyUsedByAssume(UserI)) {
4176 AssumeOnlyInst->insert(&UserI);
4179 return A.isAssumedDead(U,
this,
nullptr, UsedAssumedInformation);
4185 <<
" is assumed live!\n");
4190 const std::string getAsStr(Attributor *
A)
const override {
4194 return "assumed-dead-store";
4197 return "assumed-dead-fence";
4198 return AAIsDeadValueImpl::getAsStr(
A);
4204 if (!isDeadStore(
A, *SI))
4205 return indicatePessimisticFixpoint();
4207 if (!isDeadFence(
A, *FI))
4208 return indicatePessimisticFixpoint();
4210 if (!isAssumedSideEffectFree(
A,
I))
4211 return indicatePessimisticFixpoint();
4212 if (!areAllUsesAssumedDead(
A, getAssociatedValue()))
4213 return indicatePessimisticFixpoint();
4218 bool isRemovableStore()
const override {
4219 return isAssumed(IS_REMOVABLE) &&
isa<StoreInst>(&getAssociatedValue());
4223 Value &
V = getAssociatedValue();
4230 SmallSetVector<Instruction *, 8> AssumeOnlyInst;
4231 bool IsDead = isDeadStore(
A, *SI, &AssumeOnlyInst);
4234 A.deleteAfterManifest(*
I);
4235 for (
size_t i = 0; i < AssumeOnlyInst.
size(); ++i) {
4237 for (
auto *Usr : AOI->
users())
4239 A.deleteAfterManifest(*AOI);
4245 A.deleteAfterManifest(*FI);
4248 if (isAssumedSideEffectFree(
A,
I) && !
I->isTerminator()) {
4249 A.deleteAfterManifest(*
I);
4256 void trackStatistics()
const override {
4262 SmallSetVector<Value *, 4> PotentialCopies;
4265struct AAIsDeadArgument :
public AAIsDeadFloating {
4266 AAIsDeadArgument(
const IRPosition &IRP, Attributor &
A)
4267 : AAIsDeadFloating(IRP,
A) {}
4270 Argument &Arg = *getAssociatedArgument();
4271 if (
A.isValidFunctionSignatureRewrite(Arg, {}))
4272 if (
A.registerFunctionSignatureRewrite(
4276 return ChangeStatus::CHANGED;
4278 return ChangeStatus::UNCHANGED;
4284struct AAIsDeadCallSiteArgument :
public AAIsDeadValueImpl {
4285 AAIsDeadCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
4286 : AAIsDeadValueImpl(IRP,
A) {}
4289 AAIsDeadValueImpl::initialize(
A);
4291 indicatePessimisticFixpoint();
4299 Argument *Arg = getAssociatedArgument();
4301 return indicatePessimisticFixpoint();
4303 auto *ArgAA =
A.getAAFor<AAIsDead>(*
this, ArgPos, DepClassTy::REQUIRED);
4305 return indicatePessimisticFixpoint();
4313 "Expected undef values to be filtered out!");
4315 if (
A.changeUseAfterManifest(U, UV))
4316 return ChangeStatus::CHANGED;
4317 return ChangeStatus::UNCHANGED;
4323struct AAIsDeadCallSiteReturned :
public AAIsDeadFloating {
4324 AAIsDeadCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
4325 : AAIsDeadFloating(IRP,
A) {}
4327 bool isAssumedDead()
const override {
4328 return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
4332 AAIsDeadFloating::initialize(
A);
4334 indicatePessimisticFixpoint();
4339 IsAssumedSideEffectFree = isAssumedSideEffectFree(
A, getCtxI());
4344 if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(
A, getCtxI())) {
4345 IsAssumedSideEffectFree =
false;
4346 Changed = ChangeStatus::CHANGED;
4348 if (!areAllUsesAssumedDead(
A, getAssociatedValue()))
4349 return indicatePessimisticFixpoint();
4353 void trackStatistics()
const override {
4354 if (IsAssumedSideEffectFree)
4360 const std::string getAsStr(Attributor *
A)
const override {
4361 return isAssumedDead()
4363 : (getAssumed() ?
"assumed-dead-users" :
"assumed-live");
4367 bool IsAssumedSideEffectFree =
true;
4370struct AAIsDeadReturned :
public AAIsDeadValueImpl {
4371 AAIsDeadReturned(
const IRPosition &IRP, Attributor &
A)
4372 : AAIsDeadValueImpl(IRP,
A) {}
4376 bool UsedAssumedInformation =
false;
4377 A.checkForAllInstructions([](Instruction &) {
return true; }, *
this,
4378 {Instruction::Ret}, UsedAssumedInformation);
4380 auto PredForCallSite = [&](AbstractCallSite ACS) {
4381 if (ACS.isCallbackCall() || !ACS.getInstruction())
4383 return areAllUsesAssumedDead(
A, *ACS.getInstruction());
4386 if (!
A.checkForAllCallSites(PredForCallSite, *
this,
true,
4387 UsedAssumedInformation))
4388 return indicatePessimisticFixpoint();
4390 return ChangeStatus::UNCHANGED;
4395 bool AnyChange =
false;
4396 UndefValue &UV = *
UndefValue::get(getAssociatedFunction()->getReturnType());
4403 bool UsedAssumedInformation =
false;
4404 A.checkForAllInstructions(RetInstPred, *
this, {Instruction::Ret},
4405 UsedAssumedInformation);
4406 return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
4412struct AAIsDeadFunction :
public AAIsDead {
4413 AAIsDeadFunction(
const IRPosition &IRP, Attributor &
A) : AAIsDead(IRP,
A) {}
4417 assert(
F &&
"Did expect an anchor function");
4418 if (!isAssumedDeadInternalFunction(
A)) {
4419 ToBeExploredFrom.insert(&
F->getEntryBlock().front());
4420 assumeLive(
A,
F->getEntryBlock());
4424 bool isAssumedDeadInternalFunction(Attributor &
A) {
4425 if (!getAnchorScope()->hasLocalLinkage())
4431 if (!
A.shouldUpdateAA<AAIsDead>(getIRPosition()))
4433 bool UsedAssumedInformation =
false;
4434 return A.checkForAllCallSites([](AbstractCallSite) {
return false; }, *
this,
4435 true, UsedAssumedInformation);
4438 const std::string getAsStr(Attributor *
A)
const override {
4439 return "Live[#BB " + std::to_string(AssumedLiveBlocks.size()) +
"/" +
4440 std::to_string(getAnchorScope()->
size()) +
"][#TBEP " +
4441 std::to_string(ToBeExploredFrom.size()) +
"][#KDE " +
4442 std::to_string(KnownDeadEnds.size()) +
"]";
4446 assert(getState().isValidState() &&
4447 "Attempted to manifest an invalid state!");
4452 if (AssumedLiveBlocks.empty()) {
4453 A.deleteAfterManifest(
F);
4454 return ChangeStatus::CHANGED;
4460 bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(
F);
4462 KnownDeadEnds.set_union(ToBeExploredFrom);
4463 for (
const Instruction *DeadEndI : KnownDeadEnds) {
4467 bool IsKnownNoReturn;
4475 A.registerInvokeWithDeadSuccessor(
const_cast<InvokeInst &
>(*
II));
4477 A.changeToUnreachableAfterManifest(
4478 const_cast<Instruction *
>(DeadEndI->getNextNode()));
4479 HasChanged = ChangeStatus::CHANGED;
4482 STATS_DECL(AAIsDead, BasicBlock,
"Number of dead basic blocks deleted.");
4483 for (BasicBlock &BB :
F)
4484 if (!AssumedLiveBlocks.count(&BB)) {
4485 A.deleteAfterManifest(BB);
4487 HasChanged = ChangeStatus::CHANGED;
4495 bool isEdgeDead(
const BasicBlock *From,
const BasicBlock *To)
const override {
4498 "Used AAIsDead of the wrong function");
4499 return isValidState() && !AssumedLiveEdges.count(std::make_pair(From, To));
4502 void trackStatistics()
const override {}
4505 bool isAssumedDead()
const override {
return false; }
4507 bool isKnownDead()
const override {
return false; }
4509 bool isAssumedDead(
const BasicBlock *BB)
const override {
4511 "BB must be in the same anchor scope function.");
4515 return !AssumedLiveBlocks.count(BB);
4518 bool isKnownDead(
const BasicBlock *BB)
const override {
4519 return getKnown() && isAssumedDead(BB);
4522 bool isAssumedDead(
const Instruction *
I)
const override {
4523 assert(
I->getParent()->getParent() == getAnchorScope() &&
4524 "Instruction must be in the same anchor scope function.");
4531 if (!AssumedLiveBlocks.count(
I->getParent()))
4537 if (KnownDeadEnds.count(PrevI) || ToBeExploredFrom.count(PrevI))
4544 bool isKnownDead(
const Instruction *
I)
const override {
4545 return getKnown() && isAssumedDead(
I);
4550 bool assumeLive(Attributor &
A,
const BasicBlock &BB) {
4551 if (!AssumedLiveBlocks.insert(&BB).second)
4554 if (!
A.isDuringDeduction())
4561 for (
const Instruction &
I : BB)
4564 if (
F->hasLocalLinkage()) {
4566 dbgs() <<
"[AAIsDead] Seeding live internal callee ";
4567 F->printAsOperand(
dbgs(),
false);
4569 BB.getParent()->printAsOperand(
dbgs(),
false);
4572 A.markLiveInternalFunction(*
F);
4579 SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
4582 SmallSetVector<const Instruction *, 8> KnownDeadEnds;
4585 DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
4588 DenseSet<const BasicBlock *> AssumedLiveBlocks;
4592identifyAliveSuccessors(Attributor &
A,
const CallBase &CB,
4593 AbstractAttribute &AA,
4594 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4597 bool IsKnownNoReturn;
4600 return !IsKnownNoReturn;
4609identifyAliveSuccessors(Attributor &
A,
const InvokeInst &
II,
4610 AbstractAttribute &AA,
4611 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4612 bool UsedAssumedInformation =
4618 if (AAIsDeadFunction::mayCatchAsynchronousExceptions(*
II.getFunction())) {
4619 AliveSuccessors.
push_back(&
II.getUnwindDest()->front());
4623 bool IsKnownNoUnwind;
4626 UsedAssumedInformation |= !IsKnownNoUnwind;
4628 AliveSuccessors.
push_back(&
II.getUnwindDest()->front());
4631 return UsedAssumedInformation;
4635identifyAliveSuccessors(Attributor &,
const UncondBrInst &BI,
4636 AbstractAttribute &,
4637 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4643identifyAliveSuccessors(Attributor &
A,
const CondBrInst &BI,
4644 AbstractAttribute &AA,
4645 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4646 bool UsedAssumedInformation =
false;
4647 std::optional<Constant *>
C =
4648 A.getAssumedConstant(*BI.
getCondition(), AA, UsedAssumedInformation);
4658 UsedAssumedInformation =
false;
4660 return UsedAssumedInformation;
4664identifyAliveSuccessors(Attributor &
A,
const SwitchInst &SI,
4665 AbstractAttribute &AA,
4666 SmallVectorImpl<const Instruction *> &AliveSuccessors) {
4667 bool UsedAssumedInformation =
false;
4671 UsedAssumedInformation)) {
4673 for (
const BasicBlock *SuccBB :
successors(
SI.getParent()))
4682 return UsedAssumedInformation;
4685 Type &Ty = *
SI.getCondition()->getType();
4686 SmallPtrSet<ConstantInt *, 8>
Constants;
4687 auto CheckForConstantInt = [&](
Value *
V) {
4696 return CheckForConstantInt(VAC.
getValue());
4698 for (
const BasicBlock *SuccBB :
successors(
SI.getParent()))
4700 return UsedAssumedInformation;
4703 unsigned MatchedCases = 0;
4704 for (
const auto &CaseIt :
SI.cases()) {
4705 if (
Constants.count(CaseIt.getCaseValue())) {
4707 AliveSuccessors.
push_back(&CaseIt.getCaseSuccessor()->front());
4714 AliveSuccessors.
push_back(&
SI.getDefaultDest()->front());
4715 return UsedAssumedInformation;
4721 if (AssumedLiveBlocks.empty()) {
4722 if (isAssumedDeadInternalFunction(
A))
4726 ToBeExploredFrom.insert(&
F->getEntryBlock().front());
4727 assumeLive(
A,
F->getEntryBlock());
4731 LLVM_DEBUG(
dbgs() <<
"[AAIsDead] Live [" << AssumedLiveBlocks.size() <<
"/"
4732 << getAnchorScope()->
size() <<
"] BBs and "
4733 << ToBeExploredFrom.size() <<
" exploration points and "
4734 << KnownDeadEnds.size() <<
" known dead ends\n");
4739 ToBeExploredFrom.end());
4740 decltype(ToBeExploredFrom) NewToBeExploredFrom;
4743 while (!Worklist.
empty()) {
4750 I =
I->getNextNode();
4752 AliveSuccessors.
clear();
4754 bool UsedAssumedInformation =
false;
4755 switch (
I->getOpcode()) {
4759 "Expected non-terminators to be handled already!");
4760 for (
const BasicBlock *SuccBB :
successors(
I->getParent()))
4763 case Instruction::Call:
4765 *
this, AliveSuccessors);
4767 case Instruction::Invoke:
4769 *
this, AliveSuccessors);
4771 case Instruction::UncondBr:
4772 UsedAssumedInformation = identifyAliveSuccessors(
4775 case Instruction::CondBr:
4777 *
this, AliveSuccessors);
4779 case Instruction::Switch:
4781 *
this, AliveSuccessors);
4785 if (UsedAssumedInformation) {
4786 NewToBeExploredFrom.insert(
I);
4787 }
else if (AliveSuccessors.
empty() ||
4788 (
I->isTerminator() &&
4789 AliveSuccessors.
size() <
I->getNumSuccessors())) {
4790 if (KnownDeadEnds.insert(
I))
4795 << AliveSuccessors.
size() <<
" UsedAssumedInformation: "
4796 << UsedAssumedInformation <<
"\n");
4798 for (
const Instruction *AliveSuccessor : AliveSuccessors) {
4799 if (!
I->isTerminator()) {
4800 assert(AliveSuccessors.size() == 1 &&
4801 "Non-terminator expected to have a single successor!");
4805 auto Edge = std::make_pair(
I->getParent(), AliveSuccessor->getParent());
4806 if (AssumedLiveEdges.insert(
Edge).second)
4808 if (assumeLive(
A, *AliveSuccessor->getParent()))
4815 if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
4816 llvm::any_of(NewToBeExploredFrom, [&](
const Instruction *
I) {
4817 return !ToBeExploredFrom.count(I);
4820 ToBeExploredFrom = std::move(NewToBeExploredFrom);
4829 if (ToBeExploredFrom.empty() &&
4830 getAnchorScope()->
size() == AssumedLiveBlocks.size() &&
4831 llvm::all_of(KnownDeadEnds, [](
const Instruction *DeadEndI) {
4832 return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
4834 return indicatePessimisticFixpoint();
4839struct AAIsDeadCallSite final : AAIsDeadFunction {
4840 AAIsDeadCallSite(
const IRPosition &IRP, Attributor &
A)
4841 : AAIsDeadFunction(IRP,
A) {}
4849 "supported for call sites yet!");
4853 return indicatePessimisticFixpoint();
4856 void trackStatistics()
const override {}
4863struct AADereferenceableImpl : AADereferenceable {
4864 AADereferenceableImpl(
const IRPosition &IRP, Attributor &
A)
4865 : AADereferenceable(IRP,
A) {}
4866 using StateType = DerefState;
4869 Value &
V = *getAssociatedValue().stripPointerCasts();
4871 A.getAttrs(getIRPosition(),
4872 {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
4875 takeKnownDerefBytesMaximum(Attr.getValueAsInt());
4878 bool IsKnownNonNull;
4880 A,
this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNonNull);
4883 takeKnownDerefBytesMaximum(
V.getPointerDereferenceableBytes(
4884 A.getDataLayout(), CanBeNull,
nullptr));
4886 if (Instruction *CtxI = getCtxI())
4887 followUsesInMBEC(*
this,
A, getState(), *CtxI);
4890 StateType &getState()
override {
return *
this; }
4891 const StateType &getState()
const override {
return *
this; }
4894 void addAccessedBytesForUse(Attributor &
A,
const Use *U,
const Instruction *
I,
4895 DerefState &State) {
4896 const Value *UseV =
U->get();
4901 if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
I->isVolatile())
4906 Loc->Ptr,
Offset,
A.getDataLayout(),
true);
4907 if (
Base &&
Base == &getAssociatedValue())
4908 State.addAccessedBytes(
Offset, Loc->Size.getValue());
4912 bool followUseInMBEC(Attributor &
A,
const Use *U,
const Instruction *
I,
4913 AADereferenceable::StateType &State) {
4914 bool IsNonNull =
false;
4915 bool TrackUse =
false;
4916 int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
4917 A, *
this, getAssociatedValue(), U,
I, IsNonNull, TrackUse);
4918 LLVM_DEBUG(
dbgs() <<
"[AADereferenceable] Deref bytes: " << DerefBytes
4919 <<
" for instruction " << *
I <<
"\n");
4921 addAccessedBytesForUse(
A, U,
I, State);
4922 State.takeKnownDerefBytesMaximum(DerefBytes);
4928 bool IsKnownNonNull;
4930 A,
this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4931 if (IsAssumedNonNull &&
4932 A.hasAttr(getIRPosition(), Attribute::DereferenceableOrNull)) {
4933 A.removeAttrs(getIRPosition(), {Attribute::DereferenceableOrNull});
4934 return ChangeStatus::CHANGED;
4939 void getDeducedAttributes(Attributor &
A, LLVMContext &Ctx,
4940 SmallVectorImpl<Attribute> &Attrs)
const override {
4942 bool IsKnownNonNull;
4944 A,
this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4945 if (IsAssumedNonNull)
4946 Attrs.emplace_back(Attribute::getWithDereferenceableBytes(
4947 Ctx, getAssumedDereferenceableBytes()));
4949 Attrs.emplace_back(Attribute::getWithDereferenceableOrNullBytes(
4950 Ctx, getAssumedDereferenceableBytes()));
4953 const std::string getAsStr(Attributor *
A)
const override {
4954 if (!getAssumedDereferenceableBytes())
4955 return "unknown-dereferenceable";
4956 bool IsKnownNonNull;
4957 bool IsAssumedNonNull =
false;
4960 *
A,
this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
4961 return std::string(
"dereferenceable") +
4962 (IsAssumedNonNull ?
"" :
"_or_null") +
4963 (isAssumedGlobal() ?
"_globally" :
"") +
"<" +
4964 std::to_string(getKnownDereferenceableBytes()) +
"-" +
4965 std::to_string(getAssumedDereferenceableBytes()) +
">" +
4966 (!
A ?
" [non-null is unknown]" :
"");
4971struct AADereferenceableFloating : AADereferenceableImpl {
4972 AADereferenceableFloating(
const IRPosition &IRP, Attributor &
A)
4973 : AADereferenceableImpl(IRP,
A) {}
4977 bool UsedAssumedInformation =
false;
4979 if (!
A.getAssumedSimplifiedValues(getIRPosition(), *
this,
Values,
4981 Values.push_back({getAssociatedValue(), getCtxI()});
4984 Stripped =
Values.size() != 1 ||
4985 Values.front().getValue() != &getAssociatedValue();
4988 const DataLayout &
DL =
A.getDataLayout();
4991 auto VisitValueCB = [&](
const Value &
V) ->
bool {
4993 DL.getIndexSizeInBits(
V.getType()->getPointerAddressSpace());
4994 APInt
Offset(IdxWidth, 0);
4999 const auto *AA =
A.getAAFor<AADereferenceable>(
5001 int64_t DerefBytes = 0;
5002 if (!AA || (!Stripped &&
this == AA)) {
5006 DerefBytes =
Base->getPointerDereferenceableBytes(
5007 DL, CanBeNull,
nullptr);
5008 T.GlobalState.indicatePessimisticFixpoint();
5011 DerefBytes =
DS.DerefBytesState.getAssumed();
5012 T.GlobalState &=
DS.GlobalState;
5018 int64_t OffsetSExt =
Offset.getSExtValue();
5022 T.takeAssumedDerefBytesMinimum(
5023 std::max(int64_t(0), DerefBytes - OffsetSExt));
5028 T.takeKnownDerefBytesMaximum(
5029 std::max(int64_t(0), DerefBytes - OffsetSExt));
5030 T.indicatePessimisticFixpoint();
5031 }
else if (OffsetSExt > 0) {
5037 T.indicatePessimisticFixpoint();
5041 return T.isValidState();
5044 for (
const auto &VAC :
Values)
5045 if (!VisitValueCB(*VAC.
getValue()))
5046 return indicatePessimisticFixpoint();
5051 void trackStatistics()
const override {
5057struct AADereferenceableReturned final
5058 : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
5060 AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
5061 AADereferenceableReturned(
const IRPosition &IRP, Attributor &
A)
5064 void trackStatistics()
const override {
5070struct AADereferenceableArgument final
5071 : AAArgumentFromCallSiteArguments<AADereferenceable,
5072 AADereferenceableImpl> {
5074 AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
5075 AADereferenceableArgument(
const IRPosition &IRP, Attributor &
A)
5078 void trackStatistics()
const override {
5084struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
5085 AADereferenceableCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
5086 : AADereferenceableFloating(IRP,
A) {}
5088 void trackStatistics()
const override {
5094struct AADereferenceableCallSiteReturned final
5095 : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
5096 using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
5097 AADereferenceableCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
5100 void trackStatistics()
const override {
5110static unsigned getKnownAlignForUse(Attributor &
A, AAAlign &QueryingAA,
5111 Value &AssociatedValue,
const Use *U,
5112 const Instruction *
I,
bool &TrackUse) {
5121 if (
GEP->hasAllConstantIndices())
5126 switch (
II->getIntrinsicID()) {
5127 case Intrinsic::ptrmask: {
5129 const auto *ConstVals =
A.getAAFor<AAPotentialConstantValues>(
5131 const auto *AlignAA =
A.getAAFor<AAAlign>(
5133 if (ConstVals && ConstVals->isValidState() && ConstVals->isAtFixpoint()) {
5134 unsigned ShiftValue = std::min(ConstVals->getAssumedMinTrailingZeros(),
5137 if (ConstAlign >= AlignAA->getKnownAlign())
5138 return Align(1).value();
5141 return AlignAA->getKnownAlign().
value();
5144 case Intrinsic::amdgcn_make_buffer_rsrc: {
5145 const auto *AlignAA =
A.getAAFor<AAAlign>(
5148 return AlignAA->getKnownAlign().
value();
5166 MA = MaybeAlign(AlignAA->getKnownAlign());
5169 const DataLayout &
DL =
A.getDataLayout();
5170 const Value *UseV =
U->get();
5172 if (
SI->getPointerOperand() == UseV)
5173 MA =
SI->getAlign();
5175 if (LI->getPointerOperand() == UseV)
5176 MA = LI->getAlign();
5178 if (AI->getPointerOperand() == UseV)
5179 MA = AI->getAlign();
5181 if (AI->getPointerOperand() == UseV)
5182 MA = AI->getAlign();
5192 if (
Base == &AssociatedValue) {
5197 uint32_t
gcd = std::gcd(uint32_t(
abs((int32_t)
Offset)), Alignment);
5205struct AAAlignImpl : AAAlign {
5206 AAAlignImpl(
const IRPosition &IRP, Attributor &
A) : AAAlign(IRP,
A) {}
5210 A.getAttrs(getIRPosition(), {Attribute::Alignment},
Attrs);
5212 takeKnownMaximum(Attr.getValueAsInt());
5214 Value &
V = *getAssociatedValue().stripPointerCasts();
5215 takeKnownMaximum(
V.getPointerAlignment(
A.getDataLayout()).value());
5217 if (Instruction *CtxI = getCtxI())
5218 followUsesInMBEC(*
this,
A, getState(), *CtxI);
5225 Value &AssociatedValue = getAssociatedValue();
5227 return ChangeStatus::UNCHANGED;
5229 for (
const Use &U : AssociatedValue.
uses()) {
5231 if (
SI->getPointerOperand() == &AssociatedValue)
5232 if (
SI->getAlign() < getAssumedAlign()) {
5234 "Number of times alignment added to a store");
5235 SI->setAlignment(getAssumedAlign());
5236 InstrChanged = ChangeStatus::CHANGED;
5239 if (LI->getPointerOperand() == &AssociatedValue)
5240 if (LI->getAlign() < getAssumedAlign()) {
5241 LI->setAlignment(getAssumedAlign());
5243 "Number of times alignment added to a load");
5244 InstrChanged = ChangeStatus::CHANGED;
5247 if (RMW->getPointerOperand() == &AssociatedValue) {
5248 if (RMW->getAlign() < getAssumedAlign()) {
5250 "Number of times alignment added to atomicrmw");
5252 RMW->setAlignment(getAssumedAlign());
5253 InstrChanged = ChangeStatus::CHANGED;
5257 if (CAS->getPointerOperand() == &AssociatedValue) {
5258 if (CAS->getAlign() < getAssumedAlign()) {
5260 "Number of times alignment added to cmpxchg");
5261 CAS->setAlignment(getAssumedAlign());
5262 InstrChanged = ChangeStatus::CHANGED;
5270 Align InheritAlign =
5271 getAssociatedValue().getPointerAlignment(
A.getDataLayout());
5272 if (InheritAlign >= getAssumedAlign())
5273 return InstrChanged;
5274 return Changed | InstrChanged;
5281 void getDeducedAttributes(Attributor &
A, LLVMContext &Ctx,
5282 SmallVectorImpl<Attribute> &Attrs)
const override {
5283 if (getAssumedAlign() > 1)
5285 Attribute::getWithAlignment(Ctx,
Align(getAssumedAlign())));
5289 bool followUseInMBEC(Attributor &
A,
const Use *U,
const Instruction *
I,
5290 AAAlign::StateType &State) {
5291 bool TrackUse =
false;
5293 unsigned int KnownAlign =
5294 getKnownAlignForUse(
A, *
this, getAssociatedValue(), U,
I, TrackUse);
5295 State.takeKnownMaximum(KnownAlign);
5300 const std::string getAsStr(Attributor *
A)
const override {
5301 return "align<" + std::to_string(getKnownAlign().value()) +
"-" +
5302 std::to_string(getAssumedAlign().value()) +
">";
5307struct AAAlignFloating : AAAlignImpl {
5308 AAAlignFloating(
const IRPosition &IRP, Attributor &
A) : AAAlignImpl(IRP,
A) {}
5311 const DataLayout &
DL =
A.getDataLayout();
5314 bool UsedAssumedInformation =
false;
5316 if (!
A.getAssumedSimplifiedValues(getIRPosition(), *
this,
Values,
5318 Values.push_back({getAssociatedValue(), getCtxI()});
5321 Stripped =
Values.size() != 1 ||
5322 Values.front().getValue() != &getAssociatedValue();
5326 auto VisitValueCB = [&](
Value &
V) ->
bool {
5330 DepClassTy::REQUIRED);
5331 if (!AA || (!Stripped &&
this == AA)) {
5349 T.takeKnownMaximum(Alignment);
5350 T.indicatePessimisticFixpoint();
5353 const AAAlign::StateType &
DS = AA->
getState();
5356 return T.isValidState();
5359 for (
const auto &VAC :
Values) {
5360 if (!VisitValueCB(*VAC.
getValue()))
5361 return indicatePessimisticFixpoint();
5373struct AAAlignReturned final
5374 : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
5375 using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
5376 AAAlignReturned(
const IRPosition &IRP, Attributor &
A) :
Base(IRP,
A) {}
5382struct AAAlignArgument final
5383 : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
5384 using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
5385 AAAlignArgument(
const IRPosition &IRP, Attributor &
A) :
Base(IRP,
A) {}
5391 if (
A.getInfoCache().isInvolvedInMustTailCall(*getAssociatedArgument()))
5392 return ChangeStatus::UNCHANGED;
5393 return Base::manifest(
A);
5399struct AAAlignCallSiteArgument final : AAAlignFloating {
5400 AAAlignCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
5401 : AAAlignFloating(IRP,
A) {}
5407 if (Argument *Arg = getAssociatedArgument())
5408 if (
A.getInfoCache().isInvolvedInMustTailCall(*Arg))
5409 return ChangeStatus::UNCHANGED;
5411 Align InheritAlign =
5412 getAssociatedValue().getPointerAlignment(
A.getDataLayout());
5413 if (InheritAlign >= getAssumedAlign())
5414 Changed = ChangeStatus::UNCHANGED;
5420 if (Argument *Arg = getAssociatedArgument()) {
5423 const auto *ArgAlignAA =
A.getAAFor<AAAlign>(
5426 takeKnownMaximum(ArgAlignAA->getKnownAlign().value());
5435struct AAAlignCallSiteReturned final
5436 : AACalleeToCallSite<AAAlign, AAAlignImpl> {
5437 using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
5438 AAAlignCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
5444 switch (
II->getIntrinsicID()) {
5445 case Intrinsic::ptrmask: {
5449 const auto *ConstVals =
A.getAAFor<AAPotentialConstantValues>(
5451 if (ConstVals && ConstVals->isValidState()) {
5452 unsigned ShiftValue =
5453 std::min(ConstVals->getAssumedMinTrailingZeros(),
5454 Value::MaxAlignmentExponent);
5459 const auto *AlignAA =
5461 DepClassTy::REQUIRED);
5463 Alignment = std::max(AlignAA->getAssumedAlign(), Alignment);
5470 std::min(this->getAssumedAlign(), Alignment).value());
5476 case Intrinsic::amdgcn_make_buffer_rsrc: {
5477 const auto *AlignAA =
5479 DepClassTy::REQUIRED);
5482 this->getState(), AlignAA->getAssumedAlign().
value());
5489 return Base::updateImpl(
A);
5497struct AANoReturnImpl :
public AANoReturn {
5498 AANoReturnImpl(
const IRPosition &IRP, Attributor &
A) : AANoReturn(IRP,
A) {}
5503 A,
nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
5507 const std::string getAsStr(Attributor *
A)
const override {
5508 return getAssumed() ?
"noreturn" :
"may-return";
5512 auto CheckForNoReturn = [](
Instruction &) {
return false; };
5513 bool UsedAssumedInformation =
false;
5514 if (!
A.checkForAllInstructions(CheckForNoReturn, *
this,
5515 {(unsigned)Instruction::Ret},
5516 UsedAssumedInformation))
5517 return indicatePessimisticFixpoint();
5518 return ChangeStatus::UNCHANGED;
5522struct AANoReturnFunction final : AANoReturnImpl {
5523 AANoReturnFunction(
const IRPosition &IRP, Attributor &
A)
5524 : AANoReturnImpl(IRP,
A) {}
5530struct AANoReturnCallSite final
5531 : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
5532 AANoReturnCallSite(
const IRPosition &IRP, Attributor &
A)
5533 : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP,
A) {}
5543struct AAInstanceInfoImpl :
public AAInstanceInfo {
5544 AAInstanceInfoImpl(
const IRPosition &IRP, Attributor &
A)
5545 : AAInstanceInfo(IRP,
A) {}
5548 Value &
V = getAssociatedValue();
5550 if (
C->isThreadDependent())
5551 indicatePessimisticFixpoint();
5553 indicateOptimisticFixpoint();
5559 indicateOptimisticFixpoint();
5564 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
5567 indicatePessimisticFixpoint();
5576 Value &
V = getAssociatedValue();
5579 Scope =
I->getFunction();
5582 if (!
Scope->hasLocalLinkage())
5586 return indicateOptimisticFixpoint();
5588 bool IsKnownNoRecurse;
5594 auto UsePred = [&](
const Use &
U,
bool &Follow) {
5609 if (!Callee || !
Callee->hasLocalLinkage())
5613 const auto *ArgInstanceInfoAA =
A.getAAFor<AAInstanceInfo>(
5615 DepClassTy::OPTIONAL);
5616 if (!ArgInstanceInfoAA ||
5617 !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
5622 A, *CB, *Scope, *
this,
nullptr,
5630 auto EquivalentUseCB = [&](
const Use &OldU,
const Use &NewU) {
5632 auto *Ptr =
SI->getPointerOperand()->stripPointerCasts();
5640 if (!
A.checkForAllUses(UsePred, *
this,
V,
true,
5641 DepClassTy::OPTIONAL,
5642 true, EquivalentUseCB))
5643 return indicatePessimisticFixpoint();
5648 const std::string getAsStr(Attributor *
A)
const override {
5649 return isAssumedUniqueForAnalysis() ?
"<unique [fAa]>" :
"<unknown>";
5652 void trackStatistics()
const override {}
5656struct AAInstanceInfoFloating : AAInstanceInfoImpl {
5657 AAInstanceInfoFloating(
const IRPosition &IRP, Attributor &
A)
5658 : AAInstanceInfoImpl(IRP,
A) {}
5662struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
5663 AAInstanceInfoArgument(
const IRPosition &IRP, Attributor &
A)
5664 : AAInstanceInfoFloating(IRP,
A) {}
5668struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
5669 AAInstanceInfoCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
5670 : AAInstanceInfoImpl(IRP,
A) {}
5677 Argument *Arg = getAssociatedArgument();
5679 return indicatePessimisticFixpoint();
5682 A.getAAFor<AAInstanceInfo>(*
this, ArgPos, DepClassTy::REQUIRED);
5684 return indicatePessimisticFixpoint();
5690struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
5691 AAInstanceInfoReturned(
const IRPosition &IRP, Attributor &
A)
5692 : AAInstanceInfoImpl(IRP,
A) {
5706struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
5707 AAInstanceInfoCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
5708 : AAInstanceInfoFloating(IRP,
A) {}
5715 bool IgnoreSubsumingPositions) {
5716 assert(ImpliedAttributeKind == Attribute::Captures &&
5717 "Unexpected attribute kind");
5731 A.getAttrs(IRP, {Attribute::Captures}, Attrs,
5741 {Attribute::Captures, Attribute::ByVal}, Attrs,
5780 bool NoThrow =
F.doesNotThrow();
5781 bool WillReturn =
F.willReturn();
5782 bool IsVoidReturn =
F.getReturnType()->isVoidTy();
5783 if (
ReadOnly && NoThrow && WillReturn && IsVoidReturn) {
5796 if (NoThrow && IsVoidReturn)
5801 if (!NoThrow || ArgNo < 0 ||
5802 !
F.getAttributes().hasAttrSomewhere(Attribute::Returned))
5805 for (
unsigned U = 0, E =
F.arg_size(); U < E; ++U)
5806 if (
F.hasParamAttribute(U, Attribute::Returned)) {
5807 if (U ==
unsigned(ArgNo))
5831 void getDeducedAttributes(Attributor &
A, LLVMContext &Ctx,
5832 SmallVectorImpl<Attribute> &Attrs)
const override {
5833 if (!isAssumedNoCaptureMaybeReturned())
5836 if (isArgumentPosition()) {
5837 if (isAssumedNoCapture())
5838 Attrs.emplace_back(Attribute::get(Ctx, Attribute::Captures));
5839 else if (
A.Opts.attributor_manifest_internal)
5840 Attrs.emplace_back(Attribute::get(Ctx,
"no-capture-maybe-returned"));
5844 const std::string getAsStr(Attributor *
A)
const override {
5845 if (isKnownNoCapture())
5846 return "known not-captured";
5847 if (isAssumedNoCapture())
5848 return "assumed not-captured";
5849 if (isKnownNoCaptureMaybeReturned())
5850 return "known not-captured-maybe-returned";
5851 if (isAssumedNoCaptureMaybeReturned())
5852 return "assumed not-captured-maybe-returned";
5853 return "assumed-captured";
5858 bool checkUse(Attributor &
A, AANoCapture::StateType &State,
const Use &U,
5861 LLVM_DEBUG(
dbgs() <<
"[AANoCapture] Check use: " << *
U.get() <<
" in "
5867 return isCapturedIn(State,
true,
true,
5874 return isCapturedIn(State,
true,
true,
5880 return isCapturedIn(State,
false,
false,
5882 return isCapturedIn(State,
true,
true,
5890 return isCapturedIn(State,
true,
true,
5897 bool IsKnownNoCapture;
5898 const AANoCapture *ArgNoCaptureAA =
nullptr;
5900 A,
this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture,
false,
5902 if (IsAssumedNoCapture)
5903 return isCapturedIn(State,
false,
false,
5907 return isCapturedIn(State,
false,
false,
5912 return isCapturedIn(State,
true,
true,
5919 static bool isCapturedIn(AANoCapture::StateType &State,
bool CapturedInMem,
5920 bool CapturedInInt,
bool CapturedInRet) {
5921 LLVM_DEBUG(
dbgs() <<
" - captures [Mem " << CapturedInMem <<
"|Int "
5922 << CapturedInInt <<
"|Ret " << CapturedInRet <<
"]\n");
5934 const IRPosition &IRP = getIRPosition();
5938 return indicatePessimisticFixpoint();
5945 return indicatePessimisticFixpoint();
5953 T.addKnownBits(NOT_CAPTURED_IN_MEM);
5955 addKnownBits(NOT_CAPTURED_IN_MEM);
5962 auto CheckReturnedArgs = [&](
bool &UsedAssumedInformation) {
5966 UsedAssumedInformation))
5968 bool SeenConstant =
false;
5969 for (
const AA::ValueAndContext &VAC :
Values) {
5973 SeenConstant =
true;
5975 VAC.
getValue() == getAssociatedArgument())
5981 bool IsKnownNoUnwind;
5984 bool IsVoidTy =
F->getReturnType()->isVoidTy();
5985 bool UsedAssumedInformation =
false;
5986 if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
5987 T.addKnownBits(NOT_CAPTURED_IN_RET);
5990 bool IsKnownWillReturn;
5993 if (IsAssumedWillReturn &&
T.isKnown(NOT_CAPTURED_IN_MEM))
5995 if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
5996 addKnownBits(NOT_CAPTURED_IN_RET);
5997 if (IsKnownWillReturn && isKnown(NOT_CAPTURED_IN_MEM))
5998 return indicateOptimisticFixpoint();
6003 auto UseCheck = [&](
const Use &
U,
bool &Follow) ->
bool {
6012 return checkUse(
A,
T, U, Follow);
6015 if (!
A.checkForAllUses(UseCheck, *
this, *
V))
6016 return indicatePessimisticFixpoint();
6019 auto Assumed = S.getAssumed();
6020 S.intersectAssumedBits(
T.getAssumed());
6021 if (!isAssumedNoCaptureMaybeReturned())
6022 return indicatePessimisticFixpoint();
6028struct AANoCaptureArgument final : AANoCaptureImpl {
6029 AANoCaptureArgument(
const IRPosition &IRP, Attributor &
A)
6030 : AANoCaptureImpl(IRP,
A) {}
6036struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
6037 AANoCaptureCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
6038 : AANoCaptureImpl(IRP,
A) {}
6045 Argument *Arg = getAssociatedArgument();
6047 return indicatePessimisticFixpoint();
6049 bool IsKnownNoCapture;
6050 const AANoCapture *ArgAA =
nullptr;
6052 A,
this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture,
false,
6054 return ChangeStatus::UNCHANGED;
6056 return indicatePessimisticFixpoint();
6060 void trackStatistics()
const override {
6066struct AANoCaptureFloating final : AANoCaptureImpl {
6067 AANoCaptureFloating(
const IRPosition &IRP, Attributor &
A)
6068 : AANoCaptureImpl(IRP,
A) {}
6070 void trackStatistics()
const override {
6076struct AANoCaptureReturned final : AANoCaptureImpl {
6077 AANoCaptureReturned(
const IRPosition &IRP, Attributor &
A)
6078 : AANoCaptureImpl(IRP,
A) {
6090 void trackStatistics()
const override {}
6094struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
6095 AANoCaptureCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
6096 : AANoCaptureImpl(IRP,
A) {}
6101 determineFunctionCaptureCapabilities(getIRPosition(), *
F, *
this);
6104 void trackStatistics()
const override {
6121 dbgs() <<
"[ValueSimplify] is assumed to be "
6124 dbgs() <<
"[ValueSimplify] is assumed to be <none>\n";
6135 if (getAssociatedValue().
getType()->isVoidTy())
6136 indicatePessimisticFixpoint();
6137 if (
A.hasSimplificationCallback(getIRPosition()))
6138 indicatePessimisticFixpoint();
6141 const std::string getAsStr(Attributor *
A)
const override {
6143 dbgs() <<
"SAV: " << (bool)SimplifiedAssociatedValue <<
" ";
6144 if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
6145 dbgs() <<
"SAV: " << **SimplifiedAssociatedValue <<
" ";
6147 return isValidState() ? (isAtFixpoint() ?
"simplified" :
"maybe-simple")
6151 void trackStatistics()
const override {}
6153 std::optional<Value *>
6154 getAssumedSimplifiedValue(Attributor &
A)
const override {
6155 return SimplifiedAssociatedValue;
6162 static Value *ensureType(Attributor &
A,
Value &
V,
Type &Ty, Instruction *CtxI,
6166 if (CtxI &&
V.getType()->canLosslesslyBitCastTo(&Ty))
6168 : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6177 static Value *reproduceInst(Attributor &
A,
6178 const AbstractAttribute &QueryingAA,
6179 Instruction &
I,
Type &Ty, Instruction *CtxI,
6181 assert(CtxI &&
"Cannot reproduce an instruction without context!");
6182 if (
Check && (
I.mayReadFromMemory() ||
6187 Value *NewOp = reproduceValue(
A, QueryingAA, *
Op, Ty, CtxI,
Check, VMap);
6189 assert(
Check &&
"Manifest of new value unexpectedly failed!");
6211 static Value *reproduceValue(Attributor &
A,
6212 const AbstractAttribute &QueryingAA,
Value &
V,
6213 Type &Ty, Instruction *CtxI,
bool Check,
6215 if (
const auto &NewV = VMap.
lookup(&
V))
6217 bool UsedAssumedInformation =
false;
6218 std::optional<Value *> SimpleV =
A.getAssumedSimplified(
6220 if (!SimpleV.has_value())
6224 EffectiveV = *SimpleV;
6229 return ensureType(
A, *EffectiveV, Ty, CtxI,
Check);
6231 if (
Value *NewV = reproduceInst(
A, QueryingAA, *
I, Ty, CtxI,
Check, VMap))
6232 return ensureType(
A, *NewV, Ty, CtxI,
Check);
6238 Value *manifestReplacementValue(Attributor &
A, Instruction *CtxI)
const {
6239 Value *NewV = SimplifiedAssociatedValue
6240 ? *SimplifiedAssociatedValue
6242 if (NewV && NewV != &getAssociatedValue()) {
6246 if (reproduceValue(
A, *
this, *NewV, *getAssociatedType(), CtxI,
6248 return reproduceValue(
A, *
this, *NewV, *getAssociatedType(), CtxI,
6256 bool checkAndUpdate(Attributor &
A,
const AbstractAttribute &QueryingAA,
6257 const IRPosition &IRP,
bool Simplify =
true) {
6258 bool UsedAssumedInformation =
false;
6261 QueryingValueSimplified =
A.getAssumedSimplified(
6263 return unionAssumed(QueryingValueSimplified);
6267 template <
typename AAType>
bool askSimplifiedValueFor(Attributor &
A) {
6268 if (!getAssociatedValue().
getType()->isIntegerTy())
6273 A.getAAFor<AAType>(*
this, getIRPosition(), DepClassTy::NONE);
6277 std::optional<Constant *> COpt = AA->getAssumedConstant(
A);
6280 SimplifiedAssociatedValue = std::nullopt;
6281 A.recordDependence(*AA, *
this, DepClassTy::OPTIONAL);
6284 if (
auto *
C = *COpt) {
6285 SimplifiedAssociatedValue =
C;
6286 A.recordDependence(*AA, *
this, DepClassTy::OPTIONAL);
6292 bool askSimplifiedValueForOtherAAs(Attributor &
A) {
6293 if (askSimplifiedValueFor<AAValueConstantRange>(
A))
6295 if (askSimplifiedValueFor<AAPotentialConstantValues>(
A))
6302 for (
auto &U : getAssociatedValue().uses()) {
6307 IP =
PHI->getIncomingBlock(U)->getTerminator();
6308 if (
auto *NewV = manifestReplacementValue(
A, IP)) {
6310 <<
" -> " << *NewV <<
" :: " << *
this <<
"\n");
6311 if (
A.changeUseAfterManifest(U, *NewV))
6312 Changed = ChangeStatus::CHANGED;
6316 return Changed | AAValueSimplify::manifest(
A);
6320 SimplifiedAssociatedValue = &getAssociatedValue();
6321 return AAValueSimplify::indicatePessimisticFixpoint();
6325struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
6326 AAValueSimplifyArgument(
const IRPosition &IRP, Attributor &
A)
6327 : AAValueSimplifyImpl(IRP,
A) {}
6330 AAValueSimplifyImpl::initialize(
A);
6331 if (
A.hasAttr(getIRPosition(),
6332 {Attribute::InAlloca, Attribute::Preallocated,
6333 Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
6335 indicatePessimisticFixpoint();
6341 Argument *Arg = getAssociatedArgument();
6347 return indicatePessimisticFixpoint();
6350 auto Before = SimplifiedAssociatedValue;
6352 auto PredForCallSite = [&](AbstractCallSite ACS) {
6353 const IRPosition &ACSArgPos =
6364 bool UsedAssumedInformation =
false;
6365 std::optional<Constant *> SimpleArgOp =
6366 A.getAssumedConstant(ACSArgPos, *
this, UsedAssumedInformation);
6373 return unionAssumed(*SimpleArgOp);
6378 bool UsedAssumedInformation =
false;
6379 if (hasCallBaseContext() &&
6380 getCallBaseContext()->getCalledOperand() == Arg->
getParent())
6382 AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
6384 Success =
A.checkForAllCallSites(PredForCallSite, *
this,
true,
6385 UsedAssumedInformation);
6388 if (!askSimplifiedValueForOtherAAs(
A))
6389 return indicatePessimisticFixpoint();
6392 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6393 : ChangeStatus ::CHANGED;
6396 void trackStatistics()
const override {
6401struct AAValueSimplifyReturned : AAValueSimplifyImpl {
6402 AAValueSimplifyReturned(
const IRPosition &IRP, Attributor &
A)
6403 : AAValueSimplifyImpl(IRP,
A) {}
6405 std::optional<Value *>
6406 getAssumedSimplifiedValue(Attributor &
A)
const override {
6407 if (!isValidState())
6409 return SimplifiedAssociatedValue;
6413 auto Before = SimplifiedAssociatedValue;
6417 return checkAndUpdate(
6422 bool UsedAssumedInformation =
false;
6423 if (!
A.checkForAllInstructions(ReturnInstCB, *
this, {Instruction::Ret},
6424 UsedAssumedInformation))
6425 if (!askSimplifiedValueForOtherAAs(
A))
6426 return indicatePessimisticFixpoint();
6429 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6430 : ChangeStatus ::CHANGED;
6436 return ChangeStatus::UNCHANGED;
6439 void trackStatistics()
const override {
6444struct AAValueSimplifyFloating : AAValueSimplifyImpl {
6445 AAValueSimplifyFloating(
const IRPosition &IRP, Attributor &
A)
6446 : AAValueSimplifyImpl(IRP,
A) {}
6449 AAValueSimplifyImpl::initialize(
A);
6450 Value &
V = getAnchorValue();
6454 indicatePessimisticFixpoint();
6458 auto Before = SimplifiedAssociatedValue;
6459 if (!askSimplifiedValueForOtherAAs(
A))
6460 return indicatePessimisticFixpoint();
6463 return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
6464 : ChangeStatus ::CHANGED;
6467 void trackStatistics()
const override {
6472struct AAValueSimplifyFunction : AAValueSimplifyImpl {
6473 AAValueSimplifyFunction(
const IRPosition &IRP, Attributor &
A)
6474 : AAValueSimplifyImpl(IRP,
A) {}
6477 SimplifiedAssociatedValue =
nullptr;
6478 indicateOptimisticFixpoint();
6482 "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
6484 void trackStatistics()
const override {
6489struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
6490 AAValueSimplifyCallSite(
const IRPosition &IRP, Attributor &
A)
6491 : AAValueSimplifyFunction(IRP,
A) {}
6492 void trackStatistics()
const override {
6497struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
6498 AAValueSimplifyCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
6499 : AAValueSimplifyImpl(IRP,
A) {}
6502 AAValueSimplifyImpl::initialize(
A);
6503 Function *Fn = getAssociatedFunction();
6504 assert(Fn &&
"Did expect an associted function");
6505 for (Argument &Arg : Fn->
args()) {
6510 checkAndUpdate(
A, *
this, IRP))
6511 indicateOptimisticFixpoint();
6513 indicatePessimisticFixpoint();
6520 return indicatePessimisticFixpoint();
6523 void trackStatistics()
const override {
6528struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
6529 AAValueSimplifyCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
6530 : AAValueSimplifyFloating(IRP,
A) {}
6535 auto *FloatAA =
A.lookupAAFor<AAValueSimplify>(
6537 if (FloatAA && FloatAA->getState().isValidState())
6540 if (
auto *NewV = manifestReplacementValue(
A, getCtxI())) {
6542 ->getArgOperandUse(getCallSiteArgNo());
6543 if (
A.changeUseAfterManifest(U, *NewV))
6544 Changed = ChangeStatus::CHANGED;
6547 return Changed | AAValueSimplify::manifest(
A);
6550 void trackStatistics()
const override {
6561 bool UsedAssumedInformation =
false;
6562 return A.checkForAllInstructions(
6565 if (CI.isTailCall() && !CI.isMustTailCall())
6569 QueryingAA, {Instruction::Call}, UsedAssumedInformation);
6574struct AAHeapToStackFunction final :
public AAHeapToStack {
6576 static bool isGlobalizedLocal(
const CallBase &CB) {
6578 return A.
isValid() &&
A.getValueAsString() ==
"__kmpc_alloc_shared";
6581 struct AllocationInfo {
6586 bool IsGlobalizedLocal =
false;
6593 } Status = STACK_DUE_TO_USE;
6597 bool HasPotentiallyFreeingUnknownUses =
false;
6601 bool MoveAllocaIntoEntry =
true;
6604 SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
6607 struct DeallocationInfo {
6615 bool MightFreeUnknownObjects =
false;
6618 SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
6621 AAHeapToStackFunction(
const IRPosition &IRP, Attributor &
A)
6622 : AAHeapToStack(IRP,
A) {}
6624 ~AAHeapToStackFunction()
override {
6627 for (
auto &It : AllocationInfos)
6628 It.second->~AllocationInfo();
6629 for (
auto &It : DeallocationInfos)
6630 It.second->~DeallocationInfo();
6634 AAHeapToStack::initialize(
A);
6637 const auto *
TLI =
A.getInfoCache().getTargetLibraryInfoForFunction(*
F);
6644 DeallocationInfos[CB] =
new (
A.Allocator) DeallocationInfo{CB, FreedOp};
6651 auto *I8Ty = Type::getInt8Ty(CB->
getParent()->getContext());
6653 AllocationInfo *AI =
new (
A.Allocator) AllocationInfo{CB};
6654 AllocationInfos[CB] = AI;
6655 AI->IsGlobalizedLocal = isGlobalizedLocal(*CB);
6661 bool UsedAssumedInformation =
false;
6662 bool Success =
A.checkForAllCallLikeInstructions(
6663 AllocationIdentifierCB, *
this, UsedAssumedInformation,
6667 assert(
Success &&
"Did not expect the call base visit callback to fail!");
6670 [](
const IRPosition &,
const AbstractAttribute *,
6671 bool &) -> std::optional<Value *> {
return nullptr; };
6672 for (
const auto &It : AllocationInfos)
6675 for (
const auto &It : DeallocationInfos)
6680 const std::string getAsStr(Attributor *
A)
const override {
6681 unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
6682 for (
const auto &It : AllocationInfos) {
6683 if (It.second->Status == AllocationInfo::INVALID)
6684 ++NumInvalidMallocs;
6688 return "[H2S] Mallocs Good/Bad: " + std::to_string(NumH2SMallocs) +
"/" +
6689 std::to_string(NumInvalidMallocs);
6692 void trackStatistics()
const override {
6695 "Number of malloc/calloc/aligned_alloc calls converted to allocas");
6696 for (
const auto &It : AllocationInfos)
6697 if (It.second->Status != AllocationInfo::INVALID)
6701 bool isAssumedHeapToStack(
const CallBase &CB)
const override {
6703 if (AllocationInfo *AI =
6704 AllocationInfos.lookup(
const_cast<CallBase *
>(&CB)))
6705 return AI->Status != AllocationInfo::INVALID;
6709 bool isAssumedHeapToStackRemovedFree(CallBase &CB)
const override {
6710 if (!isValidState())
6713 for (
const auto &It : AllocationInfos) {
6714 AllocationInfo &AI = *It.second;
6715 if (AI.Status == AllocationInfo::INVALID)
6718 if (AI.PotentialFreeCalls.count(&CB))
6726 assert(getState().isValidState() &&
6727 "Attempted to manifest an invalid state!");
6731 const auto *
TLI =
A.getInfoCache().getTargetLibraryInfoForFunction(*
F);
6735 if (
any_of(AllocationInfos,
6736 [](
const std::pair<CallBase *, AllocationInfo *> &It) {
6737 return It.second->Status != AllocationInfo::INVALID;
6740 return ChangeStatus::UNCHANGED;
6742 for (
auto &It : AllocationInfos) {
6743 AllocationInfo &AI = *It.second;
6744 if (AI.Status == AllocationInfo::INVALID)
6747 for (CallBase *FreeCall : AI.PotentialFreeCalls) {
6748 LLVM_DEBUG(
dbgs() <<
"H2S: Removing free call: " << *FreeCall <<
"\n");
6749 A.deleteAfterManifest(*FreeCall);
6750 HasChanged = ChangeStatus::CHANGED;
6753 LLVM_DEBUG(
dbgs() <<
"H2S: Removing malloc-like call: " << *AI.CB
6756 auto Remark = [&](OptimizationRemark
OR) {
6757 if (AI.IsGlobalizedLocal)
6758 return OR <<
"Moving globalized variable to the stack.";
6759 return OR <<
"Moving memory allocation from the heap to the stack.";
6761 if (AI.IsGlobalizedLocal)
6762 A.emitRemark<OptimizationRemark>(AI.CB,
"OMP110",
Remark);
6764 A.emitRemark<OptimizationRemark>(AI.CB,
"HeapToStack",
Remark);
6766 const DataLayout &
DL =
A.getInfoCache().getDL();
6768 std::optional<APInt> SizeAPI =
getSize(
A, *
this, AI);
6770 Size = ConstantInt::get(AI.CB->getContext(), *SizeAPI);
6772 ObjectSizeOpts Opts;
6773 ObjectSizeOffsetEvaluator Eval(*AI.CB->getModule(),
TLI, Opts);
6774 SizeOffsetValue SizeOffsetPair = Eval.compute(AI.CB);
6781 ?
F->getEntryBlock().begin()
6782 : AI.CB->getIterator();
6785 if (MaybeAlign RetAlign = AI.CB->getRetAlign())
6786 Alignment = std::max(Alignment, *RetAlign);
6788 std::optional<APInt> AlignmentAPI = getAPInt(
A, *
this, *Align);
6789 assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
6790 "Expected an alignment during manifest!");
6792 std::max(Alignment,
assumeAligned(AlignmentAPI->getZExtValue()));
6796 unsigned AS =
DL.getAllocaAddrSpace();
6798 new AllocaInst(Type::getInt8Ty(
F->getContext()), AS,
Size, Alignment,
6799 AI.CB->getName() +
".h2s", IP);
6801 if (Alloca->
getType() != AI.CB->getType())
6802 Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
6803 Alloca, AI.CB->getType(),
"malloc_cast", AI.CB->getIterator());
6805 auto *I8Ty = Type::getInt8Ty(
F->getContext());
6808 "Must be able to materialize initial memory state of allocation");
6813 auto *NBB =
II->getNormalDest();
6815 A.deleteAfterManifest(*AI.CB);
6817 A.deleteAfterManifest(*AI.CB);
6826 Builder.CreateMemSet(Alloca, InitVal,
Size, std::nullopt);
6828 HasChanged = ChangeStatus::CHANGED;
6831 for (CallInst *CI : TailCalls)
6832 CI->setTailCall(
false);
6837 std::optional<APInt> getAPInt(Attributor &
A,
const AbstractAttribute &AA,
6839 bool UsedAssumedInformation =
false;
6840 std::optional<Constant *> SimpleV =
6841 A.getAssumedConstant(
V, AA, UsedAssumedInformation);
6843 return APInt(64, 0);
6845 return CI->getValue();
6846 return std::nullopt;
6849 std::optional<APInt>
getSize(Attributor &
A,
const AbstractAttribute &AA,
6850 AllocationInfo &AI) {
6852 bool UsedAssumedInformation =
false;
6853 if (std::optional<Constant *> SimpleV =
6854 A.getAssumedConstant(*
V, AA, UsedAssumedInformation))
6861 const auto *
TLI =
A.getInfoCache().getTargetLibraryInfoForFunction(*
F);
6867 MapVector<CallBase *, AllocationInfo *> AllocationInfos;
6871 MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
6876ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &
A) {
6879 const auto *
TLI =
A.getInfoCache().getTargetLibraryInfoForFunction(*
F);
6881 const auto *LivenessAA =
6884 MustBeExecutedContextExplorer *Explorer =
6885 A.getInfoCache().getMustBeExecutedContextExplorer();
6887 bool StackIsAccessibleByOtherThreads =
6888 A.getInfoCache().stackIsAccessibleByOtherThreads();
6891 A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(*F);
6892 std::optional<bool> MayContainIrreducibleControl;
6894 if (&
F->getEntryBlock() == &BB)
6896 if (!MayContainIrreducibleControl.has_value())
6898 if (*MayContainIrreducibleControl)
6907 bool HasUpdatedFrees =
false;
6909 auto UpdateFrees = [&]() {
6910 HasUpdatedFrees =
true;
6912 for (
auto &It : DeallocationInfos) {
6913 DeallocationInfo &DI = *It.second;
6916 if (DI.MightFreeUnknownObjects)
6920 bool UsedAssumedInformation =
false;
6921 if (
A.isAssumedDead(*DI.CB,
this, LivenessAA, UsedAssumedInformation,
6928 LLVM_DEBUG(
dbgs() <<
"[H2S] Unknown underlying object for free!\n");
6929 DI.MightFreeUnknownObjects =
true;
6942 DI.MightFreeUnknownObjects =
true;
6946 AllocationInfo *AI = AllocationInfos.lookup(ObjCB);
6948 LLVM_DEBUG(
dbgs() <<
"[H2S] Free of a non-allocation object: " << *Obj
6950 DI.MightFreeUnknownObjects =
true;
6954 DI.PotentialAllocationCalls.insert(ObjCB);
6958 auto FreeCheck = [&](AllocationInfo &AI) {
6962 if (!StackIsAccessibleByOtherThreads) {
6967 dbgs() <<
"[H2S] found an escaping use, stack is not accessible by "
6968 "other threads and function is not nosync:\n");
6972 if (!HasUpdatedFrees)
6976 if (AI.PotentialFreeCalls.size() != 1) {
6978 << AI.PotentialFreeCalls.size() <<
"\n");
6981 CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
6982 DeallocationInfo *DI = DeallocationInfos.lookup(UniqueFree);
6985 dbgs() <<
"[H2S] unique free call was not known as deallocation call "
6986 << *UniqueFree <<
"\n");
6989 if (DI->MightFreeUnknownObjects) {
6991 dbgs() <<
"[H2S] unique free call might free unknown allocations\n");
6994 if (DI->PotentialAllocationCalls.empty())
6996 if (DI->PotentialAllocationCalls.size() > 1) {
6998 << DI->PotentialAllocationCalls.size()
6999 <<
" different allocations\n");
7002 if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
7005 <<
"[H2S] unique free call not known to free this allocation but "
7006 << **DI->PotentialAllocationCalls.begin() <<
"\n");
7011 if (!AI.IsGlobalizedLocal) {
7013 if (!Explorer || !Explorer->isInContextOf(UniqueFree, CtxI)) {
7014 LLVM_DEBUG(
dbgs() <<
"[H2S] unique free call might not be executed "
7015 "with the allocation "
7016 << *UniqueFree <<
"\n");
7023 auto UsesCheck = [&](AllocationInfo &AI) {
7024 bool ValidUsesOnly =
true;
7026 auto Pred = [&](
const Use &
U,
bool &Follow) ->
bool {
7031 if (
SI->getValueOperand() ==
U.get()) {
7033 <<
"[H2S] escaping store to memory: " << *UserI <<
"\n");
7034 ValidUsesOnly =
false;
7045 ValidUsesOnly =
false;
7051 if (DeallocationInfos.count(CB)) {
7052 AI.PotentialFreeCalls.insert(CB);
7059 bool IsKnownNoCapture;
7068 if (!IsAssumedNoCapture ||
7069 (!AI.IsGlobalizedLocal && !IsAssumedNoFree)) {
7070 AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
7073 auto Remark = [&](OptimizationRemarkMissed ORM) {
7075 <<
"Could not move globalized variable to the stack. "
7076 "Variable is potentially captured in call. Mark "
7077 "parameter as `__attribute__((noescape))` to override.";
7080 if (ValidUsesOnly && AI.IsGlobalizedLocal)
7081 A.emitRemark<OptimizationRemarkMissed>(CB,
"OMP113",
Remark);
7084 ValidUsesOnly =
false;
7097 ValidUsesOnly =
false;
7100 if (!
A.checkForAllUses(
Pred, *
this, *AI.CB,
false,
7102 [&](
const Use &OldU,
const Use &NewU) {
7103 auto *SI = dyn_cast<StoreInst>(OldU.getUser());
7104 return !SI || StackIsAccessibleByOtherThreads ||
7105 AA::isAssumedThreadLocalObject(
7106 A, *SI->getPointerOperand(), *this);
7109 return ValidUsesOnly;
7114 for (
auto &It : AllocationInfos) {
7115 AllocationInfo &AI = *It.second;
7116 if (AI.Status == AllocationInfo::INVALID)
7120 std::optional<APInt> APAlign = getAPInt(
A, *
this, *Align);
7124 LLVM_DEBUG(
dbgs() <<
"[H2S] Unknown allocation alignment: " << *AI.CB
7126 AI.Status = AllocationInfo::INVALID;
7131 !APAlign->isPowerOf2()) {
7132 LLVM_DEBUG(
dbgs() <<
"[H2S] Invalid allocation alignment: " << APAlign
7134 AI.Status = AllocationInfo::INVALID;
7141 if (!AI.IsGlobalizedLocal &&
A.Opts.max_heap_to_stack_size != -1) {
7142 if (!
Size ||
Size->ugt(
A.Opts.max_heap_to_stack_size)) {
7145 dbgs() <<
"[H2S] Unknown allocation size: " << *AI.CB <<
"\n";
7147 dbgs() <<
"[H2S] Allocation size too large: " << *AI.CB <<
" vs. "
7148 <<
A.Opts.max_heap_to_stack_size <<
"\n";
7151 AI.Status = AllocationInfo::INVALID;
7157 switch (AI.Status) {
7158 case AllocationInfo::STACK_DUE_TO_USE:
7161 AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
7163 case AllocationInfo::STACK_DUE_TO_FREE:
7166 AI.Status = AllocationInfo::INVALID;
7169 case AllocationInfo::INVALID:
7176 bool IsGlobalizedLocal = AI.IsGlobalizedLocal;
7177 if (AI.MoveAllocaIntoEntry &&
7178 (!
Size.has_value() ||
7179 (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
7180 AI.MoveAllocaIntoEntry =
false;
7189struct AAPrivatizablePtrImpl :
public AAPrivatizablePtr {
7190 AAPrivatizablePtrImpl(
const IRPosition &IRP, Attributor &
A)
7191 : AAPrivatizablePtr(IRP,
A), PrivatizableType(std::nullopt) {}
7194 AAPrivatizablePtr::indicatePessimisticFixpoint();
7195 PrivatizableType =
nullptr;
7196 return ChangeStatus::CHANGED;
7202 virtual std::optional<Type *> identifyPrivatizableType(Attributor &
A) = 0;
7206 std::optional<Type *> combineTypes(std::optional<Type *> T0,
7207 std::optional<Type *>
T1) {
7217 std::optional<Type *> getPrivatizableType()
const override {
7218 return PrivatizableType;
7221 const std::string getAsStr(Attributor *
A)
const override {
7222 return isAssumedPrivatizablePtr() ?
"[priv]" :
"[no-priv]";
7226 std::optional<Type *> PrivatizableType;
7231struct AAPrivatizablePtrArgument final :
public AAPrivatizablePtrImpl {
7232 AAPrivatizablePtrArgument(
const IRPosition &IRP, Attributor &
A)
7233 : AAPrivatizablePtrImpl(IRP,
A) {}
7235 std::optional<Type *> identifyPrivatizableType(Attributor &
A)
override {
7238 bool UsedAssumedInformation =
false;
7240 A.getAttrs(getIRPosition(), {Attribute::ByVal},
Attrs,
7242 if (!
Attrs.empty() &&
7243 A.checkForAllCallSites([](AbstractCallSite ACS) { return true; }, *
this,
7244 true, UsedAssumedInformation))
7245 return Attrs[0].getValueAsType();
7247 std::optional<Type *> Ty;
7248 unsigned ArgNo = getIRPosition().getCallSiteArgNo();
7256 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7265 A.getAAFor<AAPrivatizablePtr>(*
this, ACSArgPos, DepClassTy::REQUIRED);
7268 std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
7271 dbgs() <<
"[AAPrivatizablePtr] ACSPos: " << ACSArgPos <<
", CSTy: ";
7275 dbgs() <<
"<nullptr>";
7280 Ty = combineTypes(Ty, CSTy);
7283 dbgs() <<
" : New Type: ";
7285 (*Ty)->print(
dbgs());
7287 dbgs() <<
"<nullptr>";
7296 if (!
A.checkForAllCallSites(CallSiteCheck, *
this,
true,
7297 UsedAssumedInformation))
7303 PrivatizableType = identifyPrivatizableType(
A);
7304 if (!PrivatizableType)
7305 return ChangeStatus::UNCHANGED;
7306 if (!*PrivatizableType)
7307 return indicatePessimisticFixpoint();
7312 DepClassTy::OPTIONAL);
7315 if (!
A.hasAttr(getIRPosition(), Attribute::ByVal) &&
7318 return indicatePessimisticFixpoint();
7324 identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
7328 Function &Fn = *getIRPosition().getAnchorScope();
7330 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(Fn);
7332 LLVM_DEBUG(
dbgs() <<
"[AAPrivatizablePtr] Missing TTI for function "
7334 return indicatePessimisticFixpoint();
7337 auto CallSiteCheck = [&](AbstractCallSite ACS) {
7344 bool UsedAssumedInformation =
false;
7345 if (!
A.checkForAllCallSites(CallSiteCheck, *
this,
true,
7346 UsedAssumedInformation)) {
7348 dbgs() <<
"[AAPrivatizablePtr] ABI incompatibility detected for "
7350 return indicatePessimisticFixpoint();
7354 Argument *Arg = getAssociatedArgument();
7355 if (!
A.isValidFunctionSignatureRewrite(*Arg, ReplacementTypes)) {
7357 return indicatePessimisticFixpoint();
7364 auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
7367 for (
const Use *U : CallbackUses) {
7368 AbstractCallSite CBACS(U);
7369 assert(CBACS && CBACS.isCallbackCall());
7370 for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
7371 int CBArgNo = CBACS.getCallArgOperandNo(CBArg);
7375 <<
"[AAPrivatizablePtr] Argument " << *Arg
7376 <<
"check if can be privatized in the context of its parent ("
7378 <<
")\n[AAPrivatizablePtr] because it is an argument in a "
7380 << CBArgNo <<
"@" << CBACS.getCalledFunction()->getName()
7381 <<
")\n[AAPrivatizablePtr] " << CBArg <<
" : "
7382 << CBACS.getCallArgOperand(CBArg) <<
" vs "
7384 <<
"[AAPrivatizablePtr] " << CBArg <<
" : "
7385 << CBACS.getCallArgOperandNo(CBArg) <<
" vs " << ArgNo <<
"\n";
7388 if (CBArgNo !=
int(ArgNo))
7390 const auto *CBArgPrivAA =
A.getAAFor<AAPrivatizablePtr>(
7392 if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
7393 auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
7396 if (*CBArgPrivTy == PrivatizableType)
7401 dbgs() <<
"[AAPrivatizablePtr] Argument " << *Arg
7402 <<
" cannot be privatized in the context of its parent ("
7404 <<
")\n[AAPrivatizablePtr] because it is an argument in a "
7406 << CBArgNo <<
"@" << CBACS.getCalledFunction()->getName()
7407 <<
").\n[AAPrivatizablePtr] for which the argument "
7408 "privatization is not compatible.\n";
7418 auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
7422 "Expected a direct call operand for callback call operand");
7427 dbgs() <<
"[AAPrivatizablePtr] Argument " << *Arg
7428 <<
" check if be privatized in the context of its parent ("
7430 <<
")\n[AAPrivatizablePtr] because it is an argument in a "
7432 << DCArgNo <<
"@" << DCCallee->
getName() <<
").\n";
7435 if (
unsigned(DCArgNo) < DCCallee->
arg_size()) {
7436 const auto *DCArgPrivAA =
A.getAAFor<AAPrivatizablePtr>(
7438 DepClassTy::REQUIRED);
7439 if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
7440 auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
7443 if (*DCArgPrivTy == PrivatizableType)
7449 dbgs() <<
"[AAPrivatizablePtr] Argument " << *Arg
7450 <<
" cannot be privatized in the context of its parent ("
7452 <<
")\n[AAPrivatizablePtr] because it is an argument in a "
7455 <<
").\n[AAPrivatizablePtr] for which the argument "
7456 "privatization is not compatible.\n";
7464 auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
7468 return IsCompatiblePrivArgOfDirectCS(ACS);
7472 if (!
A.checkForAllCallSites(IsCompatiblePrivArgOfOtherCallSite, *
this,
true,
7473 UsedAssumedInformation))
7474 return indicatePessimisticFixpoint();
7476 return ChangeStatus::UNCHANGED;
7482 identifyReplacementTypes(
Type *PrivType,
7483 SmallVectorImpl<Type *> &ReplacementTypes) {
7486 assert(PrivType &&
"Expected privatizable type!");
7490 for (
unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
7491 ReplacementTypes.
push_back(PrivStructType->getElementType(u));
7493 ReplacementTypes.
append(PrivArrayType->getNumElements(),
7494 PrivArrayType->getElementType());
7505 assert(PrivType &&
"Expected privatizable type!");
7508 const DataLayout &
DL =
F.getDataLayout();
7512 const StructLayout *PrivStructLayout =
DL.getStructLayout(PrivStructType);
7513 for (
unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7516 new StoreInst(
F.getArg(ArgNo + u), Ptr, IP);
7519 Type *PointeeTy = PrivArrayType->getElementType();
7520 uint64_t PointeeTySize =
DL.getTypeStoreSize(PointeeTy);
7521 for (
unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7523 new StoreInst(
F.getArg(ArgNo + u), Ptr, IP);
7526 new StoreInst(
F.getArg(ArgNo), &
Base, IP);
7532 void createReplacementValues(Align Alignment,
Type *PrivType,
7534 SmallVectorImpl<Value *> &ReplacementValues) {
7536 assert(PrivType &&
"Expected privatizable type!");
7544 const StructLayout *PrivStructLayout =
DL.getStructLayout(PrivStructType);
7545 for (
unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
7546 Type *PointeeTy = PrivStructType->getElementType(u);
7549 LoadInst *
L =
new LoadInst(PointeeTy, Ptr,
"", IP->
getIterator());
7550 L->setAlignment(Alignment);
7554 Type *PointeeTy = PrivArrayType->getElementType();
7555 uint64_t PointeeTySize =
DL.getTypeStoreSize(PointeeTy);
7556 for (
unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
7558 LoadInst *
L =
new LoadInst(PointeeTy, Ptr,
"", IP->
getIterator());
7559 L->setAlignment(Alignment);
7564 L->setAlignment(Alignment);
7570 if (!PrivatizableType)
7571 return ChangeStatus::UNCHANGED;
7572 assert(*PrivatizableType &&
"Expected privatizable type!");
7574 Argument *Arg = getAssociatedArgument();
7577 const auto *AlignAA =
7584 [=](
const Attributor::ArgumentReplacementInfo &ARI,
7586 BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
7588 const DataLayout &
DL = IP->getDataLayout();
7589 unsigned AS =
DL.getAllocaAddrSpace();
7590 Instruction *AI =
new AllocaInst(*PrivatizableType, AS,
7591 Arg->
getName() +
".priv", IP);
7592 createInitialization(*PrivatizableType, *AI, ReplacementFn,
7593 ArgIt->getArgNo(), IP);
7596 AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
7604 if (CI && CI->isTailCall() && !CI->isMustTailCall())
7605 CI->setTailCall(
false);
7613 [=](
const Attributor::ArgumentReplacementInfo &ARI,
7614 AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
7617 createReplacementValues(
7618 AlignAA ? AlignAA->getAssumedAlign() :
Align(0),
7619 *PrivatizableType, ACS,
7627 identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
7630 if (
A.registerFunctionSignatureRewrite(*Arg, ReplacementTypes,
7631 std::move(FnRepairCB),
7632 std::move(ACSRepairCB)))
7633 return ChangeStatus::CHANGED;
7634 return ChangeStatus::UNCHANGED;
7637 void trackStatistics()
const override {
7642struct AAPrivatizablePtrFloating :
public AAPrivatizablePtrImpl {
7643 AAPrivatizablePtrFloating(
const IRPosition &IRP, Attributor &
A)
7644 : AAPrivatizablePtrImpl(IRP,
A) {}
7648 indicatePessimisticFixpoint();
7653 "updateImpl will not be called");
7656 std::optional<Type *> identifyPrivatizableType(Attributor &
A)
override {
7659 LLVM_DEBUG(
dbgs() <<
"[AAPrivatizablePtr] No underlying object found!\n");
7666 return AI->getAllocatedType();
7668 auto *PrivArgAA =
A.getAAFor<AAPrivatizablePtr>(
7670 if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
7671 return PrivArgAA->getPrivatizableType();
7674 LLVM_DEBUG(
dbgs() <<
"[AAPrivatizablePtr] Underlying object neither valid "
7675 "alloca nor privatizable argument: "
7680 void trackStatistics()
const override {
7685struct AAPrivatizablePtrCallSiteArgument final
7686 :
public AAPrivatizablePtrFloating {
7687 AAPrivatizablePtrCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
7688 : AAPrivatizablePtrFloating(IRP,
A) {}
7691 if (
A.hasAttr(getIRPosition(), Attribute::ByVal))
7692 indicateOptimisticFixpoint();
7696 PrivatizableType = identifyPrivatizableType(
A);
7697 if (!PrivatizableType)
7698 return ChangeStatus::UNCHANGED;
7699 if (!*PrivatizableType)
7700 return indicatePessimisticFixpoint();
7702 const IRPosition &IRP = getIRPosition();
7703 bool IsKnownNoCapture;
7705 A,
this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture);
7706 if (!IsAssumedNoCapture) {
7707 LLVM_DEBUG(
dbgs() <<
"[AAPrivatizablePtr] pointer might be captured!\n");
7708 return indicatePessimisticFixpoint();
7711 bool IsKnownNoAlias;
7713 A,
this, IRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
7714 LLVM_DEBUG(
dbgs() <<
"[AAPrivatizablePtr] pointer might alias!\n");
7715 return indicatePessimisticFixpoint();
7720 LLVM_DEBUG(
dbgs() <<
"[AAPrivatizablePtr] pointer is written!\n");
7721 return indicatePessimisticFixpoint();
7724 return ChangeStatus::UNCHANGED;
7727 void trackStatistics()
const override {
7732struct AAPrivatizablePtrCallSiteReturned final
7733 :
public AAPrivatizablePtrFloating {
7734 AAPrivatizablePtrCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
7735 : AAPrivatizablePtrFloating(IRP,
A) {}
7739 indicatePessimisticFixpoint();
7742 void trackStatistics()
const override {
7747struct AAPrivatizablePtrReturned final :
public AAPrivatizablePtrFloating {
7748 AAPrivatizablePtrReturned(
const IRPosition &IRP, Attributor &
A)
7749 : AAPrivatizablePtrFloating(IRP,
A) {}
7753 indicatePessimisticFixpoint();
7756 void trackStatistics()
const override {
7766struct AAMemoryBehaviorImpl :
public AAMemoryBehavior {
7767 AAMemoryBehaviorImpl(
const IRPosition &IRP, Attributor &
A)
7768 : AAMemoryBehavior(IRP,
A) {}
7771 intersectAssumedBits(BEST_STATE);
7772 getKnownStateFromValue(
A, getIRPosition(), getState());
7773 AAMemoryBehavior::initialize(
A);
7777 static void getKnownStateFromValue(Attributor &
A,
const IRPosition &IRP,
7778 BitIntegerState &State,
7779 bool IgnoreSubsumingPositions =
false) {
7781 A.getAttrs(IRP, AttrKinds, Attrs, IgnoreSubsumingPositions);
7783 switch (Attr.getKindAsEnum()) {
7784 case Attribute::ReadNone:
7787 case Attribute::ReadOnly:
7790 case Attribute::WriteOnly:
7814 void getDeducedAttributes(Attributor &
A, LLVMContext &Ctx,
7815 SmallVectorImpl<Attribute> &Attrs)
const override {
7818 Attrs.push_back(Attribute::get(Ctx, Attribute::ReadNone));
7820 Attrs.push_back(Attribute::get(Ctx, Attribute::ReadOnly));
7821 else if (isAssumedWriteOnly())
7822 Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly));
7827 const IRPosition &IRP = getIRPosition();
7829 if (
A.hasAttr(IRP, Attribute::ReadNone,
7831 return ChangeStatus::UNCHANGED;
7840 return ChangeStatus::UNCHANGED;
7843 A.removeAttrs(IRP, AttrKinds);
7846 A.removeAttrs(IRP, Attribute::Writable);
7852 const std::string getAsStr(Attributor *
A)
const override {
7857 if (isAssumedWriteOnly())
7859 return "may-read/write";
7863 static const Attribute::AttrKind AttrKinds[3];
7867 Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
7870struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
7871 AAMemoryBehaviorFloating(
const IRPosition &IRP, Attributor &
A)
7872 : AAMemoryBehaviorImpl(IRP,
A) {}
7876 void trackStatistics()
const override {
7881 else if (isAssumedWriteOnly())
7888 bool followUsersOfUseIn(Attributor &
A,
const Use &U,
7889 const Instruction *UserI);
7892 void analyzeUseIn(Attributor &
A,
const Use &U,
const Instruction *UserI);
7896struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
7897 AAMemoryBehaviorArgument(
const IRPosition &IRP, Attributor &
A)
7898 : AAMemoryBehaviorFloating(IRP,
A) {}
7901 intersectAssumedBits(BEST_STATE);
7902 const IRPosition &IRP = getIRPosition();
7906 bool HasByVal =
A.hasAttr(IRP, {Attribute::ByVal},
7908 getKnownStateFromValue(
A, IRP, getState(),
7915 return ChangeStatus::UNCHANGED;
7919 if (
A.hasAttr(getIRPosition(),
7920 {Attribute::InAlloca, Attribute::Preallocated})) {
7921 removeKnownBits(NO_WRITES);
7922 removeAssumedBits(NO_WRITES);
7924 A.removeAttrs(getIRPosition(), AttrKinds);
7925 return AAMemoryBehaviorFloating::manifest(
A);
7928 void trackStatistics()
const override {
7933 else if (isAssumedWriteOnly())
7938struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
7939 AAMemoryBehaviorCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
7940 : AAMemoryBehaviorArgument(IRP,
A) {}
7945 Argument *Arg = getAssociatedArgument();
7947 indicatePessimisticFixpoint();
7951 addKnownBits(NO_WRITES);
7952 removeKnownBits(NO_READS);
7953 removeAssumedBits(NO_READS);
7955 AAMemoryBehaviorArgument::initialize(
A);
7956 if (getAssociatedFunction()->isDeclaration())
7957 indicatePessimisticFixpoint();
7965 Argument *Arg = getAssociatedArgument();
7968 A.getAAFor<AAMemoryBehavior>(*
this, ArgPos, DepClassTy::REQUIRED);
7970 return indicatePessimisticFixpoint();
7974 void trackStatistics()
const override {
7979 else if (isAssumedWriteOnly())
7985struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
7986 AAMemoryBehaviorCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
7987 : AAMemoryBehaviorFloating(IRP,
A) {}
7990 AAMemoryBehaviorImpl::initialize(
A);
7994 return ChangeStatus::UNCHANGED;
7997 void trackStatistics()
const override {}
8001struct AAMemoryBehaviorFunction final :
public AAMemoryBehaviorImpl {
8002 AAMemoryBehaviorFunction(
const IRPosition &IRP, Attributor &
A)
8003 : AAMemoryBehaviorImpl(IRP,
A) {}
8017 else if (isAssumedWriteOnly())
8020 A.removeAttrs(getIRPosition(), AttrKinds);
8023 for (Argument &Arg :
F.args())
8025 return A.manifestAttrs(getIRPosition(),
8026 Attribute::getWithMemoryEffects(
F.getContext(), ME));
8029 void trackStatistics()
const override {
8034 else if (isAssumedWriteOnly())
8040struct AAMemoryBehaviorCallSite final
8041 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
8042 AAMemoryBehaviorCallSite(
const IRPosition &IRP, Attributor &
A)
8043 : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP,
A) {}
8053 else if (isAssumedWriteOnly())
8056 A.removeAttrs(getIRPosition(), AttrKinds);
8059 for (Use &U : CB.
args())
8061 Attribute::Writable);
8062 return A.manifestAttrs(
8063 getIRPosition(), Attribute::getWithMemoryEffects(CB.
getContext(), ME));
8066 void trackStatistics()
const override {
8071 else if (isAssumedWriteOnly())
8076ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &
A) {
8079 auto AssumedState = getAssumed();
8086 const auto *MemBehaviorAA =
A.getAAFor<AAMemoryBehavior>(
8088 if (MemBehaviorAA) {
8089 intersectAssumedBits(MemBehaviorAA->
getAssumed());
8090 return !isAtFixpoint();
8095 if (
I.mayReadFromMemory())
8096 removeAssumedBits(NO_READS);
8097 if (
I.mayWriteToMemory())
8098 removeAssumedBits(NO_WRITES);
8099 return !isAtFixpoint();
8102 bool UsedAssumedInformation =
false;
8103 if (!
A.checkForAllReadWriteInstructions(CheckRWInst, *
this,
8104 UsedAssumedInformation))
8105 return indicatePessimisticFixpoint();
8111ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &
A) {
8113 const IRPosition &IRP = getIRPosition();
8124 const auto *FnMemAA =
8127 FnMemAssumedState = FnMemAA->getAssumed();
8128 S.addKnownBits(FnMemAA->getKnown());
8129 if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
8135 auto AssumedState = S.getAssumed();
8141 bool IsKnownNoCapture;
8142 const AANoCapture *ArgNoCaptureAA =
nullptr;
8147 if (!IsAssumedNoCapture &&
8149 S.intersectAssumedBits(FnMemAssumedState);
8155 auto UsePred = [&](
const Use &
U,
bool &Follow) ->
bool {
8157 LLVM_DEBUG(
dbgs() <<
"[AAMemoryBehavior] Use: " << *U <<
" in " << *UserI
8165 Follow = followUsersOfUseIn(
A, U, UserI);
8169 analyzeUseIn(
A, U, UserI);
8171 return !isAtFixpoint();
8174 if (!
A.checkForAllUses(UsePred, *
this, getAssociatedValue()))
8175 return indicatePessimisticFixpoint();
8181bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &
A,
const Use &U,
8182 const Instruction *UserI) {
8200 if (
U.get()->getType()->isPointerTy()) {
8202 bool IsKnownNoCapture;
8211void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &
A,
const Use &U,
8212 const Instruction *UserI) {
8219 case Instruction::Load:
8221 removeAssumedBits(NO_READS);
8224 case Instruction::Store:
8229 removeAssumedBits(NO_WRITES);
8231 indicatePessimisticFixpoint();
8234 case Instruction::Call:
8235 case Instruction::CallBr:
8236 case Instruction::Invoke: {
8243 indicatePessimisticFixpoint();
8250 removeAssumedBits(NO_READS);
8257 if (
U.get()->getType()->isPointerTy())
8261 const auto *MemBehaviorAA =
8267 intersectAssumedBits(MemBehaviorAA->
getAssumed());
8275 removeAssumedBits(NO_READS);
8277 removeAssumedBits(NO_WRITES);
8289 return "all memory";
8292 std::string S =
"memory:";
8298 S +=
"internal global,";
8300 S +=
"external global,";
8304 S +=
"inaccessible,";
8318 AccessKind2Accesses.fill(
nullptr);
8321 ~AAMemoryLocationImpl()
override {
8324 for (AccessSet *AS : AccessKind2Accesses)
8330 intersectAssumedBits(BEST_STATE);
8331 getKnownStateFromValue(
A, getIRPosition(), getState());
8332 AAMemoryLocation::initialize(
A);
8336 static void getKnownStateFromValue(Attributor &
A,
const IRPosition &IRP,
8337 BitIntegerState &State,
8338 bool IgnoreSubsumingPositions =
false) {
8347 bool UseArgMemOnly =
true;
8349 if (AnchorFn &&
A.isRunOn(*AnchorFn))
8353 A.getAttrs(IRP, {Attribute::Memory},
Attrs, IgnoreSubsumingPositions);
8362 State.
addKnownBits(inverseLocation(NO_INACCESSIBLE_MEM,
true,
true));
8367 State.
addKnownBits(inverseLocation(NO_ARGUMENT_MEM,
true,
true));
8371 A.manifestAttrs(IRP,
8372 Attribute::getWithMemoryEffects(
8381 NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM,
true,
true));
8385 A.manifestAttrs(IRP,
8386 Attribute::getWithMemoryEffects(
8395 void getDeducedAttributes(Attributor &
A, LLVMContext &Ctx,
8396 SmallVectorImpl<Attribute> &Attrs)
const override {
8403 else if (isAssumedInaccessibleMemOnly())
8404 Attrs.push_back(Attribute::getWithMemoryEffects(
8406 else if (isAssumedArgMemOnly())
8409 else if (isAssumedInaccessibleOrArgMemOnly())
8410 Attrs.push_back(Attribute::getWithMemoryEffects(
8419 const IRPosition &IRP = getIRPosition();
8423 if (DeducedAttrs.
size() != 1)
8424 return ChangeStatus::UNCHANGED;
8434 return A.manifestAttrs(IRP, Attribute::getWithMemoryEffects(
8438 bool checkForAllAccessesToMemoryKind(
8440 MemoryLocationsKind)>
8442 MemoryLocationsKind RequestedMLK)
const override {
8443 if (!isValidState())
8446 MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
8447 if (AssumedMLK == NO_LOCATIONS)
8451 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
8452 CurMLK *= 2, ++
Idx) {
8453 if (CurMLK & RequestedMLK)
8456 if (
const AccessSet *
Accesses = AccessKind2Accesses[Idx])
8457 for (
const AccessInfo &AI : *
Accesses)
8458 if (!
Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
8471 MemoryLocationsKind KnownMLK = getKnown();
8473 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
8474 if (!(CurMLK & KnownMLK))
8475 updateStateAndAccessesMap(getState(), CurMLK,
I,
nullptr,
Changed,
8476 getAccessKindFromInst(
I));
8477 return AAMemoryLocation::indicatePessimisticFixpoint();
8497 bool operator()(
const AccessInfo &
LHS,
const AccessInfo &
RHS)
const {
8501 return LHS.Ptr <
RHS.Ptr;
8502 if (
LHS.Kind !=
RHS.Kind)
8503 return LHS.Kind <
RHS.Kind;
8510 using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
8511 std::array<AccessSet *, llvm::ConstantLog2<VALID_STATE>()>
8512 AccessKind2Accesses;
8517 categorizeArgumentPointerLocations(Attributor &
A, CallBase &CB,
8518 AAMemoryLocation::StateType &AccessedLocs,
8523 categorizeAccessedLocations(Attributor &
A, Instruction &
I,
bool &
Changed);
8526 AccessKind getAccessKindFromInst(
const Instruction *
I) {
8529 AK =
I->mayReadFromMemory() ? READ :
NONE;
8538 void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
8539 MemoryLocationsKind MLK,
const Instruction *
I,
8548 if (MLK == NO_UNKOWN_MEM)
8550 State.removeAssumedBits(MLK);
8555 void categorizePtrValue(Attributor &
A,
const Instruction &
I,
const Value &Ptr,
8556 AAMemoryLocation::StateType &State,
bool &
Changed,
8557 unsigned AccessAS = 0);
8563void AAMemoryLocationImpl::categorizePtrValue(
8564 Attributor &
A,
const Instruction &
I,
const Value &Ptr,
8566 LLVM_DEBUG(
dbgs() <<
"[AAMemoryLocation] Categorize pointer locations for "
8571 unsigned ObjectAS =
Obj.getType()->getPointerAddressSpace();
8573 MemoryLocationsKind MLK = NO_LOCATIONS;
8593 MLK = NO_ARGUMENT_MEM;
8599 if (GVar->isConstant())
8602 if (GV->hasLocalLinkage())
8603 MLK = NO_GLOBAL_INTERNAL_MEM;
8605 MLK = NO_GLOBAL_EXTERNAL_MEM;
8614 bool IsKnownNoAlias;
8618 MLK = NO_MALLOCED_MEM;
8620 MLK = NO_UNKOWN_MEM;
8622 MLK = NO_UNKOWN_MEM;
8625 assert(MLK != NO_LOCATIONS &&
"No location specified!");
8626 LLVM_DEBUG(
dbgs() <<
"[AAMemoryLocation] Ptr value can be categorized: "
8627 << Obj <<
" -> " << getMemoryLocationsAsStr(MLK) <<
"\n");
8629 getAccessKindFromInst(&
I));
8634 const auto *AA =
A.getAAFor<AAUnderlyingObjects>(
8638 dbgs() <<
"[AAMemoryLocation] Pointer locations not categorized\n");
8639 updateStateAndAccessesMap(
State, NO_UNKOWN_MEM, &
I,
nullptr,
Changed,
8640 getAccessKindFromInst(&
I));
8645 dbgs() <<
"[AAMemoryLocation] Accessed locations with pointer locations: "
8649void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
8652 for (
unsigned ArgNo = 0,
E = CB.
arg_size(); ArgNo <
E; ++ArgNo) {
8661 const auto *ArgOpMemLocationAA =
8664 if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
8669 categorizePtrValue(
A, CB, *ArgOp, AccessedLocs,
Changed);
8674AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &
A, Instruction &
I,
8676 LLVM_DEBUG(
dbgs() <<
"[AAMemoryLocation] Categorize accessed locations for "
8680 AccessedLocs.intersectAssumedBits(NO_LOCATIONS);
8685 const auto *CBMemLocationAA =
A.getAAFor<AAMemoryLocation>(
8688 <<
" [" << CBMemLocationAA <<
"]\n");
8689 if (!CBMemLocationAA) {
8690 updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &
I,
nullptr,
8691 Changed, getAccessKindFromInst(&
I));
8692 return NO_UNKOWN_MEM;
8695 if (CBMemLocationAA->isAssumedReadNone())
8696 return NO_LOCATIONS;
8698 if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
8699 updateStateAndAccessesMap(AccessedLocs, NO_INACCESSIBLE_MEM, &
I,
nullptr,
8700 Changed, getAccessKindFromInst(&
I));
8701 return AccessedLocs.getAssumed();
8704 uint32_t CBAssumedNotAccessedLocs =
8705 CBMemLocationAA->getAssumedNotAccessedLocation();
8708 uint32_t CBAssumedNotAccessedLocsNoArgMem =
8709 CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
8711 for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
8712 if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
8714 updateStateAndAccessesMap(AccessedLocs, CurMLK, &
I,
nullptr,
Changed,
8715 getAccessKindFromInst(&
I));
8720 bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
8721 if (HasGlobalAccesses) {
8724 updateStateAndAccessesMap(AccessedLocs, MLK, &
I, Ptr,
Changed,
8725 getAccessKindFromInst(&
I));
8728 if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
8729 AccessPred, inverseLocation(NO_GLOBAL_MEM,
false,
false)))
8730 return AccessedLocs.getWorstState();
8734 dbgs() <<
"[AAMemoryLocation] Accessed state before argument handling: "
8735 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) <<
"\n");
8738 bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
8740 categorizeArgumentPointerLocations(
A, *CB, AccessedLocs,
Changed);
8743 dbgs() <<
"[AAMemoryLocation] Accessed state after argument handling: "
8744 << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) <<
"\n");
8746 return AccessedLocs.getAssumed();
8751 dbgs() <<
"[AAMemoryLocation] Categorize memory access with pointer: "
8752 <<
I <<
" [" << *Ptr <<
"]\n");
8753 categorizePtrValue(
A,
I, *Ptr, AccessedLocs,
Changed,
8754 Ptr->getType()->getPointerAddressSpace());
8755 return AccessedLocs.getAssumed();
8758 LLVM_DEBUG(
dbgs() <<
"[AAMemoryLocation] Failed to categorize instruction: "
8760 updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &
I,
nullptr,
Changed,
8761 getAccessKindFromInst(&
I));
8762 return AccessedLocs.getAssumed();
8766struct AAMemoryLocationFunction final :
public AAMemoryLocationImpl {
8767 AAMemoryLocationFunction(
const IRPosition &IRP, Attributor &
A)
8768 : AAMemoryLocationImpl(IRP,
A) {}
8772 const auto *MemBehaviorAA =
8773 A.getAAFor<AAMemoryBehavior>(*
this, getIRPosition(), DepClassTy::NONE);
8776 return indicateOptimisticFixpoint();
8778 "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
8779 A.recordDependence(*MemBehaviorAA, *
this, DepClassTy::OPTIONAL);
8780 return ChangeStatus::UNCHANGED;
8784 auto AssumedState = getAssumed();
8788 MemoryLocationsKind MLK = categorizeAccessedLocations(
A,
I,
Changed);
8789 LLVM_DEBUG(
dbgs() <<
"[AAMemoryLocation] Accessed locations for " <<
I
8790 <<
": " << getMemoryLocationsAsStr(MLK) <<
"\n");
8791 removeAssumedBits(inverseLocation(MLK,
false,
false));
8794 return getAssumedNotAccessedLocation() != VALID_STATE;
8797 bool UsedAssumedInformation =
false;
8798 if (!
A.checkForAllReadWriteInstructions(CheckRWInst, *
this,
8799 UsedAssumedInformation))
8800 return indicatePessimisticFixpoint();
8802 Changed |= AssumedState != getAssumed();
8803 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8806 void trackStatistics()
const override {
8809 else if (isAssumedArgMemOnly())
8811 else if (isAssumedInaccessibleMemOnly())
8813 else if (isAssumedInaccessibleOrArgMemOnly())
8819struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
8820 AAMemoryLocationCallSite(
const IRPosition &IRP, Attributor &
A)
8821 : AAMemoryLocationImpl(IRP,
A) {}
8824 AAMemoryLocationImpl::initialize(
A);
8831 removeKnownBits(NO_INACCESSIBLE_MEM);
8832 updateStateAndAccessesMap(getState(), NO_INACCESSIBLE_MEM, &CB,
nullptr,
8833 Changed, getAccessKindFromInst(&CB));
8845 A.getAAFor<AAMemoryLocation>(*
this, FnPos, DepClassTy::REQUIRED);
8847 return indicatePessimisticFixpoint();
8851 updateStateAndAccessesMap(getState(), MLK,
I, Ptr,
Changed,
8852 getAccessKindFromInst(
I));
8855 if (!FnAA->checkForAllAccessesToMemoryKind(AccessPred, ALL_LOCATIONS))
8856 return indicatePessimisticFixpoint();
8857 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
8860 void trackStatistics()
const override {
8870struct AADenormalFPMathImpl :
public AADenormalFPMath {
8871 AADenormalFPMathImpl(
const IRPosition &IRP, Attributor &
A)
8872 : AADenormalFPMath(IRP,
A) {}
8874 const std::string getAsStr(Attributor *
A)
const override {
8875 std::string Str(
"AADenormalFPMath[");
8876 raw_string_ostream OS(Str);
8878 DenormalState
Known = getKnown();
8879 if (
Known.Mode.isValid())
8880 OS <<
"denormal-fp-math=" <<
Known.Mode;
8884 if (
Known.ModeF32.isValid())
8885 OS <<
" denormal-fp-math-f32=" <<
Known.ModeF32;
8891struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
8892 AADenormalFPMathFunction(
const IRPosition &IRP, Attributor &
A)
8893 : AADenormalFPMathImpl(IRP,
A) {}
8897 DenormalFPEnv DenormEnv =
F->getDenormalFPEnv();
8907 auto CheckCallSite = [=, &Change, &
A](AbstractCallSite CS) {
8910 <<
"->" << getAssociatedFunction()->
getName() <<
'\n');
8912 const auto *CallerInfo =
A.getAAFor<AADenormalFPMath>(
8918 CallerInfo->getState());
8922 bool AllCallSitesKnown =
true;
8923 if (!
A.checkForAllCallSites(CheckCallSite, *
this,
true, AllCallSitesKnown))
8924 return indicatePessimisticFixpoint();
8926 if (Change == ChangeStatus::CHANGED && isModeFixed())
8932 LLVMContext &Ctx = getAssociatedFunction()->getContext();
8938 DenormalFPEnv KnownEnv(
Known.Mode,
Known.ModeF32);
8941 AttrToRemove.
push_back(Attribute::DenormalFPEnv);
8944 Ctx, Attribute::DenormalFPEnv,
8945 DenormalFPEnv(
Known.Mode,
Known.ModeF32).toIntValue()));
8948 auto &IRP = getIRPosition();
8951 return A.removeAttrs(IRP, AttrToRemove) |
8952 A.manifestAttrs(IRP, AttrToAdd,
true);
8955 void trackStatistics()
const override {
8964struct AAValueConstantRangeImpl : AAValueConstantRange {
8965 using StateType = IntegerRangeState;
8966 AAValueConstantRangeImpl(
const IRPosition &IRP, Attributor &
A)
8967 : AAValueConstantRange(IRP,
A) {}
8970 if (
A.hasSimplificationCallback(getIRPosition())) {
8971 indicatePessimisticFixpoint();
8976 intersectKnown(getConstantRangeFromSCEV(
A, getCtxI()));
8979 intersectKnown(getConstantRangeFromLVI(
A, getCtxI()));
8982 const std::string getAsStr(Attributor *
A)
const override {
8984 llvm::raw_string_ostream OS(Str);
8986 getKnown().print(OS);
8988 getAssumed().print(OS);
8995 const SCEV *getSCEV(Attributor &
A,
const Instruction *
I =
nullptr)
const {
8996 if (!getAnchorScope())
8999 ScalarEvolution *SE =
9000 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9003 LoopInfo *LI =
A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
9009 const SCEV *S = SE->
getSCEV(&getAssociatedValue());
9018 ConstantRange getConstantRangeFromSCEV(Attributor &
A,
9019 const Instruction *
I =
nullptr)
const {
9020 if (!getAnchorScope())
9023 ScalarEvolution *SE =
9024 A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
9027 const SCEV *S = getSCEV(
A,
I);
9037 getConstantRangeFromLVI(Attributor &
A,
9038 const Instruction *CtxI =
nullptr)
const {
9039 if (!getAnchorScope())
9042 LazyValueInfo *LVI =
9043 A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
9058 bool isValidCtxInstructionForOutsideAnalysis(Attributor &
A,
9059 const Instruction *CtxI,
9060 bool AllowAACtxI)
const {
9061 if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
9073 InformationCache &InfoCache =
A.getInfoCache();
9074 const DominatorTree *DT =
9084 getAssumedConstantRange(Attributor &
A,
9085 const Instruction *CtxI =
nullptr)
const override {
9090 if (!isValidCtxInstructionForOutsideAnalysis(
A, CtxI,
9092 return getAssumed();
9094 ConstantRange LVIR = getConstantRangeFromLVI(
A, CtxI);
9095 ConstantRange SCEVR = getConstantRangeFromSCEV(
A, CtxI);
9096 return getAssumed().intersectWith(SCEVR).intersectWith(LVIR);
9101 getMDNodeForConstantRange(
Type *Ty, LLVMContext &Ctx,
9102 const ConstantRange &AssumedConstantRange) {
9104 Ty, AssumedConstantRange.
getLower())),
9106 Ty, AssumedConstantRange.
getUpper()))};
9111 static bool isBetterRange(
const ConstantRange &Assumed,
9112 const Instruction &
I) {
9116 std::optional<ConstantRange>
Known;
9120 }
else if (MDNode *KnownRanges =
I.getMetadata(LLVMContext::MD_range)) {
9126 if (KnownRanges->getNumOperands() > 2)
9129 ConstantInt *
Lower =
9131 ConstantInt *
Upper =
9141 setRangeMetadataIfisBetterRange(Instruction *
I,
9142 const ConstantRange &AssumedConstantRange) {
9143 if (isBetterRange(AssumedConstantRange, *
I)) {
9144 I->setMetadata(LLVMContext::MD_range,
9145 getMDNodeForConstantRange(
I->getType(),
I->getContext(),
9146 AssumedConstantRange));
9153 setRangeRetAttrIfisBetterRange(Attributor &
A,
const IRPosition &IRP,
9155 const ConstantRange &AssumedConstantRange) {
9156 if (isBetterRange(AssumedConstantRange, *
I)) {
9157 A.manifestAttrs(IRP,
9158 Attribute::get(
I->getContext(), Attribute::Range,
9159 AssumedConstantRange),
9168 ConstantRange AssumedConstantRange = getAssumedConstantRange(
A);
9171 auto &
V = getAssociatedValue();
9175 assert(
I == getCtxI() &&
"Should not annotate an instruction which is "
9176 "not the context instruction");
9178 if (setRangeMetadataIfisBetterRange(
I, AssumedConstantRange))
9179 Changed = ChangeStatus::CHANGED;
9181 if (setRangeRetAttrIfisBetterRange(
A, getIRPosition(),
I,
9182 AssumedConstantRange))
9183 Changed = ChangeStatus::CHANGED;
9191struct AAValueConstantRangeArgument final
9192 : AAArgumentFromCallSiteArguments<
9193 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9195 using Base = AAArgumentFromCallSiteArguments<
9196 AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
9198 AAValueConstantRangeArgument(
const IRPosition &IRP, Attributor &
A)
9201 void trackStatistics()
const override {
9206struct AAValueConstantRangeReturned
9207 : AAReturnedFromReturnedValues<AAValueConstantRange,
9208 AAValueConstantRangeImpl,
9209 AAValueConstantRangeImpl::StateType,
9212 AAReturnedFromReturnedValues<AAValueConstantRange,
9213 AAValueConstantRangeImpl,
9214 AAValueConstantRangeImpl::StateType,
9216 AAValueConstantRangeReturned(
const IRPosition &IRP, Attributor &
A)
9220 if (!
A.isFunctionIPOAmendable(*getAssociatedFunction()))
9221 indicatePessimisticFixpoint();
9224 void trackStatistics()
const override {
9229struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
9230 AAValueConstantRangeFloating(
const IRPosition &IRP, Attributor &
A)
9231 : AAValueConstantRangeImpl(IRP,
A) {}
9234 AAValueConstantRangeImpl::initialize(
A);
9238 Value &
V = getAssociatedValue();
9241 unionAssumed(ConstantRange(
C->getValue()));
9242 indicateOptimisticFixpoint();
9248 unionAssumed(ConstantRange(APInt(
getBitWidth(), 0)));
9249 indicateOptimisticFixpoint();
9261 if (
auto *RangeMD = LI->getMetadata(LLVMContext::MD_range)) {
9272 indicatePessimisticFixpoint();
9275 << getAssociatedValue() <<
"\n");
9278 bool calculateBinaryOperator(
9279 Attributor &
A, BinaryOperator *BinOp, IntegerRangeState &
T,
9280 const Instruction *CtxI,
9281 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9286 bool UsedAssumedInformation =
false;
9287 const auto &SimplifiedLHS =
A.getAssumedSimplified(
9290 if (!SimplifiedLHS.has_value())
9292 if (!*SimplifiedLHS)
9294 LHS = *SimplifiedLHS;
9296 const auto &SimplifiedRHS =
A.getAssumedSimplified(
9299 if (!SimplifiedRHS.has_value())
9301 if (!*SimplifiedRHS)
9303 RHS = *SimplifiedRHS;
9309 auto *LHSAA =
A.getAAFor<AAValueConstantRange>(
9311 DepClassTy::REQUIRED);
9315 auto LHSAARange = LHSAA->getAssumedConstantRange(
A, CtxI);
9317 auto *RHSAA =
A.getAAFor<AAValueConstantRange>(
9319 DepClassTy::REQUIRED);
9323 auto RHSAARange = RHSAA->getAssumedConstantRange(
A, CtxI);
9325 auto AssumedRange = LHSAARange.binaryOp(BinOp->
getOpcode(), RHSAARange);
9327 T.unionAssumed(AssumedRange);
9331 return T.isValidState();
9334 bool calculateCastInst(
9335 Attributor &
A, CastInst *CastI, IntegerRangeState &
T,
9336 const Instruction *CtxI,
9337 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9343 bool UsedAssumedInformation =
false;
9344 const auto &SimplifiedOpV =
A.getAssumedSimplified(
9347 if (!SimplifiedOpV.has_value())
9349 if (!*SimplifiedOpV)
9351 OpV = *SimplifiedOpV;
9356 auto *OpAA =
A.getAAFor<AAValueConstantRange>(
9358 DepClassTy::REQUIRED);
9362 T.unionAssumed(OpAA->getAssumed().castOp(CastI->
getOpcode(),
9364 return T.isValidState();
9368 calculateCmpInst(Attributor &
A, CmpInst *CmpI, IntegerRangeState &
T,
9369 const Instruction *CtxI,
9370 SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
9375 bool UsedAssumedInformation =
false;
9376 const auto &SimplifiedLHS =
A.getAssumedSimplified(
9379 if (!SimplifiedLHS.has_value())
9381 if (!*SimplifiedLHS)
9383 LHS = *SimplifiedLHS;
9385 const auto &SimplifiedRHS =
A.getAssumedSimplified(
9388 if (!SimplifiedRHS.has_value())
9390 if (!*SimplifiedRHS)
9392 RHS = *SimplifiedRHS;
9398 auto *LHSAA =
A.getAAFor<AAValueConstantRange>(
9400 DepClassTy::REQUIRED);
9404 auto *RHSAA =
A.getAAFor<AAValueConstantRange>(
9406 DepClassTy::REQUIRED);
9410 auto LHSAARange = LHSAA->getAssumedConstantRange(
A, CtxI);
9411 auto RHSAARange = RHSAA->getAssumedConstantRange(
A, CtxI);
9414 if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
9417 bool MustTrue =
false, MustFalse =
false;
9419 auto AllowedRegion =
9422 if (AllowedRegion.intersectWith(LHSAARange).isEmptySet())
9428 assert((!MustTrue || !MustFalse) &&
9429 "Either MustTrue or MustFalse should be false!");
9432 T.unionAssumed(ConstantRange(APInt( 1, 1)));
9434 T.unionAssumed(ConstantRange(APInt( 1, 0)));
9436 T.unionAssumed(ConstantRange( 1,
true));
9438 LLVM_DEBUG(
dbgs() <<
"[AAValueConstantRange] " << *CmpI <<
" after "
9439 << (MustTrue ?
"true" : (MustFalse ?
"false" :
"unknown"))
9440 <<
": " <<
T <<
"\n\t" << *LHSAA <<
"\t<op>\n\t"
9444 return T.isValidState();
9455 bool UsedAssumedInformation =
false;
9456 const auto &SimplifiedOpV =
A.getAssumedSimplified(
9459 if (!SimplifiedOpV.has_value())
9461 if (!*SimplifiedOpV)
9463 Value *VPtr = *SimplifiedOpV;
9466 const auto *AA =
A.getAAFor<AAValueConstantRange>(
9468 DepClassTy::REQUIRED);
9472 T.unionAssumed(AA->getAssumedConstantRange(
A, CtxI));
9476 return T.isValidState();
9481 if (!calculateBinaryOperator(
A, BinOp,
T, CtxI, QuerriedAAs))
9484 if (!calculateCmpInst(
A, CmpI,
T, CtxI, QuerriedAAs))
9487 if (!calculateCastInst(
A, CastI,
T, CtxI, QuerriedAAs))
9493 T.indicatePessimisticFixpoint();
9500 for (
const AAValueConstantRange *QueriedAA : QuerriedAAs) {
9501 if (QueriedAA !=
this)
9504 if (
T.getAssumed() == getState().getAssumed())
9506 T.indicatePessimisticFixpoint();
9509 return T.isValidState();
9512 if (!VisitValueCB(getAssociatedValue(), getCtxI()))
9513 return indicatePessimisticFixpoint();
9518 return ChangeStatus::UNCHANGED;
9519 if (++NumChanges > MaxNumChanges) {
9520 LLVM_DEBUG(
dbgs() <<
"[AAValueConstantRange] performed " << NumChanges
9521 <<
" but only " << MaxNumChanges
9522 <<
" are allowed to avoid cyclic reasoning.");
9523 return indicatePessimisticFixpoint();
9525 return ChangeStatus::CHANGED;
9528 void trackStatistics()
const override {
9537 static constexpr int MaxNumChanges = 5;
9540struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
9541 AAValueConstantRangeFunction(
const IRPosition &IRP, Attributor &
A)
9542 : AAValueConstantRangeImpl(IRP,
A) {}
9545 llvm_unreachable(
"AAValueConstantRange(Function|CallSite)::updateImpl will "
9552struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
9553 AAValueConstantRangeCallSite(
const IRPosition &IRP, Attributor &
A)
9554 : AAValueConstantRangeFunction(IRP,
A) {}
9559struct AAValueConstantRangeCallSiteReturned
9560 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9561 AAValueConstantRangeImpl::StateType,
9563 AAValueConstantRangeCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
9564 : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
9565 AAValueConstantRangeImpl::StateType,
9571 if (std::optional<ConstantRange>
Range = CI->getRange())
9572 intersectKnown(*
Range);
9575 AAValueConstantRangeImpl::initialize(
A);
9578 void trackStatistics()
const override {
9582struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
9583 AAValueConstantRangeCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
9584 : AAValueConstantRangeFloating(IRP,
A) {}
9587 return ChangeStatus::UNCHANGED;
9590 void trackStatistics()
const override {
9600 :
Base(IRP,
true,
A.Opts.attributor_max_potential_values) {}
9610 if (
A.hasSimplificationCallback(getIRPosition()))
9611 indicatePessimisticFixpoint();
9616 bool fillSetWithConstantValues(Attributor &
A,
const IRPosition &IRP, SetTy &S,
9617 bool &ContainsUndef,
bool ForSelf) {
9619 bool UsedAssumedInformation =
false;
9621 UsedAssumedInformation)) {
9628 auto *PotentialValuesAA =
A.getAAFor<AAPotentialConstantValues>(
9629 *
this, IRP, DepClassTy::REQUIRED);
9630 if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
9632 ContainsUndef = PotentialValuesAA->getState().undefIsContained();
9633 S = PotentialValuesAA->getState().getAssumedSet();
9640 ContainsUndef =
false;
9641 for (
auto &It :
Values) {
9643 ContainsUndef =
true;
9649 S.insert(CI->getValue());
9651 ContainsUndef &= S.empty();
9656 const std::string getAsStr(Attributor *
A)
const override {
9658 llvm::raw_string_ostream OS(Str);
9664 return indicatePessimisticFixpoint();
9668struct AAPotentialConstantValuesArgument final
9669 : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9670 AAPotentialConstantValuesImpl,
9671 PotentialConstantIntValuesState> {
9672 using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
9673 AAPotentialConstantValuesImpl,
9675 AAPotentialConstantValuesArgument(
const IRPosition &IRP, Attributor &
A)
9678 void trackStatistics()
const override {
9683struct AAPotentialConstantValuesReturned
9684 : AAReturnedFromReturnedValues<AAPotentialConstantValues,
9685 AAPotentialConstantValuesImpl> {
9686 using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
9687 AAPotentialConstantValuesImpl>;
9688 AAPotentialConstantValuesReturned(
const IRPosition &IRP, Attributor &
A)
9692 if (!
A.isFunctionIPOAmendable(*getAssociatedFunction()))
9693 indicatePessimisticFixpoint();
9694 Base::initialize(
A);
9697 void trackStatistics()
const override {
9702struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
9703 AAPotentialConstantValuesFloating(
const IRPosition &IRP, Attributor &
A)
9704 : AAPotentialConstantValuesImpl(IRP,
A) {}
9707 AAPotentialConstantValuesImpl::initialize(
A);
9711 Value &
V = getAssociatedValue();
9714 unionAssumed(
C->getValue());
9715 indicateOptimisticFixpoint();
9720 unionAssumedWithUndef();
9721 indicateOptimisticFixpoint();
9731 indicatePessimisticFixpoint();
9734 << getAssociatedValue() <<
"\n");
9737 static bool calculateICmpInst(
const ICmpInst *ICI,
const APInt &
LHS,
9742 static APInt calculateCastInst(
const CastInst *CI,
const APInt &Src,
9743 uint32_t ResultBitWidth) {
9748 case Instruction::Trunc:
9749 return Src.trunc(ResultBitWidth);
9750 case Instruction::SExt:
9751 return Src.sext(ResultBitWidth);
9752 case Instruction::ZExt:
9753 return Src.zext(ResultBitWidth);
9754 case Instruction::BitCast:
9759 static APInt calculateBinaryOperator(
const BinaryOperator *BinOp,
9760 const APInt &
LHS,
const APInt &
RHS,
9761 bool &SkipOperation,
bool &Unsupported) {
9768 switch (BinOpcode) {
9772 case Instruction::Add:
9774 case Instruction::Sub:
9776 case Instruction::Mul:
9778 case Instruction::UDiv:
9780 SkipOperation =
true;
9784 case Instruction::SDiv:
9786 SkipOperation =
true;
9790 case Instruction::URem:
9792 SkipOperation =
true;
9796 case Instruction::SRem:
9798 SkipOperation =
true;
9802 case Instruction::Shl:
9804 case Instruction::LShr:
9806 case Instruction::AShr:
9808 case Instruction::And:
9810 case Instruction::Or:
9812 case Instruction::Xor:
9817 bool calculateBinaryOperatorAndTakeUnion(
const BinaryOperator *BinOp,
9818 const APInt &
LHS,
const APInt &
RHS) {
9819 bool SkipOperation =
false;
9822 calculateBinaryOperator(BinOp,
LHS,
RHS, SkipOperation, Unsupported);
9827 unionAssumed(Result);
9828 return isValidState();
9831 ChangeStatus updateWithICmpInst(Attributor &
A, ICmpInst *ICI) {
9832 auto AssumedBefore = getAssumed();
9836 bool LHSContainsUndef =
false, RHSContainsUndef =
false;
9837 SetTy LHSAAPVS, RHSAAPVS;
9839 LHSContainsUndef,
false) ||
9841 RHSContainsUndef,
false))
9842 return indicatePessimisticFixpoint();
9845 bool MaybeTrue =
false, MaybeFalse =
false;
9847 if (LHSContainsUndef && RHSContainsUndef) {
9850 unionAssumedWithUndef();
9851 }
else if (LHSContainsUndef) {
9852 for (
const APInt &R : RHSAAPVS) {
9853 bool CmpResult = calculateICmpInst(ICI,
Zero, R);
9854 MaybeTrue |= CmpResult;
9855 MaybeFalse |= !CmpResult;
9856 if (MaybeTrue & MaybeFalse)
9857 return indicatePessimisticFixpoint();
9859 }
else if (RHSContainsUndef) {
9860 for (
const APInt &L : LHSAAPVS) {
9861 bool CmpResult = calculateICmpInst(ICI, L,
Zero);
9862 MaybeTrue |= CmpResult;
9863 MaybeFalse |= !CmpResult;
9864 if (MaybeTrue & MaybeFalse)
9865 return indicatePessimisticFixpoint();
9868 for (
const APInt &L : LHSAAPVS) {
9869 for (
const APInt &R : RHSAAPVS) {
9870 bool CmpResult = calculateICmpInst(ICI, L, R);
9871 MaybeTrue |= CmpResult;
9872 MaybeFalse |= !CmpResult;
9873 if (MaybeTrue & MaybeFalse)
9874 return indicatePessimisticFixpoint();
9879 unionAssumed(APInt( 1, 1));
9881 unionAssumed(APInt( 1, 0));
9882 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9883 : ChangeStatus::CHANGED;
9886 ChangeStatus updateWithSelectInst(Attributor &
A, SelectInst *SI) {
9887 auto AssumedBefore = getAssumed();
9891 bool UsedAssumedInformation =
false;
9892 std::optional<Constant *>
C =
A.getAssumedConstant(
9893 *
SI->getCondition(), *
this, UsedAssumedInformation);
9896 bool OnlyLeft =
false, OnlyRight =
false;
9897 if (
C && *
C && (*C)->isOneValue())
9899 else if (
C && *
C && (*C)->isNullValue())
9902 bool LHSContainsUndef =
false, RHSContainsUndef =
false;
9903 SetTy LHSAAPVS, RHSAAPVS;
9906 LHSContainsUndef,
false))
9907 return indicatePessimisticFixpoint();
9911 RHSContainsUndef,
false))
9912 return indicatePessimisticFixpoint();
9914 if (OnlyLeft || OnlyRight) {
9916 auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
9917 auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
9920 unionAssumedWithUndef();
9922 for (
const auto &It : *OpAA)
9926 }
else if (LHSContainsUndef && RHSContainsUndef) {
9928 unionAssumedWithUndef();
9930 for (
const auto &It : LHSAAPVS)
9932 for (
const auto &It : RHSAAPVS)
9935 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9936 : ChangeStatus::CHANGED;
9939 ChangeStatus updateWithCastInst(Attributor &
A, CastInst *CI) {
9940 auto AssumedBefore = getAssumed();
9942 return indicatePessimisticFixpoint();
9947 bool SrcContainsUndef =
false;
9950 SrcContainsUndef,
false))
9951 return indicatePessimisticFixpoint();
9953 if (SrcContainsUndef)
9954 unionAssumedWithUndef();
9956 for (
const APInt &S : SrcPVS) {
9957 APInt
T = calculateCastInst(CI, S, ResultBitWidth);
9961 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
9962 : ChangeStatus::CHANGED;
9965 ChangeStatus updateWithBinaryOperator(Attributor &
A, BinaryOperator *BinOp) {
9966 auto AssumedBefore = getAssumed();
9970 bool LHSContainsUndef =
false, RHSContainsUndef =
false;
9971 SetTy LHSAAPVS, RHSAAPVS;
9973 LHSContainsUndef,
false) ||
9975 RHSContainsUndef,
false))
9976 return indicatePessimisticFixpoint();
9981 if (LHSContainsUndef && RHSContainsUndef) {
9982 if (!calculateBinaryOperatorAndTakeUnion(BinOp,
Zero,
Zero))
9983 return indicatePessimisticFixpoint();
9984 }
else if (LHSContainsUndef) {
9985 for (
const APInt &R : RHSAAPVS) {
9986 if (!calculateBinaryOperatorAndTakeUnion(BinOp,
Zero, R))
9987 return indicatePessimisticFixpoint();
9989 }
else if (RHSContainsUndef) {
9990 for (
const APInt &L : LHSAAPVS) {
9991 if (!calculateBinaryOperatorAndTakeUnion(BinOp, L,
Zero))
9992 return indicatePessimisticFixpoint();
9995 for (
const APInt &L : LHSAAPVS) {
9996 for (
const APInt &R : RHSAAPVS) {
9997 if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, R))
9998 return indicatePessimisticFixpoint();
10002 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10003 : ChangeStatus::CHANGED;
10006 ChangeStatus updateWithInstruction(Attributor &
A, Instruction *Inst) {
10007 auto AssumedBefore = getAssumed();
10009 bool ContainsUndef;
10011 ContainsUndef,
true))
10012 return indicatePessimisticFixpoint();
10013 if (ContainsUndef) {
10014 unionAssumedWithUndef();
10016 for (
const auto &It : Incoming)
10019 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10020 : ChangeStatus::CHANGED;
10024 Value &
V = getAssociatedValue();
10028 return updateWithICmpInst(
A, ICI);
10031 return updateWithSelectInst(
A, SI);
10034 return updateWithCastInst(
A, CI);
10037 return updateWithBinaryOperator(
A, BinOp);
10040 return updateWithInstruction(
A,
I);
10042 return indicatePessimisticFixpoint();
10045 void trackStatistics()
const override {
10050struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
10051 AAPotentialConstantValuesFunction(
const IRPosition &IRP, Attributor &
A)
10052 : AAPotentialConstantValuesImpl(IRP,
A) {}
10056 "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
10060 void trackStatistics()
const override {
10065struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
10066 AAPotentialConstantValuesCallSite(
const IRPosition &IRP, Attributor &
A)
10067 : AAPotentialConstantValuesFunction(IRP,
A) {}
10069 void trackStatistics()
const override {
10074struct AAPotentialConstantValuesCallSiteReturned
10075 : AACalleeToCallSite<AAPotentialConstantValues,
10076 AAPotentialConstantValuesImpl> {
10077 AAPotentialConstantValuesCallSiteReturned(
const IRPosition &IRP,
10079 : AACalleeToCallSite<AAPotentialConstantValues,
10080 AAPotentialConstantValuesImpl>(IRP,
A) {}
10082 void trackStatistics()
const override {
10087struct AAPotentialConstantValuesCallSiteArgument
10088 : AAPotentialConstantValuesFloating {
10089 AAPotentialConstantValuesCallSiteArgument(
const IRPosition &IRP,
10091 : AAPotentialConstantValuesFloating(IRP,
A) {}
10094 AAPotentialConstantValuesImpl::initialize(
A);
10095 if (isAtFixpoint())
10098 Value &
V = getAssociatedValue();
10101 unionAssumed(
C->getValue());
10102 indicateOptimisticFixpoint();
10107 unionAssumedWithUndef();
10108 indicateOptimisticFixpoint();
10114 Value &
V = getAssociatedValue();
10115 auto AssumedBefore = getAssumed();
10116 auto *AA =
A.getAAFor<AAPotentialConstantValues>(
10119 return indicatePessimisticFixpoint();
10120 const auto &S = AA->getAssumed();
10122 return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
10123 : ChangeStatus::CHANGED;
10126 void trackStatistics()
const override {
10135 bool IgnoreSubsumingPositions) {
10136 assert(ImpliedAttributeKind == Attribute::NoUndef &&
10137 "Unexpected attribute kind");
10138 if (
A.hasAttr(IRP, {Attribute::NoUndef}, IgnoreSubsumingPositions,
10139 Attribute::NoUndef))
10158 Value &
V = getAssociatedValue();
10160 indicatePessimisticFixpoint();
10161 assert(!isImpliedByIR(
A, getIRPosition(), Attribute::NoUndef));
10165 bool followUseInMBEC(Attributor &
A,
const Use *U,
const Instruction *
I,
10166 AANoUndef::StateType &State) {
10167 const Value *UseV =
U->get();
10168 const DominatorTree *DT =
nullptr;
10169 AssumptionCache *AC =
nullptr;
10170 InformationCache &InfoCache =
A.getInfoCache();
10176 bool TrackUse =
false;
10184 const std::string getAsStr(Attributor *
A)
const override {
10185 return getAssumed() ?
"noundef" :
"may-undef-or-poison";
10192 bool UsedAssumedInformation =
false;
10193 if (
A.isAssumedDead(getIRPosition(),
nullptr,
nullptr,
10194 UsedAssumedInformation))
10195 return ChangeStatus::UNCHANGED;
10199 if (!
A.getAssumedSimplified(getIRPosition(), *
this, UsedAssumedInformation,
10202 return ChangeStatus::UNCHANGED;
10203 return AANoUndef::manifest(
A);
10207struct AANoUndefFloating :
public AANoUndefImpl {
10208 AANoUndefFloating(
const IRPosition &IRP, Attributor &
A)
10209 : AANoUndefImpl(IRP,
A) {}
10212 AANoUndefImpl::initialize(
A);
10213 if (!getState().isAtFixpoint() && getAnchorScope() &&
10214 !getAnchorScope()->isDeclaration())
10215 if (Instruction *CtxI = getCtxI())
10216 followUsesInMBEC(*
this,
A, getState(), *CtxI);
10220 auto VisitValueCB = [&](
const IRPosition &IRP) ->
bool {
10221 bool IsKnownNoUndef;
10223 A,
this, IRP, DepClassTy::REQUIRED, IsKnownNoUndef);
10227 bool UsedAssumedInformation =
false;
10228 Value *AssociatedValue = &getAssociatedValue();
10230 if (!
A.getAssumedSimplifiedValues(getIRPosition(), *
this,
Values,
10235 Values.size() != 1 ||
Values.front().getValue() != AssociatedValue;
10243 if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
10244 return indicatePessimisticFixpoint();
10245 return ChangeStatus::UNCHANGED;
10248 for (
const auto &VAC :
Values)
10250 return indicatePessimisticFixpoint();
10252 return ChangeStatus::UNCHANGED;
10260static bool checkPoisonGeneratingAttrsHold(
Attributor &
A,
10265 AttributeMask AM = AttributeFuncs::getPoisonGeneratingAttributes();
10271 case Attribute::NonNull: {
10278 case Attribute::Alignment: {
10279 const auto *AlignAA =
10281 if (!AlignAA || AlignAA->getAssumedAlign() < *Attr.
getAlignment())
10285 case Attribute::Range: {
10293 case Attribute::NoFPClass: {
10295 const auto *NoFPClassAA =
10297 if (!NoFPClassAA ||
10298 (NoFPClassAA->getAssumedNoFPClass() & NoFPClass) != NoFPClass)
10311struct AANoUndefReturned final
10312 : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
10313 using Base = AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>;
10314 AANoUndefReturned(
const IRPosition &IRP, Attributor &
A) :
Base(IRP,
A) {}
10317 Base::initialize(
A);
10318 RetAttrs = getAssociatedFunction()->getAttributes().getRetAttrs();
10324 if (RetAttrs.hasAttributes()) {
10327 return checkPoisonGeneratingAttrsHold(
A, *
this, RetAttrs,
10330 bool UsedAssumedInformation =
false;
10331 if (!
A.checkForAllInstructions(AttrsHold, *
this, {Instruction::Ret},
10332 UsedAssumedInformation))
10333 return indicatePessimisticFixpoint();
10336 return Base::updateImpl(
A);
10343 AttributeSet RetAttrs;
10346struct AANoUndefArgument final
10347 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
10348 AANoUndefArgument(
const IRPosition &IRP, Attributor &
A)
10349 : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP,
A) {}
10354struct AANoUndefCallSiteArgument final : AANoUndefFloating {
10355 AANoUndefCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
10356 : AANoUndefFloating(IRP,
A) {}
10361struct AANoUndefCallSiteReturned final
10362 : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
10363 using Base = AACalleeToCallSite<AANoUndef, AANoUndefImpl>;
10364 AANoUndefCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
10368 Base::initialize(
A);
10376 if (RetAttrs.hasAttributes()) {
10379 return checkPoisonGeneratingAttrsHold(
10383 if (!
A.checkForAllCallees(AttrsHold, *
this,
10385 return indicatePessimisticFixpoint();
10388 return Base::updateImpl(
A);
10395 AttributeSet RetAttrs;
10401 AANoFPClassImpl(
const IRPosition &IRP, Attributor &
A) : AANoFPClass(IRP,
A) {}
10404 const IRPosition &IRP = getIRPosition();
10408 indicateOptimisticFixpoint();
10413 A.getAttrs(getIRPosition(), {Attribute::NoFPClass},
Attrs,
false);
10414 for (
const auto &Attr : Attrs) {
10421 const DataLayout &
DL =
A.getDataLayout();
10422 InformationCache &InfoCache =
A.getInfoCache();
10424 const DominatorTree *DT =
nullptr;
10425 AssumptionCache *AC =
nullptr;
10426 const TargetLibraryInfo *
TLI =
nullptr;
10430 if (!
F->isDeclaration()) {
10437 SimplifyQuery Q(
DL,
TLI, DT, AC, CtxI);
10444 followUsesInMBEC(*
this,
A, getState(), *CtxI);
10448 bool followUseInMBEC(Attributor &
A,
const Use *U,
const Instruction *
I,
10449 AANoFPClass::StateType &State) {
10460 if (
auto *NoFPAA =
A.getAAFor<AANoFPClass>(*
this, IRP, DepClassTy::NONE))
10461 State.addKnownBits(NoFPAA->getState().getKnown());
10465 const std::string getAsStr(Attributor *
A)
const override {
10466 std::string
Result =
"nofpclass";
10467 raw_string_ostream OS(Result);
10468 OS << getKnownNoFPClass() <<
'/' << getAssumedNoFPClass();
10472 void getDeducedAttributes(Attributor &
A, LLVMContext &Ctx,
10473 SmallVectorImpl<Attribute> &Attrs)
const override {
10474 Attrs.emplace_back(Attribute::getWithNoFPClass(Ctx, getAssumedNoFPClass()));
10478struct AANoFPClassFloating :
public AANoFPClassImpl {
10479 AANoFPClassFloating(
const IRPosition &IRP, Attributor &
A)
10480 : AANoFPClassImpl(IRP,
A) {}
10484 bool UsedAssumedInformation =
false;
10485 if (!
A.getAssumedSimplifiedValues(getIRPosition(), *
this,
Values,
10487 Values.push_back({getAssociatedValue(), getCtxI()});
10493 DepClassTy::REQUIRED);
10494 if (!AA ||
this == AA) {
10495 T.indicatePessimisticFixpoint();
10497 const AANoFPClass::StateType &S =
10498 static_cast<const AANoFPClass::StateType &
>(AA->
getState());
10501 return T.isValidState();
10504 for (
const auto &VAC :
Values)
10506 return indicatePessimisticFixpoint();
10511 void trackStatistics()
const override {
10516struct AANoFPClassReturned final
10517 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10518 AANoFPClassImpl::StateType, false,
10519 Attribute::None, false> {
10520 AANoFPClassReturned(
const IRPosition &IRP, Attributor &
A)
10521 : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
10522 AANoFPClassImpl::StateType,
false,
10525 void trackStatistics()
const override {
10530struct AANoFPClassArgument final
10531 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
10532 AANoFPClassArgument(
const IRPosition &IRP, Attributor &
A)
10533 : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP,
A) {}
10538struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
10539 AANoFPClassCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
10540 : AANoFPClassFloating(IRP,
A) {}
10542 void trackStatistics()
const override {
10547struct AANoFPClassCallSiteReturned final
10548 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
10549 AANoFPClassCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
10550 : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP,
A) {}
10552 void trackStatistics()
const override {
10558 AACallEdgesImpl(
const IRPosition &IRP, Attributor &
A) : AACallEdges(IRP,
A) {}
10560 const SetVector<Function *> &getOptimisticEdges()
const override {
10561 return CalledFunctions;
10564 bool hasUnknownCallee()
const override {
return HasUnknownCallee; }
10566 bool hasNonAsmUnknownCallee()
const override {
10567 return HasUnknownCalleeNonAsm;
10570 const std::string getAsStr(Attributor *
A)
const override {
10571 return "CallEdges[" + std::to_string(HasUnknownCallee) +
"," +
10572 std::to_string(CalledFunctions.size()) +
"]";
10575 void trackStatistics()
const override {}
10580 static Function *getOnlyCallee(
const CallBase &CB) {
10582 if (Callee && !
Callee->hasMetadata(LLVMContext::MD_callback))
10588 if (CalledFunctions.insert(Fn)) {
10589 Change = ChangeStatus::CHANGED;
10595 void setHasUnknownCallee(
bool NonAsm,
ChangeStatus &Change) {
10596 if (!HasUnknownCallee)
10597 Change = ChangeStatus::CHANGED;
10598 if (NonAsm && !HasUnknownCalleeNonAsm)
10599 Change = ChangeStatus::CHANGED;
10600 HasUnknownCalleeNonAsm |= NonAsm;
10601 HasUnknownCallee =
true;
10606 SetVector<Function *> CalledFunctions;
10609 bool HasUnknownCallee =
false;
10612 bool HasUnknownCalleeNonAsm =
false;
10615struct AACallEdgesCallSite :
public AACallEdgesImpl {
10616 AACallEdgesCallSite(
const IRPosition &IRP, Attributor &
A)
10617 : AACallEdgesImpl(IRP,
A) {}
10623 addCalledFunction(Callee, Change);
10624 indicateOptimisticFixpoint();
10633 addCalledFunction(Fn, Change);
10635 LLVM_DEBUG(
dbgs() <<
"[AACallEdges] Unrecognized value: " <<
V <<
"\n");
10636 setHasUnknownCallee(
true, Change);
10647 VisitValue(*
V, CtxI);
10651 bool UsedAssumedInformation =
false;
10655 Values.push_back({*
V, CtxI});
10657 for (
auto &VAC :
Values)
10664 if (
IA->hasSideEffects() &&
10667 setHasUnknownCallee(
false, Change);
10673 if (
auto *IndirectCallAA =
A.getAAFor<AAIndirectCallInfo>(
10674 *
this, getIRPosition(), DepClassTy::OPTIONAL))
10675 if (IndirectCallAA->foreachCallee(
10676 [&](
Function *Fn) { return VisitValue(*Fn, CB); }))
10685 for (
const Use *U : CallbackUses)
10686 ProcessCalledOperand(
U->get(), CB);
10692struct AACallEdgesFunction :
public AACallEdgesImpl {
10693 AACallEdgesFunction(
const IRPosition &IRP, Attributor &
A)
10694 : AACallEdgesImpl(IRP,
A) {}
10698 bool AllCallsDirect =
true;
10705 if (
Function *Callee = getOnlyCallee(CB)) {
10706 addCalledFunction(Callee, Change);
10710 AllCallsDirect =
false;
10711 auto *CBEdges =
A.getAAFor<AACallEdges>(
10715 if (CBEdges->hasNonAsmUnknownCallee())
10716 setHasUnknownCallee(
true, Change);
10717 if (CBEdges->hasUnknownCallee())
10718 setHasUnknownCallee(
false, Change);
10720 for (
Function *
F : CBEdges->getOptimisticEdges())
10721 addCalledFunction(
F, Change);
10727 bool UsedAssumedInformation =
false;
10728 if (!
A.checkForAllCallLikeInstructions(ProcessCallInst, *
this,
10729 UsedAssumedInformation,
10733 setHasUnknownCallee(
true, Change);
10737 if (AllCallsDirect && !UsedAssumedInformation)
10738 indicateOptimisticFixpoint();
10746struct AAInterFnReachabilityFunction
10747 :
public CachedReachabilityAA<AAInterFnReachability, Function> {
10748 using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
10749 AAInterFnReachabilityFunction(
const IRPosition &IRP, Attributor &
A)
10752 bool instructionCanReach(
10753 Attributor &
A,
const Instruction &From,
const Function &To,
10756 auto *NonConstThis =
const_cast<AAInterFnReachabilityFunction *
>(
this);
10758 RQITy StackRQI(
A, From, To, ExclusionSet,
false);
10759 RQITy::Reachable
Result;
10760 if (!NonConstThis->checkQueryCache(
A, StackRQI, Result))
10761 return NonConstThis->isReachableImpl(
A, StackRQI,
10763 return Result == RQITy::Reachable::Yes;
10767 bool IsTemporaryRQI)
override {
10769 &RQI.From->getFunction()->getEntryBlock().front();
10770 if (EntryI != RQI.From &&
10771 !instructionCanReach(
A, *EntryI, *RQI.To,
nullptr))
10772 return rememberResult(
A, RQITy::Reachable::No, RQI,
false,
10775 auto CheckReachableCallBase = [&](CallBase *CB) {
10776 auto *CBEdges =
A.getAAFor<AACallEdges>(
10778 if (!CBEdges || !CBEdges->getState().isValidState())
10781 if (CBEdges->hasUnknownCallee())
10784 for (
Function *Fn : CBEdges->getOptimisticEdges()) {
10795 if (Fn == getAnchorScope()) {
10796 if (EntryI == RQI.From)
10801 const AAInterFnReachability *InterFnReachability =
10803 DepClassTy::OPTIONAL);
10806 if (!InterFnReachability ||
10814 const auto *IntraFnReachability =
A.getAAFor<AAIntraFnReachability>(
10816 DepClassTy::OPTIONAL);
10824 return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
10825 A, *RQI.From, CBInst, RQI.ExclusionSet);
10828 bool UsedExclusionSet =
true;
10829 bool UsedAssumedInformation =
false;
10830 if (!
A.checkForAllCallLikeInstructions(CheckCallBase, *
this,
10831 UsedAssumedInformation,
10833 return rememberResult(
A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
10836 return rememberResult(
A, RQITy::Reachable::No, RQI, UsedExclusionSet,
10840 void trackStatistics()
const override {}
10844template <
typename AAType>
10845static std::optional<Constant *>
10848 if (!Ty.isIntegerTy())
10856 std::optional<Constant *> COpt =
AA->getAssumedConstant(
A);
10858 if (!COpt.has_value()) {
10860 return std::nullopt;
10862 if (
auto *
C = *COpt) {
10873 std::optional<Value *> V;
10874 for (
auto &It :
Values) {
10876 if (V.has_value() && !*V)
10879 if (!V.has_value())
10892 if (
A.hasSimplificationCallback(getIRPosition())) {
10893 indicatePessimisticFixpoint();
10896 Value *Stripped = getAssociatedValue().stripPointerCasts();
10898 addValue(
A, getState(), *Stripped, getCtxI(),
AA::AnyScope,
10900 indicateOptimisticFixpoint();
10903 AAPotentialValues::initialize(
A);
10906 const std::string getAsStr(Attributor *
A)
const override {
10908 llvm::raw_string_ostream OS(Str);
10913 template <
typename AAType>
10914 static std::optional<Value *> askOtherAA(Attributor &
A,
10915 const AbstractAttribute &AA,
10916 const IRPosition &IRP,
Type &Ty) {
10921 return std::nullopt;
10928 virtual void addValue(Attributor &
A, StateType &State,
Value &
V,
10934 for (
const auto &U : CB->
args()) {
10944 Type &Ty = *getAssociatedType();
10945 std::optional<Value *> SimpleV =
10946 askOtherAA<AAValueConstantRange>(
A, *
this, ValIRP, Ty);
10947 if (SimpleV.has_value() && !*SimpleV) {
10948 auto *PotentialConstantsAA =
A.getAAFor<AAPotentialConstantValues>(
10949 *
this, ValIRP, DepClassTy::OPTIONAL);
10950 if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
10951 for (
const auto &It : PotentialConstantsAA->getAssumedSet())
10952 State.unionAssumed({{*ConstantInt::get(&Ty, It),
nullptr}, S});
10953 if (PotentialConstantsAA->undefIsContained())
10958 if (!SimpleV.has_value())
10970 State.unionAssumed({{*VPtr, CtxI}, S});
10976 AA::ValueAndContext
I;
10980 return II.I ==
I &&
II.S == S;
10983 return std::tie(
I, S) < std::tie(
II.I,
II.S);
10987 bool recurseForValue(Attributor &
A,
const IRPosition &IRP,
AA::ValueScope S) {
10988 SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
10993 bool UsedAssumedInformation =
false;
10995 if (!
A.getAssumedSimplifiedValues(IRP,
this,
Values, CS,
10996 UsedAssumedInformation))
11000 ValueScopeMap[It] += CS;
11002 for (
auto &It : ValueScopeMap)
11003 addValue(
A, getState(), *It.first.getValue(), It.first.getCtxI(),
11009 void giveUpOnIntraprocedural(Attributor &
A) {
11010 auto NewS = StateType::getBestState(getState());
11011 for (
const auto &It : getAssumedSet()) {
11014 addValue(
A, NewS, *It.first.getValue(), It.first.getCtxI(),
11017 assert(!undefIsContained() &&
"Undef should be an explicit value!");
11024 getState() = StateType::getBestState(getState());
11025 getState().unionAssumed({{getAssociatedValue(), getCtxI()},
AA::AnyScope});
11026 AAPotentialValues::indicateOptimisticFixpoint();
11027 return ChangeStatus::CHANGED;
11031 return indicatePessimisticFixpoint();
11038 if (!getAssumedSimplifiedValues(
A,
Values, S))
11040 Value &OldV = getAssociatedValue();
11043 Value *NewV = getSingleValue(
A, *
this, getIRPosition(),
Values);
11044 if (!NewV || NewV == &OldV)
11049 if (
A.changeAfterManifest(getIRPosition(), *NewV))
11050 return ChangeStatus::CHANGED;
11052 return ChangeStatus::UNCHANGED;
11055 bool getAssumedSimplifiedValues(
11056 Attributor &
A, SmallVectorImpl<AA::ValueAndContext> &
Values,
11057 AA::ValueScope S,
bool RecurseForSelectAndPHI =
false)
const override {
11058 if (!isValidState())
11060 bool UsedAssumedInformation =
false;
11061 for (
const auto &It : getAssumedSet())
11062 if (It.second & S) {
11063 if (RecurseForSelectAndPHI && (
isa<PHINode>(It.first.getValue()) ||
11065 if (
A.getAssumedSimplifiedValues(
11067 this,
Values, S, UsedAssumedInformation))
11070 Values.push_back(It.first);
11072 assert(!undefIsContained() &&
"Undef should be an explicit value!");
11077struct AAPotentialValuesFloating : AAPotentialValuesImpl {
11078 AAPotentialValuesFloating(
const IRPosition &IRP, Attributor &
A)
11079 : AAPotentialValuesImpl(IRP,
A) {}
11082 auto AssumedBefore = getAssumed();
11084 genericValueTraversal(
A, &getAssociatedValue());
11086 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11087 : ChangeStatus::CHANGED;
11091 struct LivenessInfo {
11092 const AAIsDead *LivenessAA =
nullptr;
11093 bool AnyDead =
false;
11103 SmallVectorImpl<ItemInfo> &Worklist) {
11106 bool UsedAssumedInformation =
false;
11108 auto GetSimplifiedValues = [&](
Value &
V,
11110 if (!
A.getAssumedSimplifiedValues(
11114 Values.push_back(AA::ValueAndContext{
V,
II.I.getCtxI()});
11118 if (GetSimplifiedValues(*
LHS, LHSValues))
11120 if (GetSimplifiedValues(*
RHS, RHSValues))
11125 InformationCache &InfoCache =
A.getInfoCache();
11132 F ?
A.getInfoCache().getTargetLibraryInfoForFunction(*
F) :
nullptr;
11137 const DataLayout &
DL =
A.getDataLayout();
11138 SimplifyQuery Q(
DL,
TLI, DT, AC, CmpI);
11140 auto CheckPair = [&](
Value &LHSV,
Value &RHSV) {
11143 nullptr,
II.S, getAnchorScope());
11149 if (&LHSV == &RHSV &&
11151 Constant *NewV = ConstantInt::get(Type::getInt1Ty(Ctx),
11153 addValue(
A, getState(), *NewV,
nullptr,
II.S,
11160 if (TypedLHS && TypedRHS) {
11162 if (NewV && NewV != &Cmp) {
11163 addValue(
A, getState(), *NewV,
nullptr,
II.S,
11175 if (!LHSIsNull && !RHSIsNull)
11181 assert((LHSIsNull || RHSIsNull) &&
11182 "Expected nullptr versus non-nullptr comparison at this point");
11185 unsigned PtrIdx = LHSIsNull;
11186 bool IsKnownNonNull;
11189 DepClassTy::REQUIRED, IsKnownNonNull);
11190 if (!IsAssumedNonNull)
11196 addValue(
A, getState(), *NewV,
nullptr,
II.S,
11201 for (
auto &LHSValue : LHSValues)
11202 for (
auto &RHSValue : RHSValues)
11203 if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
11208 bool handleSelectInst(Attributor &
A, SelectInst &SI, ItemInfo
II,
11209 SmallVectorImpl<ItemInfo> &Worklist) {
11211 bool UsedAssumedInformation =
false;
11213 std::optional<Constant *>
C =
11214 A.getAssumedConstant(*
SI.getCondition(), *
this, UsedAssumedInformation);
11215 bool NoValueYet = !
C.has_value();
11223 }
else if (&SI == &getAssociatedValue()) {
11228 std::optional<Value *> SimpleV =
A.getAssumedSimplified(
11230 if (!SimpleV.has_value())
11233 addValue(
A, getState(), **SimpleV, CtxI,
II.S, getAnchorScope());
11241 bool handleLoadInst(Attributor &
A, LoadInst &LI, ItemInfo
II,
11242 SmallVectorImpl<ItemInfo> &Worklist) {
11243 SmallSetVector<Value *, 4> PotentialCopies;
11244 SmallSetVector<Instruction *, 4> PotentialValueOrigins;
11245 bool UsedAssumedInformation =
false;
11247 PotentialValueOrigins, *
this,
11248 UsedAssumedInformation,
11250 LLVM_DEBUG(
dbgs() <<
"[AAPotentialValues] Failed to get potentially "
11251 "loaded values for load instruction "
11259 InformationCache &InfoCache =
A.getInfoCache();
11261 if (!
llvm::all_of(PotentialValueOrigins, [&](Instruction *
I) {
11265 return A.isAssumedDead(
SI->getOperandUse(0),
this,
11267 UsedAssumedInformation,
11269 return A.isAssumedDead(*
I,
this,
nullptr,
11270 UsedAssumedInformation,
11273 LLVM_DEBUG(
dbgs() <<
"[AAPotentialValues] Load is onl used by assumes "
11274 "and we cannot delete all the stores: "
11285 bool AllLocal = ScopeIsLocal;
11290 if (!DynamicallyUnique) {
11291 LLVM_DEBUG(
dbgs() <<
"[AAPotentialValues] Not all potentially loaded "
11292 "values are dynamically unique: "
11297 for (
auto *PotentialCopy : PotentialCopies) {
11299 Worklist.
push_back({{*PotentialCopy, CtxI},
II.S});
11304 if (!AllLocal && ScopeIsLocal)
11309 bool handlePHINode(
11310 Attributor &
A, PHINode &
PHI, ItemInfo
II,
11311 SmallVectorImpl<ItemInfo> &Worklist,
11312 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11313 auto GetLivenessInfo = [&](
const Function &
F) -> LivenessInfo & {
11314 LivenessInfo &LI = LivenessAAs[&
F];
11315 if (!LI.LivenessAA)
11321 if (&
PHI == &getAssociatedValue()) {
11322 LivenessInfo &LI = GetLivenessInfo(*
PHI.getFunction());
11324 A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
11325 *
PHI.getFunction());
11329 for (
unsigned u = 0, e =
PHI.getNumIncomingValues(); u < e; u++) {
11331 if (LI.LivenessAA &&
11332 LI.LivenessAA->isEdgeDead(IncomingBB,
PHI.getParent())) {
11351 bool UsedAssumedInformation =
false;
11352 std::optional<Value *> SimpleV =
A.getAssumedSimplified(
11354 if (!SimpleV.has_value())
11358 addValue(
A, getState(), **SimpleV, &
PHI,
II.S, getAnchorScope());
11365 bool handleGenericInst(Attributor &
A, Instruction &
I, ItemInfo
II,
11366 SmallVectorImpl<ItemInfo> &Worklist) {
11367 bool SomeSimplified =
false;
11368 bool UsedAssumedInformation =
false;
11370 SmallVector<Value *, 8> NewOps(
I.getNumOperands());
11373 const auto &SimplifiedOp =
A.getAssumedSimplified(
11378 if (!SimplifiedOp.has_value())
11382 NewOps[
Idx] = *SimplifiedOp;
11386 SomeSimplified |= (NewOps[
Idx] !=
Op);
11392 if (!SomeSimplified)
11395 InformationCache &InfoCache =
A.getInfoCache();
11399 const auto *
TLI =
A.getInfoCache().getTargetLibraryInfoForFunction(*
F);
11402 const DataLayout &
DL =
I.getDataLayout();
11403 SimplifyQuery Q(
DL,
TLI, DT, AC, &
I);
11405 if (!NewV || NewV == &
I)
11408 LLVM_DEBUG(
dbgs() <<
"Generic inst " <<
I <<
" assumed simplified to "
11415 Attributor &
A, Instruction &
I, ItemInfo
II,
11416 SmallVectorImpl<ItemInfo> &Worklist,
11417 SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
11420 CI->getPredicate(),
II, Worklist);
11422 switch (
I.getOpcode()) {
11423 case Instruction::Select:
11425 case Instruction::PHI:
11427 case Instruction::Load:
11430 return handleGenericInst(
A,
I,
II, Worklist);
11435 void genericValueTraversal(Attributor &
A,
Value *InitialV) {
11436 SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
11438 SmallSet<ItemInfo, 16> Visited;
11456 if (Iteration++ >=
A.Opts.attributor_max_potential_values_iterations) {
11457 LLVM_DEBUG(
dbgs() <<
"Generic value traversal reached iteration limit: "
11458 << Iteration <<
"!\n");
11459 addValue(
A, getState(), *
V, CtxI, S, getAnchorScope());
11465 Value *NewV =
nullptr;
11466 if (
V->getType()->isPointerTy()) {
11472 for (Argument &Arg :
Callee->args())
11479 if (NewV && NewV !=
V) {
11480 Worklist.
push_back({{*NewV, CtxI}, S});
11494 if (
V == InitialV && CtxI == getCtxI()) {
11495 indicatePessimisticFixpoint();
11499 addValue(
A, getState(), *
V, CtxI, S, getAnchorScope());
11500 }
while (!Worklist.
empty());
11504 for (
auto &It : LivenessAAs)
11505 if (It.second.AnyDead)
11506 A.recordDependence(*It.second.LivenessAA, *
this, DepClassTy::OPTIONAL);
11509 void trackStatistics()
const override {
11514struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
11515 using Base = AAPotentialValuesImpl;
11516 AAPotentialValuesArgument(
const IRPosition &IRP, Attributor &
A)
11522 indicatePessimisticFixpoint();
11526 auto AssumedBefore = getAssumed();
11528 unsigned ArgNo = getCalleeArgNo();
11530 bool UsedAssumedInformation =
false;
11532 auto CallSitePred = [&](AbstractCallSite ACS) {
11534 if (CSArgIRP.getPositionKind() == IRP_INVALID)
11537 if (!
A.getAssumedSimplifiedValues(CSArgIRP,
this,
Values,
11539 UsedAssumedInformation))
11542 return isValidState();
11545 if (!
A.checkForAllCallSites(CallSitePred, *
this,
11547 UsedAssumedInformation))
11548 return indicatePessimisticFixpoint();
11550 Function *Fn = getAssociatedFunction();
11551 bool AnyNonLocal =
false;
11552 for (
auto &It :
Values) {
11554 addValue(
A, getState(), *It.getValue(), It.getCtxI(),
AA::AnyScope,
11559 return indicatePessimisticFixpoint();
11563 addValue(
A, getState(), *It.getValue(), It.getCtxI(),
AA::AnyScope,
11569 AnyNonLocal =
true;
11571 assert(!undefIsContained() &&
"Undef should be an explicit value!");
11573 giveUpOnIntraprocedural(
A);
11575 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11576 : ChangeStatus::CHANGED;
11579 void trackStatistics()
const override {
11584struct AAPotentialValuesReturned :
public AAPotentialValuesFloating {
11585 using Base = AAPotentialValuesFloating;
11586 AAPotentialValuesReturned(
const IRPosition &IRP, Attributor &
A)
11591 if (!
F ||
F->isDeclaration() ||
F->getReturnType()->isVoidTy()) {
11592 indicatePessimisticFixpoint();
11596 for (Argument &Arg :
F->args())
11599 ReturnedArg = &Arg;
11602 if (!
A.isFunctionIPOAmendable(*
F) ||
11603 A.hasSimplificationCallback(getIRPosition())) {
11605 indicatePessimisticFixpoint();
11607 indicateOptimisticFixpoint();
11612 auto AssumedBefore = getAssumed();
11613 bool UsedAssumedInformation =
false;
11616 Function *AnchorScope = getAnchorScope();
11622 UsedAssumedInformation,
11628 bool AllInterAreIntra =
false;
11635 for (
const AA::ValueAndContext &VAC :
Values) {
11636 addValue(
A, getState(), *VAC.
getValue(),
11640 if (AllInterAreIntra)
11647 HandleReturnedValue(*ReturnedArg,
nullptr,
true);
11650 bool AddValues =
true;
11653 addValue(
A, getState(), *RetI.getOperand(0), &RetI,
AA::AnyScope,
11657 return HandleReturnedValue(*RetI.getOperand(0), &RetI, AddValues);
11660 if (!
A.checkForAllInstructions(RetInstPred, *
this, {Instruction::Ret},
11661 UsedAssumedInformation,
11663 return indicatePessimisticFixpoint();
11666 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11667 : ChangeStatus::CHANGED;
11672 return ChangeStatus::UNCHANGED;
11674 if (!getAssumedSimplifiedValues(
A,
Values, AA::ValueScope::Intraprocedural,
11676 return ChangeStatus::UNCHANGED;
11677 Value *NewVal = getSingleValue(
A, *
this, getIRPosition(),
Values);
11679 return ChangeStatus::UNCHANGED;
11684 "Number of function with unique return");
11687 {Attribute::get(Arg->
getContext(), Attribute::Returned)});
11692 Value *RetOp = RetI.getOperand(0);
11696 if (
A.changeUseAfterManifest(RetI.getOperandUse(0), *NewVal))
11697 Changed = ChangeStatus::CHANGED;
11700 bool UsedAssumedInformation =
false;
11701 (void)
A.checkForAllInstructions(RetInstPred, *
this, {Instruction::Ret},
11702 UsedAssumedInformation,
11708 return AAPotentialValues::indicatePessimisticFixpoint();
11711 void trackStatistics()
const override{
11718struct AAPotentialValuesFunction : AAPotentialValuesImpl {
11719 AAPotentialValuesFunction(
const IRPosition &IRP, Attributor &
A)
11720 : AAPotentialValuesImpl(IRP,
A) {}
11727 void trackStatistics()
const override {
11732struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
11733 AAPotentialValuesCallSite(
const IRPosition &IRP, Attributor &
A)
11734 : AAPotentialValuesFunction(IRP,
A) {}
11736 void trackStatistics()
const override {
11741struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
11742 AAPotentialValuesCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
11743 : AAPotentialValuesImpl(IRP,
A) {}
11746 auto AssumedBefore = getAssumed();
11750 return indicatePessimisticFixpoint();
11752 bool UsedAssumedInformation =
false;
11756 UsedAssumedInformation))
11757 return indicatePessimisticFixpoint();
11764 Values, S, UsedAssumedInformation))
11767 for (
auto &It :
Values) {
11768 Value *
V = It.getValue();
11769 std::optional<Value *> CallerV =
A.translateArgumentToCallSiteContent(
11770 V, *CB, *
this, UsedAssumedInformation);
11771 if (!CallerV.has_value()) {
11775 V = *CallerV ? *CallerV :
V;
11781 giveUpOnIntraprocedural(
A);
11784 addValue(
A, getState(), *
V, CB, S, getAnchorScope());
11789 return indicatePessimisticFixpoint();
11791 return indicatePessimisticFixpoint();
11792 return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
11793 : ChangeStatus::CHANGED;
11797 return AAPotentialValues::indicatePessimisticFixpoint();
11800 void trackStatistics()
const override {
11805struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
11806 AAPotentialValuesCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
11807 : AAPotentialValuesFloating(IRP,
A) {}
11809 void trackStatistics()
const override {
11818 AAAssumptionInfoImpl(
const IRPosition &IRP, Attributor &
A,
11819 const DenseSet<StringRef> &
Known)
11820 : AAAssumptionInfo(IRP,
A,
Known) {}
11824 if (getKnown().isUniversal())
11825 return ChangeStatus::UNCHANGED;
11827 const IRPosition &IRP = getIRPosition();
11829 getAssumed().getSet().
end());
11831 return A.manifestAttrs(IRP,
11838 bool hasAssumption(
const StringRef Assumption)
const override {
11839 return isValidState() && setContains(Assumption);
11842 const std::string getAsStr(Attributor *
A)
const override {
11843 const SetContents &
Known = getKnown();
11844 const SetContents &Assumed = getAssumed();
11848 const std::string KnownStr =
llvm::join(Set,
",");
11850 std::string AssumedStr =
"Universal";
11851 if (!Assumed.isUniversal()) {
11852 Set.assign(Assumed.getSet().begin(), Assumed.getSet().end());
11855 return "Known [" + KnownStr +
"]," +
" Assumed [" + AssumedStr +
"]";
11870struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
11871 AAAssumptionInfoFunction(
const IRPosition &IRP, Attributor &
A)
11872 : AAAssumptionInfoImpl(IRP,
A,
11878 auto CallSitePred = [&](AbstractCallSite ACS) {
11879 const auto *AssumptionAA =
A.getAAFor<AAAssumptionInfo>(
11881 DepClassTy::REQUIRED);
11885 Changed |= getIntersection(AssumptionAA->getAssumed());
11886 return !getAssumed().empty() || !getKnown().empty();
11889 bool UsedAssumedInformation =
false;
11894 if (!
A.checkForAllCallSites(CallSitePred, *
this,
true,
11895 UsedAssumedInformation))
11896 return indicatePessimisticFixpoint();
11898 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11901 void trackStatistics()
const override {}
11905struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
11907 AAAssumptionInfoCallSite(
const IRPosition &IRP, Attributor &
A)
11908 : AAAssumptionInfoImpl(IRP,
A, getInitialAssumptions(IRP)) {}
11912 A.getAAFor<AAAssumptionInfo>(*
this, FnPos, DepClassTy::REQUIRED);
11917 auto *AssumptionAA =
11918 A.getAAFor<AAAssumptionInfo>(*
this, FnPos, DepClassTy::REQUIRED);
11920 return indicatePessimisticFixpoint();
11921 bool Changed = getIntersection(AssumptionAA->getAssumed());
11922 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
11925 void trackStatistics()
const override {}
11930 DenseSet<StringRef> getInitialAssumptions(
const IRPosition &IRP) {
11937 return Assumptions;
11952struct AAUnderlyingObjectsImpl
11957 const std::string getAsStr(
Attributor *
A)
const override {
11958 if (!isValidState())
11959 return "<invalid>";
11962 OS <<
"underlying objects: inter " << InterAssumedUnderlyingObjects.size()
11963 <<
" objects, intra " << IntraAssumedUnderlyingObjects.size()
11965 if (!InterAssumedUnderlyingObjects.empty()) {
11966 OS <<
"inter objects:\n";
11967 for (
auto *Obj : InterAssumedUnderlyingObjects)
11968 OS << *Obj <<
'\n';
11970 if (!IntraAssumedUnderlyingObjects.empty()) {
11971 OS <<
"intra objects:\n";
11972 for (
auto *Obj : IntraAssumedUnderlyingObjects)
11973 OS << *
Obj <<
'\n';
11978 void trackStatistics()
const override {}
11981 auto &Ptr = getAssociatedValue();
11983 bool UsedAssumedInformation =
false;
11984 auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
11986 SmallPtrSet<Value *, 8> SeenObjects;
11990 Scope, UsedAssumedInformation))
11991 return UnderlyingObjects.
insert(&Ptr);
11995 for (
unsigned I = 0;
I <
Values.size(); ++
I) {
11999 if (!SeenObjects.
insert(UO ? UO : Obj).second)
12001 if (UO && UO != Obj) {
12007 const auto *OtherAA =
A.getAAFor<AAUnderlyingObjects>(
12013 Values.emplace_back(
V,
nullptr);
12017 if (!OtherAA || !OtherAA->forallUnderlyingObjects(
Pred, Scope))
12019 "The forall call should not return false at this position");
12025 Changed |= handleIndirect(
A, *Obj, UnderlyingObjects, Scope,
12026 UsedAssumedInformation);
12032 for (
unsigned u = 0, e =
PHI->getNumIncomingValues(); u < e; u++) {
12034 handleIndirect(
A, *
PHI->getIncomingValue(u), UnderlyingObjects,
12035 Scope, UsedAssumedInformation);
12049 if (!UsedAssumedInformation)
12050 indicateOptimisticFixpoint();
12051 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
12054 bool forallUnderlyingObjects(
12057 if (!isValidState())
12058 return Pred(getAssociatedValue());
12061 ? IntraAssumedUnderlyingObjects
12062 : InterAssumedUnderlyingObjects;
12063 for (
Value *Obj : AssumedUnderlyingObjects)
12073 bool handleIndirect(Attributor &
A,
Value &
V,
12074 SmallSetVector<Value *, 8> &UnderlyingObjects,
12077 const auto *AA =
A.getAAFor<AAUnderlyingObjects>(
12083 if (!AA || !AA->forallUnderlyingObjects(
Pred, Scope))
12085 "The forall call should not return false at this position");
12091 SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
12093 SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
12096struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
12097 AAUnderlyingObjectsFloating(
const IRPosition &IRP, Attributor &
A)
12098 : AAUnderlyingObjectsImpl(IRP,
A) {}
12101struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
12102 AAUnderlyingObjectsArgument(
const IRPosition &IRP, Attributor &
A)
12103 : AAUnderlyingObjectsImpl(IRP,
A) {}
12106struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
12107 AAUnderlyingObjectsCallSite(
const IRPosition &IRP, Attributor &
A)
12108 : AAUnderlyingObjectsImpl(IRP,
A) {}
12111struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
12112 AAUnderlyingObjectsCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
12113 : AAUnderlyingObjectsImpl(IRP,
A) {}
12116struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
12117 AAUnderlyingObjectsReturned(
const IRPosition &IRP, Attributor &
A)
12118 : AAUnderlyingObjectsImpl(IRP,
A) {}
12121struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
12122 AAUnderlyingObjectsCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
12123 : AAUnderlyingObjectsImpl(IRP,
A) {}
12126struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
12127 AAUnderlyingObjectsFunction(
const IRPosition &IRP, Attributor &
A)
12128 : AAUnderlyingObjectsImpl(IRP,
A) {}
12135 AAGlobalValueInfoFloating(
const IRPosition &IRP, Attributor &
A)
12136 : AAGlobalValueInfo(IRP,
A) {}
12140 bool checkUse(Attributor &
A,
const Use &U,
bool &Follow,
12141 SmallVectorImpl<const Value *> &Worklist) {
12147 if (!GV->hasLocalLinkage() && !
A.isClosedWorldModule())
12153 LLVM_DEBUG(
dbgs() <<
"[AAGlobalValueInfo] Check use: " << *
U.get() <<
" in "
12154 << *UInst <<
"\n");
12157 int Idx = &
Cmp->getOperandUse(0) == &
U;
12160 return U == &getAnchorValue();
12165 auto CallSitePred = [&](AbstractCallSite ACS) {
12166 Worklist.
push_back(ACS.getInstruction());
12169 bool UsedAssumedInformation =
false;
12171 if (!
A.checkForAllCallSites(CallSitePred, *UInst->
getFunction(),
12173 UsedAssumedInformation))
12191 if (!Fn || !
A.isFunctionIPOAmendable(*Fn))
12200 unsigned NumUsesBefore =
Uses.size();
12202 SmallPtrSet<const Value *, 8> Visited;
12206 auto UsePred = [&](
const Use &
U,
bool &Follow) ->
bool {
12214 return checkUse(
A, U, Follow, Worklist);
12216 auto EquivalentUseCB = [&](
const Use &OldU,
const Use &NewU) {
12217 Uses.insert(&OldU);
12221 while (!Worklist.
empty()) {
12223 if (!Visited.
insert(
V).second)
12225 if (!
A.checkForAllUses(UsePred, *
this, *
V,
12227 DepClassTy::OPTIONAL,
12228 true, EquivalentUseCB)) {
12229 return indicatePessimisticFixpoint();
12233 return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
12234 : ChangeStatus::CHANGED;
12237 bool isPotentialUse(
const Use &U)
const override {
12238 return !isValidState() ||
Uses.contains(&U);
12242 return ChangeStatus::UNCHANGED;
12245 const std::string getAsStr(Attributor *
A)
const override {
12246 return "[" + std::to_string(
Uses.size()) +
" uses]";
12249 void trackStatistics()
const override {
12255 SmallPtrSet<const Use *, 8>
Uses;
12262 AAIndirectCallInfoCallSite(
const IRPosition &IRP, Attributor &
A)
12263 : AAIndirectCallInfo(IRP,
A) {}
12266 auto *MD = getCtxI()->getMetadata(LLVMContext::MD_callees);
12267 if (!MD && !
A.isClosedWorldModule())
12271 for (
const auto &
Op : MD->operands())
12273 PotentialCallees.insert(Callee);
12274 }
else if (
A.isClosedWorldModule()) {
12276 A.getInfoCache().getIndirectlyCallableFunctions(
A);
12277 PotentialCallees.insert_range(IndirectlyCallableFunctions);
12280 if (PotentialCallees.empty())
12281 indicateOptimisticFixpoint();
12289 SmallSetVector<Function *, 4> AssumedCalleesNow;
12290 bool AllCalleesKnownNow = AllCalleesKnown;
12292 auto CheckPotentialCalleeUse = [&](
Function &PotentialCallee,
12293 bool &UsedAssumedInformation) {
12294 const auto *GIAA =
A.getAAFor<AAGlobalValueInfo>(
12296 if (!GIAA || GIAA->isPotentialUse(CalleeUse))
12298 UsedAssumedInformation = !GIAA->isAtFixpoint();
12302 auto AddPotentialCallees = [&]() {
12303 for (
auto *PotentialCallee : PotentialCallees) {
12304 bool UsedAssumedInformation =
false;
12305 if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
12306 AssumedCalleesNow.
insert(PotentialCallee);
12312 bool UsedAssumedInformation =
false;
12315 AA::ValueScope::AnyScope,
12316 UsedAssumedInformation)) {
12317 if (PotentialCallees.empty())
12318 return indicatePessimisticFixpoint();
12319 AddPotentialCallees();
12324 auto CheckPotentialCallee = [&](
Function &Fn) {
12325 if (!PotentialCallees.empty() && !PotentialCallees.count(&Fn))
12328 auto &CachedResult = FilterResults[&Fn];
12329 if (CachedResult.has_value())
12330 return CachedResult.value();
12332 bool UsedAssumedInformation =
false;
12333 if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
12334 if (!UsedAssumedInformation)
12335 CachedResult =
false;
12344 for (
int I = NumCBArgs;
I < NumFnArgs; ++
I) {
12345 bool IsKnown =
false;
12348 DepClassTy::OPTIONAL, IsKnown)) {
12350 CachedResult =
false;
12355 CachedResult =
true;
12361 for (
auto &VAC :
Values) {
12369 if (CheckPotentialCallee(*VACFn))
12370 AssumedCalleesNow.
insert(VACFn);
12373 if (!PotentialCallees.empty()) {
12374 AddPotentialCallees();
12377 AllCalleesKnownNow =
false;
12380 if (AssumedCalleesNow == AssumedCallees &&
12381 AllCalleesKnown == AllCalleesKnownNow)
12382 return ChangeStatus::UNCHANGED;
12384 std::swap(AssumedCallees, AssumedCalleesNow);
12385 AllCalleesKnown = AllCalleesKnownNow;
12386 return ChangeStatus::CHANGED;
12391 if (!AllCalleesKnown && AssumedCallees.empty())
12392 return ChangeStatus::UNCHANGED;
12395 bool UsedAssumedInformation =
false;
12396 if (
A.isAssumedDead(*CB,
this,
nullptr,
12397 UsedAssumedInformation))
12398 return ChangeStatus::UNCHANGED;
12403 if (
FP->getType()->getPointerAddressSpace() != ProgramAS)
12404 FP =
new AddrSpaceCastInst(
12405 FP, PointerType::get(
FP->getContext(), ProgramAS),
12406 FP->getName() +
".as" + Twine(ProgramAS), CB->
getIterator());
12415 if (AssumedCallees.empty()) {
12416 assert(AllCalleesKnown &&
12417 "Expected all callees to be known if there are none.");
12418 A.changeToUnreachableAfterManifest(CB);
12419 return ChangeStatus::CHANGED;
12423 if (AllCalleesKnown && AssumedCallees.size() == 1) {
12424 auto *NewCallee = AssumedCallees.front();
12427 NumIndirectCallsPromoted++;
12428 return ChangeStatus::CHANGED;
12436 A.deleteAfterManifest(*CB);
12437 return ChangeStatus::CHANGED;
12447 bool SpecializedForAnyCallees =
false;
12448 bool SpecializedForAllCallees = AllCalleesKnown;
12449 ICmpInst *LastCmp =
nullptr;
12452 for (
Function *NewCallee : AssumedCallees) {
12453 if (!
A.shouldSpecializeCallSiteForCallee(*
this, *CB, *NewCallee,
12454 AssumedCallees.size())) {
12455 SkippedAssumedCallees.
push_back(NewCallee);
12456 SpecializedForAllCallees =
false;
12459 SpecializedForAnyCallees =
true;
12465 A.registerManifestAddedBasicBlock(*ThenTI->
getParent());
12466 A.registerManifestAddedBasicBlock(*IP->getParent());
12472 A.registerManifestAddedBasicBlock(*ElseBB);
12474 SplitTI->replaceUsesOfWith(CBBB, ElseBB);
12479 CastInst *RetBC =
nullptr;
12480 CallInst *NewCall =
nullptr;
12485 NumIndirectCallsPromoted++;
12494 auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
12495 if (!AllCalleesKnown)
12496 return ChangeStatus::UNCHANGED;
12497 MDBuilder MDB(IndirectCB.getContext());
12498 MDNode *Callees = MDB.createCallees(SkippedAssumedCallees);
12499 IndirectCB.setMetadata(LLVMContext::MD_callees, Callees);
12500 return ChangeStatus::CHANGED;
12503 if (!SpecializedForAnyCallees)
12504 return AttachCalleeMetadata(*CB);
12507 if (SpecializedForAllCallees) {
12510 new UnreachableInst(IP->getContext(), IP);
12511 IP->eraseFromParent();
12514 CBClone->setName(CB->
getName());
12515 CBClone->insertBefore(*IP->getParent(), IP);
12516 NewCalls.
push_back({CBClone,
nullptr});
12517 AttachCalleeMetadata(*CBClone);
12524 CB->
getParent()->getFirstInsertionPt());
12525 for (
auto &It : NewCalls) {
12526 CallBase *NewCall = It.first;
12527 Instruction *CallRet = It.second ? It.second : It.first;
12539 A.deleteAfterManifest(*CB);
12540 Changed = ChangeStatus::CHANGED;
12545 const std::string getAsStr(Attributor *
A)
const override {
12546 return std::string(AllCalleesKnown ?
"eliminate" :
"specialize") +
12547 " indirect call site with " + std::to_string(AssumedCallees.size()) +
12551 void trackStatistics()
const override {
12552 if (AllCalleesKnown) {
12554 Eliminated, CallSites,
12555 "Number of indirect call sites eliminated via specialization")
12558 "Number of indirect call sites specialized")
12562 bool foreachCallee(function_ref<
bool(
Function *)> CB)
const override {
12563 return isValidState() && AllCalleesKnown &&
all_of(AssumedCallees, CB);
12568 DenseMap<Function *, std::optional<bool>> FilterResults;
12572 SmallSetVector<Function *, 4> PotentialCallees;
12576 SmallSetVector<Function *, 4> AssumedCallees;
12580 bool AllCalleesKnown =
true;
12587struct AAInvariantLoadPointerImpl
12589 AAInvariantLoadPointer> {
12593 IS_NOALIAS = 1 << 0,
12596 IS_NOEFFECT = 1 << 1,
12598 IS_LOCALLY_INVARIANT = 1 << 2,
12600 IS_LOCALLY_CONSTRAINED = 1 << 3,
12602 IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
12603 IS_LOCALLY_CONSTRAINED,
12605 static_assert(getBestState() == IS_BEST_STATE,
"Unexpected best state");
12608 StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
12612 AAInvariantLoadPointerImpl(
const IRPosition &IRP, Attributor &
A)
12615 bool isKnownInvariant()
const final {
12616 return isKnownLocallyInvariant() && isKnown(IS_LOCALLY_CONSTRAINED);
12619 bool isKnownLocallyInvariant()
const final {
12620 if (isKnown(IS_LOCALLY_INVARIANT))
12622 return isKnown(IS_NOALIAS | IS_NOEFFECT);
12625 bool isAssumedInvariant()
const final {
12626 return isAssumedLocallyInvariant() && isAssumed(IS_LOCALLY_CONSTRAINED);
12629 bool isAssumedLocallyInvariant()
const final {
12630 if (isAssumed(IS_LOCALLY_INVARIANT))
12632 return isAssumed(IS_NOALIAS | IS_NOEFFECT);
12639 if (requiresNoAlias() && !isAssumed(IS_NOALIAS))
12640 return indicatePessimisticFixpoint();
12644 Changed |= updateLocalInvariance(
A);
12650 if (!isKnownInvariant())
12651 return ChangeStatus::UNCHANGED;
12654 const Value *Ptr = &getAssociatedValue();
12655 const auto TagInvariantLoads = [&](
const Use &
U,
bool &) {
12656 if (
U.get() != Ptr)
12664 if (!
A.isRunOn(
I->getFunction()))
12667 if (
I->hasMetadata(LLVMContext::MD_invariant_load))
12671 LI->setMetadata(LLVMContext::MD_invariant_load,
12673 Changed = ChangeStatus::CHANGED;
12678 (void)
A.checkForAllUses(TagInvariantLoads, *
this, *Ptr);
12682 const std::string getAsStr(Attributor *)
const override {
12683 if (isKnownInvariant())
12684 return "load-invariant pointer";
12685 return "non-invariant pointer";
12688 void trackStatistics()
const override {}
12692 bool requiresNoAlias()
const {
12693 switch (getPositionKind()) {
12699 case IRP_CALL_SITE:
12701 case IRP_CALL_SITE_RETURNED: {
12706 case IRP_ARGUMENT: {
12707 const Function *
F = getAssociatedFunction();
12708 assert(
F &&
"no associated function for argument");
12714 bool isExternal()
const {
12715 const Function *
F = getAssociatedFunction();
12719 getPositionKind() != IRP_CALL_SITE_RETURNED;
12723 if (isKnown(IS_NOALIAS) || !isAssumed(IS_NOALIAS))
12724 return ChangeStatus::UNCHANGED;
12727 if (
const auto *ANoAlias =
A.getOrCreateAAFor<AANoAlias>(
12728 getIRPosition(),
this, DepClassTy::REQUIRED)) {
12729 if (ANoAlias->isKnownNoAlias()) {
12730 addKnownBits(IS_NOALIAS);
12731 return ChangeStatus::CHANGED;
12734 if (!ANoAlias->isAssumedNoAlias()) {
12735 removeAssumedBits(IS_NOALIAS);
12736 return ChangeStatus::CHANGED;
12739 return ChangeStatus::UNCHANGED;
12744 if (
const Argument *Arg = getAssociatedArgument()) {
12746 addKnownBits(IS_NOALIAS);
12747 return ChangeStatus::UNCHANGED;
12752 removeAssumedBits(IS_NOALIAS);
12753 return ChangeStatus::CHANGED;
12756 return ChangeStatus::UNCHANGED;
12760 if (isKnown(IS_NOEFFECT) || !isAssumed(IS_NOEFFECT))
12761 return ChangeStatus::UNCHANGED;
12763 if (!getAssociatedFunction())
12764 return indicatePessimisticFixpoint();
12767 return indicatePessimisticFixpoint();
12769 const auto HasNoEffectLoads = [&](
const Use &
U,
bool &) {
12771 return !LI || !LI->mayHaveSideEffects();
12773 if (!
A.checkForAllUses(HasNoEffectLoads, *
this, getAssociatedValue()))
12774 return indicatePessimisticFixpoint();
12776 if (
const auto *AMemoryBehavior =
A.getOrCreateAAFor<AAMemoryBehavior>(
12777 getIRPosition(),
this, DepClassTy::REQUIRED)) {
12780 if (!AMemoryBehavior->isAssumedReadOnly())
12781 return indicatePessimisticFixpoint();
12783 if (AMemoryBehavior->isKnownReadOnly()) {
12784 addKnownBits(IS_NOEFFECT);
12785 return ChangeStatus::UNCHANGED;
12788 return ChangeStatus::UNCHANGED;
12791 if (
const Argument *Arg = getAssociatedArgument()) {
12793 addKnownBits(IS_NOEFFECT);
12794 return ChangeStatus::UNCHANGED;
12799 return indicatePessimisticFixpoint();
12802 return ChangeStatus::UNCHANGED;
12806 if (isKnown(IS_LOCALLY_INVARIANT) || !isAssumed(IS_LOCALLY_INVARIANT))
12807 return ChangeStatus::UNCHANGED;
12810 const auto *AUO =
A.getOrCreateAAFor<AAUnderlyingObjects>(
12811 getIRPosition(),
this, DepClassTy::REQUIRED);
12813 return ChangeStatus::UNCHANGED;
12815 bool UsedAssumedInformation =
false;
12816 const auto IsLocallyInvariantLoadIfPointer = [&](
const Value &
V) {
12817 if (!
V.getType()->isPointerTy())
12819 const auto *IsInvariantLoadPointer =
12821 DepClassTy::REQUIRED);
12823 if (!IsInvariantLoadPointer)
12826 if (IsInvariantLoadPointer->isKnownLocallyInvariant())
12828 if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
12831 UsedAssumedInformation =
true;
12834 if (!AUO->forallUnderlyingObjects(IsLocallyInvariantLoadIfPointer))
12835 return indicatePessimisticFixpoint();
12841 if (!IsLocallyInvariantLoadIfPointer(*Arg))
12842 return indicatePessimisticFixpoint();
12847 if (!UsedAssumedInformation) {
12849 addKnownBits(IS_LOCALLY_INVARIANT);
12850 return ChangeStatus::CHANGED;
12853 return ChangeStatus::UNCHANGED;
12857struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
12858 AAInvariantLoadPointerFloating(
const IRPosition &IRP, Attributor &
A)
12859 : AAInvariantLoadPointerImpl(IRP,
A) {}
12862struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
12863 AAInvariantLoadPointerReturned(
const IRPosition &IRP, Attributor &
A)
12864 : AAInvariantLoadPointerImpl(IRP,
A) {}
12867 removeAssumedBits(IS_LOCALLY_CONSTRAINED);
12871struct AAInvariantLoadPointerCallSiteReturned final
12872 : AAInvariantLoadPointerImpl {
12873 AAInvariantLoadPointerCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
12874 : AAInvariantLoadPointerImpl(IRP,
A) {}
12878 const Function *
F = getAssociatedFunction();
12880 indicatePessimisticFixpoint();
12884 if (!
F->isDeclaration() && !
F->isIntrinsic())
12885 return AAInvariantLoadPointerImpl::initialize(
A);
12890 return AAInvariantLoadPointerImpl::initialize(
A);
12892 if (
F->onlyReadsMemory() &&
F->hasNoSync())
12893 return AAInvariantLoadPointerImpl::initialize(
A);
12897 indicatePessimisticFixpoint();
12901struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
12902 AAInvariantLoadPointerArgument(
const IRPosition &IRP, Attributor &
A)
12903 : AAInvariantLoadPointerImpl(IRP,
A) {}
12906 const Function *
F = getAssociatedFunction();
12907 assert(
F &&
"no associated function for argument");
12910 addKnownBits(IS_LOCALLY_CONSTRAINED);
12914 if (!
F->hasLocalLinkage())
12915 removeAssumedBits(IS_LOCALLY_CONSTRAINED);
12919struct AAInvariantLoadPointerCallSiteArgument final
12920 : AAInvariantLoadPointerImpl {
12921 AAInvariantLoadPointerCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
12922 : AAInvariantLoadPointerImpl(IRP,
A) {}
12929template <
typename InstType>
12930static bool makeChange(
Attributor &
A, InstType *MemInst,
const Use &U,
12932 bool UseOriginalValue) {
12933 if (
U.getOperandNo() != InstType::getPointerOperandIndex())
12936 if (MemInst->isVolatile()) {
12938 *MemInst->getFunction());
12939 unsigned NewAS = NewPtrTy->getPointerAddressSpace();
12940 if (!
TTI || !
TTI->hasVolatileVariant(MemInst, NewAS))
12944 if (UseOriginalValue) {
12945 A.changeUseAfterManifest(
const_cast<Use &
>(U), *OriginalValue);
12951 A.changeUseAfterManifest(
const_cast<Use &
>(U), *
CastInst);
12956 AAAddressSpaceImpl(
const IRPosition &IRP, Attributor &
A)
12957 : AAAddressSpace(IRP,
A) {}
12960 assert(isValidState() &&
"the AA is invalid");
12961 return AssumedAddressSpace;
12965 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
12966 "Associated value is not a pointer");
12968 if (!
A.getInfoCache().getFlatAddressSpace().has_value()) {
12969 indicatePessimisticFixpoint();
12973 unsigned FlatAS =
A.getInfoCache().getFlatAddressSpace().value();
12974 unsigned AS = getAssociatedType()->getPointerAddressSpace();
12975 if (AS != FlatAS) {
12976 [[maybe_unused]]
bool R = takeAddressSpace(AS);
12977 assert(R &&
"The take should happen");
12978 indicateOptimisticFixpoint();
12983 uint32_t OldAddressSpace = AssumedAddressSpace;
12984 unsigned FlatAS =
A.getInfoCache().getFlatAddressSpace().value();
12986 auto CheckAddressSpace = [&](
Value &
Obj) {
12992 unsigned ObjAS =
Obj.getType()->getPointerAddressSpace();
12993 if (ObjAS != FlatAS)
12994 return takeAddressSpace(ObjAS);
13008 A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(*F);
13010 if (AssumedAS != ~0U)
13011 return takeAddressSpace(AssumedAS);
13015 return takeAddressSpace(FlatAS);
13018 auto *AUO =
A.getOrCreateAAFor<AAUnderlyingObjects>(getIRPosition(),
this,
13019 DepClassTy::REQUIRED);
13020 if (!AUO->forallUnderlyingObjects(CheckAddressSpace))
13021 return indicatePessimisticFixpoint();
13023 return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
13024 : ChangeStatus::CHANGED;
13030 if (NewAS == InvalidAddressSpace ||
13032 return ChangeStatus::UNCHANGED;
13034 unsigned FlatAS =
A.getInfoCache().getFlatAddressSpace().value();
13036 Value *AssociatedValue = &getAssociatedValue();
13037 Value *OriginalValue = peelAddrspacecast(AssociatedValue, FlatAS);
13040 PointerType::get(getAssociatedType()->
getContext(), NewAS);
13041 bool UseOriginalValue =
13046 auto Pred = [&](
const Use &
U,
bool &) {
13047 if (
U.get() != AssociatedValue)
13058 makeChange(
A, LI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13061 makeChange(
A, SI, U, OriginalValue, NewPtrTy, UseOriginalValue);
13064 makeChange(
A, RMW, U, OriginalValue, NewPtrTy, UseOriginalValue);
13067 makeChange(
A, CmpX, U, OriginalValue, NewPtrTy, UseOriginalValue);
13074 (void)
A.checkForAllUses(
Pred, *
this, getAssociatedValue(),
13077 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13080 const std::string getAsStr(Attributor *
A)
const override {
13081 if (!isValidState())
13082 return "addrspace(<invalid>)";
13083 return "addrspace(" +
13084 (AssumedAddressSpace == InvalidAddressSpace
13086 : std::to_string(AssumedAddressSpace)) +
13091 uint32_t AssumedAddressSpace = InvalidAddressSpace;
13093 bool takeAddressSpace(uint32_t AS) {
13094 if (AssumedAddressSpace == InvalidAddressSpace) {
13095 AssumedAddressSpace = AS;
13098 return AssumedAddressSpace == AS;
13101 static Value *peelAddrspacecast(
Value *
V,
unsigned FlatAS) {
13103 assert(
I->getSrcAddressSpace() != FlatAS &&
13104 "there should not be flat AS -> non-flat AS");
13105 return I->getPointerOperand();
13108 if (
C->getOpcode() == Instruction::AddrSpaceCast) {
13109 assert(
C->getOperand(0)->getType()->getPointerAddressSpace() !=
13111 "there should not be flat AS -> non-flat AS X");
13112 return C->getOperand(0);
13118struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
13119 AAAddressSpaceFloating(
const IRPosition &IRP, Attributor &
A)
13120 : AAAddressSpaceImpl(IRP,
A) {}
13122 void trackStatistics()
const override {
13127struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
13128 AAAddressSpaceReturned(
const IRPosition &IRP, Attributor &
A)
13129 : AAAddressSpaceImpl(IRP,
A) {}
13134 (void)indicatePessimisticFixpoint();
13137 void trackStatistics()
const override {
13142struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
13143 AAAddressSpaceCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
13144 : AAAddressSpaceImpl(IRP,
A) {}
13146 void trackStatistics()
const override {
13151struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
13152 AAAddressSpaceArgument(
const IRPosition &IRP, Attributor &
A)
13153 : AAAddressSpaceImpl(IRP,
A) {}
13158struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
13159 AAAddressSpaceCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
13160 : AAAddressSpaceImpl(IRP,
A) {}
13165 (void)indicatePessimisticFixpoint();
13168 void trackStatistics()
const override {
13184 AANoAliasAddrSpaceImpl(
const IRPosition &IRP, Attributor &
A)
13185 : AANoAliasAddrSpace(IRP,
A) {}
13188 assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
13189 "Associated value is not a pointer");
13193 std::optional<unsigned> FlatAS =
A.getInfoCache().getFlatAddressSpace();
13194 if (!FlatAS.has_value()) {
13195 indicatePessimisticFixpoint();
13201 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13202 if (AS != *FlatAS) {
13204 indicateOptimisticFixpoint();
13209 unsigned FlatAS =
A.getInfoCache().getFlatAddressSpace().value();
13210 uint32_t OldAssumed = getAssumed();
13212 auto CheckAddressSpace = [&](
Value &
Obj) {
13216 unsigned AS =
Obj.getType()->getPointerAddressSpace();
13220 removeAS(
Obj.getType()->getPointerAddressSpace());
13224 const AAUnderlyingObjects *AUO =
A.getOrCreateAAFor<AAUnderlyingObjects>(
13225 getIRPosition(),
this, DepClassTy::REQUIRED);
13227 return indicatePessimisticFixpoint();
13229 return OldAssumed == getAssumed() ? ChangeStatus::UNCHANGED
13230 : ChangeStatus::CHANGED;
13234 unsigned FlatAS =
A.getInfoCache().getFlatAddressSpace().value();
13236 unsigned AS = getAssociatedType()->getPointerAddressSpace();
13237 if (AS != FlatAS ||
Map.empty())
13238 return ChangeStatus::UNCHANGED;
13240 LLVMContext &Ctx = getAssociatedValue().getContext();
13241 MDNode *NoAliasASNode =
nullptr;
13242 MDBuilder MDB(Ctx);
13244 for (RangeMap::const_iterator
I =
Map.begin();
I !=
Map.end();
I++) {
13247 unsigned Upper =
I.stop();
13248 unsigned Lower =
I.start();
13249 if (!NoAliasASNode) {
13250 NoAliasASNode = MDB.createRange(APInt(32,
Lower), APInt(32,
Upper + 1));
13253 MDNode *ASRange = MDB.createRange(APInt(32,
Lower), APInt(32,
Upper + 1));
13257 Value *AssociatedValue = &getAssociatedValue();
13260 auto AddNoAliasAttr = [&](
const Use &
U,
bool &) {
13261 if (
U.get() != AssociatedValue)
13264 if (!Inst || Inst->
hasMetadata(LLVMContext::MD_noalias_addrspace))
13271 Inst->
setMetadata(LLVMContext::MD_noalias_addrspace, NoAliasASNode);
13275 (void)
A.checkForAllUses(AddNoAliasAttr, *
this, *AssociatedValue,
13277 return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
13280 const std::string getAsStr(Attributor *
A)
const override {
13281 if (!isValidState())
13282 return "<invalid>";
13284 raw_string_ostream OS(Str);
13285 OS <<
"CanNotBeAddrSpace(";
13286 for (RangeMap::const_iterator
I =
Map.begin();
I !=
Map.end();
I++) {
13287 unsigned Upper =
I.stop();
13288 unsigned Lower =
I.start();
13289 OS <<
' ' <<
'[' <<
Upper <<
',' <<
Lower + 1 <<
')';
13296 void removeAS(
unsigned AS) {
13297 RangeMap::iterator
I =
Map.find(AS);
13299 if (
I !=
Map.end()) {
13300 unsigned Upper =
I.stop();
13301 unsigned Lower =
I.start();
13305 if (AS != ~((
unsigned)0) && AS + 1 <=
Upper)
13307 if (AS != 0 &&
Lower <= AS - 1)
13312 void resetASRanges(Attributor &
A) {
13314 Map.insert(0,
A.getInfoCache().getMaxAddrSpace(),
true);
13318struct AANoAliasAddrSpaceFloating final : AANoAliasAddrSpaceImpl {
13319 AANoAliasAddrSpaceFloating(
const IRPosition &IRP, Attributor &
A)
13320 : AANoAliasAddrSpaceImpl(IRP,
A) {}
13322 void trackStatistics()
const override {
13327struct AANoAliasAddrSpaceReturned final : AANoAliasAddrSpaceImpl {
13328 AANoAliasAddrSpaceReturned(
const IRPosition &IRP, Attributor &
A)
13329 : AANoAliasAddrSpaceImpl(IRP,
A) {}
13331 void trackStatistics()
const override {
13336struct AANoAliasAddrSpaceCallSiteReturned final : AANoAliasAddrSpaceImpl {
13337 AANoAliasAddrSpaceCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
13338 : AANoAliasAddrSpaceImpl(IRP,
A) {}
13340 void trackStatistics()
const override {
13345struct AANoAliasAddrSpaceArgument final : AANoAliasAddrSpaceImpl {
13346 AANoAliasAddrSpaceArgument(
const IRPosition &IRP, Attributor &
A)
13347 : AANoAliasAddrSpaceImpl(IRP,
A) {}
13349 void trackStatistics()
const override {
13354struct AANoAliasAddrSpaceCallSiteArgument final : AANoAliasAddrSpaceImpl {
13355 AANoAliasAddrSpaceCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
13356 : AANoAliasAddrSpaceImpl(IRP,
A) {}
13358 void trackStatistics()
const override {
13366 AAAllocationInfoImpl(
const IRPosition &IRP, Attributor &
A)
13367 : AAAllocationInfo(IRP,
A) {}
13369 std::optional<TypeSize> getAllocatedSize()
const override {
13370 assert(isValidState() &&
"the AA is invalid");
13371 return AssumedAllocatedSize;
13374 std::optional<TypeSize> findInitialAllocationSize(Instruction *
I,
13375 const DataLayout &
DL) {
13378 switch (
I->getOpcode()) {
13379 case Instruction::Alloca: {
13384 return std::nullopt;
13390 const IRPosition &IRP = getIRPosition();
13395 return indicatePessimisticFixpoint();
13397 bool IsKnownNoCapture;
13399 A,
this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture))
13400 return indicatePessimisticFixpoint();
13402 const AAPointerInfo *PI =
13403 A.getOrCreateAAFor<AAPointerInfo>(IRP, *
this, DepClassTy::REQUIRED);
13406 return indicatePessimisticFixpoint();
13409 return indicatePessimisticFixpoint();
13411 const DataLayout &
DL =
A.getDataLayout();
13412 const auto AllocationSize = findInitialAllocationSize(
I,
DL);
13415 if (!AllocationSize)
13416 return indicatePessimisticFixpoint();
13420 if (*AllocationSize == 0)
13421 return indicatePessimisticFixpoint();
13427 return indicatePessimisticFixpoint();
13429 if (BinSize == 0) {
13430 auto NewAllocationSize = std::make_optional<TypeSize>(0,
false);
13431 if (!changeAllocationSize(NewAllocationSize))
13432 return ChangeStatus::UNCHANGED;
13433 return ChangeStatus::CHANGED;
13437 const auto &It = PI->
begin();
13440 if (It->first.Offset != 0)
13441 return indicatePessimisticFixpoint();
13443 uint64_t SizeOfBin = It->first.Offset + It->first.Size;
13445 if (SizeOfBin >= *AllocationSize)
13446 return indicatePessimisticFixpoint();
13448 auto NewAllocationSize = std::make_optional<TypeSize>(SizeOfBin * 8,
false);
13450 if (!changeAllocationSize(NewAllocationSize))
13451 return ChangeStatus::UNCHANGED;
13453 return ChangeStatus::CHANGED;
13458 assert(isValidState() &&
13459 "Manifest should only be called if the state is valid.");
13463 auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
13465 unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
13467 switch (
I->getOpcode()) {
13469 case Instruction::Alloca: {
13473 Type *CharType = Type::getInt8Ty(
I->getContext());
13475 auto *NumBytesToValue =
13476 ConstantInt::get(
I->getContext(), APInt(32, NumBytesToAllocate));
13479 insertPt = std::next(insertPt);
13480 AllocaInst *NewAllocaInst =
13485 return ChangeStatus::CHANGED;
13493 return ChangeStatus::UNCHANGED;
13496 const std::string getAsStr(Attributor *
A)
const override {
13497 if (!isValidState())
13498 return "allocationinfo(<invalid>)";
13499 return "allocationinfo(" +
13500 (AssumedAllocatedSize == HasNoAllocationSize
13502 : std::to_string(AssumedAllocatedSize->getFixedValue())) +
13507 std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
13511 bool changeAllocationSize(std::optional<TypeSize>
Size) {
13512 if (AssumedAllocatedSize == HasNoAllocationSize ||
13513 AssumedAllocatedSize !=
Size) {
13514 AssumedAllocatedSize =
Size;
13521struct AAAllocationInfoFloating : AAAllocationInfoImpl {
13522 AAAllocationInfoFloating(
const IRPosition &IRP, Attributor &
A)
13523 : AAAllocationInfoImpl(IRP,
A) {}
13525 void trackStatistics()
const override {
13530struct AAAllocationInfoReturned : AAAllocationInfoImpl {
13531 AAAllocationInfoReturned(
const IRPosition &IRP, Attributor &
A)
13532 : AAAllocationInfoImpl(IRP,
A) {}
13537 (void)indicatePessimisticFixpoint();
13540 void trackStatistics()
const override {
13545struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
13546 AAAllocationInfoCallSiteReturned(
const IRPosition &IRP, Attributor &
A)
13547 : AAAllocationInfoImpl(IRP,
A) {}
13549 void trackStatistics()
const override {
13554struct AAAllocationInfoArgument : AAAllocationInfoImpl {
13555 AAAllocationInfoArgument(
const IRPosition &IRP, Attributor &
A)
13556 : AAAllocationInfoImpl(IRP,
A) {}
13558 void trackStatistics()
const override {
13563struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
13564 AAAllocationInfoCallSiteArgument(
const IRPosition &IRP, Attributor &
A)
13565 : AAAllocationInfoImpl(IRP,
A) {}
13569 (void)indicatePessimisticFixpoint();
13572 void trackStatistics()
const override {
13621#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
13622 case IRPosition::PK: \
13623 llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
13625#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
13626 case IRPosition::PK: \
13627 AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
13631#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13632 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13633 CLASS *AA = nullptr; \
13634 switch (IRP.getPositionKind()) { \
13635 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13636 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13637 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13638 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13639 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13640 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13641 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13642 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13647#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13648 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13649 CLASS *AA = nullptr; \
13650 switch (IRP.getPositionKind()) { \
13651 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13652 SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
13653 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13654 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13655 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13656 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13657 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13658 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13663#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
13664 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13665 CLASS *AA = nullptr; \
13666 switch (IRP.getPositionKind()) { \
13667 SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
13669 llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
13675#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13676 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13677 CLASS *AA = nullptr; \
13678 switch (IRP.getPositionKind()) { \
13679 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13680 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13681 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13682 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13683 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13684 SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
13685 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13686 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13691#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13692 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13693 CLASS *AA = nullptr; \
13694 switch (IRP.getPositionKind()) { \
13695 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13696 SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
13697 SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
13698 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13699 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
13700 SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
13701 SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
13702 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13707#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
13708 CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
13709 CLASS *AA = nullptr; \
13710 switch (IRP.getPositionKind()) { \
13711 SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
13712 SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
13713 SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
13714 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
13715 SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
13716 SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
13717 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
13718 SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
13770#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
13771#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
13772#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
13773#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
13774#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
13775#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
13776#undef SWITCH_PK_CREATE
13777#undef SWITCH_PK_INV
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
ReachingDefInfo InstSet & ToRemove
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis false
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
#define STATS_DECLTRACK(NAME, TYPE, MSG)
static std::optional< Constant * > askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA, const IRPosition &IRP, Type &Ty)
static bool collectTailCalls(Attributor &A, const AbstractAttribute &QueryingAA, SmallVectorImpl< CallInst * > &TailCalls)
Collect the non-musttail tail calls in the function of QueryingAA.
static void clampReturnedValueStates(Attributor &A, const AAType &QueryingAA, StateType &S, const IRPosition::CallBaseContext *CBContext=nullptr)
Clamp the information known for all returned values of a function (identified by QueryingAA) into S.
#define STATS_DECLTRACK_FN_ATTR(NAME)
#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define STATS_DECLTRACK_CS_ATTR(NAME)
#define PIPE_OPERATOR(CLASS)
#define STATS_DECLTRACK_ARG_ATTR(NAME)
static const Value * stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA, const Value *Val, const DataLayout &DL, APInt &Offset, bool GetMinOffset, bool AllowNonInbounds, bool UseAssumed=false)
#define STATS_DECLTRACK_CSRET_ATTR(NAME)
static Value * constructPointer(Value *Ptr, int64_t Offset, IRBuilder< NoFolder > &IRB)
Helper function to create a pointer based on Ptr, and advanced by Offset bytes.
#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define BUILD_STAT_NAME(NAME, TYPE)
static bool isDenselyPacked(Type *Ty, const DataLayout &DL)
Checks if a type could have padding bytes.
#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
static const Value * getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA, const Value *Ptr, int64_t &BytesOffset, const DataLayout &DL, bool AllowNonInbounds=false)
static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I, bool HeaderOnly, CycleRef *CPtr=nullptr)
#define STATS_DECLTRACK_FNRET_ATTR(NAME)
#define STATS_DECLTRACK_CSARG_ATTR(NAME)
#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS)
#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)
#define STATS_DECLTRACK_FLOATING_ATTR(NAME)
#define STATS_DECL(NAME, TYPE, MSG)
bpf unwind BPF Unwind Lower
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool isReachableImpl(SmallVectorImpl< BasicBlock * > &Worklist, const StopSetT &StopSet, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet, const DominatorTree *DT, const LoopInfo *LI, const CycleInfo *CI)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file declares an analysis pass that computes CycleInfo for LLVM IR, specialized from GenericCycl...
static uint64_t align(uint64_t Size)
DXIL Forward Handle Accesses
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
static Value * getCondition(Instruction *I)
Machine Check Debug Module
This file implements a map that provides insertion order iteration.
static unsigned getAddressSpace(const Value *V, unsigned MaxLookup)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
static StringRef getName(Value *V)
dot regions Print regions of function to dot true view regions View regions of function(with no function bodies)"
Remove Loads Into Fake Uses
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
std::pair< BasicBlock *, BasicBlock * > Edge
BaseType
A given derived pointer can have multiple base pointers through phi/selects.
This file defines generic set operations that may be used on set's of different types,...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
static unsigned getSize(unsigned Kind)
LLVM_ABI AACallGraphNode * operator*() const
bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB)
A trivial helper function to check to see if the specified pointers are no-alias.
Class for arbitrary precision integers.
int64_t getSExtValue() const
Get sign extended value.
CallBase * getInstruction() const
Return the underlying instruction.
bool isCallbackCall() const
Return true if this ACS represents a callback call.
bool isDirectCall() const
Return true if this ACS represents a direct call.
static LLVM_ABI void getCallbackUses(const CallBase &CB, SmallVectorImpl< const Use * > &CallbackUses)
Add operand uses of CB that represent callback uses into CallbackUses.
int getCallArgOperandNo(Argument &Arg) const
Return the operand index of the underlying instruction associated with Arg.
This class represents a conversion between pointers from one address space to another.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
unsigned getAddressSpace() const
Return the address space for the allocation.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
This class represents an incoming formal argument to a Function.
LLVM_ABI bool hasNoAliasAttr() const
Return true if this argument has the noalias attribute.
LLVM_ABI bool onlyReadsMemory() const
Return true if this argument has the readonly or readnone attribute.
LLVM_ABI bool hasPointeeInMemoryValueAttr() const
Return true if this argument has the byval, sret, inalloca, preallocated, or byref attribute.
LLVM_ABI bool hasReturnedAttr() const
Return true if this argument has the returned attribute.
LLVM_ABI bool hasByValAttr() const
Return true if this argument has the byval attribute.
const Function * getParent() const
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
This class stores enough information to efficiently remove some attributes from an existing AttrBuild...
bool contains(Attribute::AttrKind A) const
Return true if the builder has the specified attribute.
This class holds the attributes for a particular argument, parameter, function, or return value.
Functions, function parameters, and return types can have attributes to indicate how they should be t...
LLVM_ABI const ConstantRange & getRange() const
Returns the value of the range attribute.
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
LLVM_ABI FPClassTest getNoFPClass() const
Return the FPClassTest for nofpclass.
LLVM_ABI Attribute::AttrKind getKindAsEnum() const
Return the attribute's kind as an enum (Attribute::AttrKind).
LLVM_ABI MemoryEffects getMemoryEffects() const
Returns memory effects.
static LLVM_ABI Attribute getWithCaptureInfo(LLVMContext &Context, CaptureInfo CI)
static bool isEnumAttrKind(AttrKind Kind)
bool isValid() const
Return true if the attribute is any kind of attribute.
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
LLVM_ABI MaybeAlign getAlignment() const
Returns the alignment field of an attribute as a byte alignment value.
LLVM_ABI CaptureInfo getCaptureInfo() const
Returns information from captures attribute.
LLVM Basic Block Representation.
LLVM_ABI const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
const Instruction & front() const
InstListType::iterator iterator
Instruction iterators...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
BinaryOps getOpcode() const
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI MemoryEffects getMemoryEffects() const
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
LLVM_ABI bool isIndirectCall() const
Return true if the callsite is an indirect call.
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
Value * getCalledOperand() const
const Use & getCalledOperandUse() const
Attribute getFnAttr(StringRef Kind) const
Get the attribute of a given kind for the function.
const Use & getArgOperandUse(unsigned i) const
Wrappers for getting the Use of a call argument.
LLVM_ABI bool isVolatile() const
Return true if this is a call to an intrinsic with a volatile flag that is set, e....
OperandBundleUse getOperandBundleForOperand(unsigned OpIdx) const
Return the operand bundle for the operand at index OpIdx.
LLVM_ABI std::optional< ConstantRange > getRange() const
If this return value has a range attribute, return the value range of the argument.
Value * getArgOperand(unsigned i) const
bool isBundleOperand(unsigned Idx) const
Return true if the operand at index Idx is a bundle operand.
bool isConvergent() const
Determine if the invoke is convergent.
FunctionType * getFunctionType() const
LLVM_ABI Intrinsic::ID getIntrinsicID() const
Returns the intrinsic ID of the intrinsic called or Intrinsic::not_intrinsic if the called function i...
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
unsigned getArgOperandNo(const Use *U) const
Given a use for a arg operand, get the arg operand number that corresponds to it.
unsigned arg_size() const
bool isArgOperand(const Use *U) const
LLVM_ABI Function * getCaller()
Helper to get the caller (the parent function).
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
static CaptureInfo none()
Create CaptureInfo that does not capture any components of the pointer.
This is the base class for all instructions that perform data casts.
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
LLVM_ABI bool isIntegerCast() const
There are several places where we need to know if a cast instruction only deals with integer source a...
Type * getDestTy() const
Return the destination type, as a convenience.
bool isEquality() const
Determine if this is an equals/not equals predicate.
bool isFalseWhenEqual() const
This is just a convenience.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
bool isTrueWhenEqual() const
This is just a convenience.
Predicate getPredicate() const
Return the predicate for this instruction.
Conditional Branch instruction.
Value * getCondition() const
BasicBlock * getSuccessor(unsigned i) const
static LLVM_ABI Constant * getExtractElement(Constant *Vec, Constant *Idx, Type *OnlyIfReducedTy=nullptr)
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
This class represents a range of values.
const APInt & getLower() const
Return the lower value for this range.
LLVM_ABI bool isFullSet() const
Return true if this set contains all of the elements possible for this data-type.
LLVM_ABI bool isEmptySet() const
Return true if this set contains no members.
bool isSingleElement() const
Return true if this set contains exactly one member.
static LLVM_ABI ConstantRange makeAllowedICmpRegion(CmpInst::Predicate Pred, const ConstantRange &Other)
Produce the smallest range such that all values that may satisfy the given predicate with any value c...
const APInt & getUpper() const
Return the upper value for this range.
LLVM_ABI bool contains(const APInt &Val) const
Return true if the specified value is in the set.
Opaque handle to a cycle within a GenericCycleInfo that wraps the cycle's preorder index.
A parsed version of the target data layout string in and methods for querying it.
unsigned getProgramAddressSpace() const
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
const BasicBlock & getEntryBlock() const
iterator_range< arg_iterator > args()
const Function & getFunction() const
Argument * getArg(unsigned i) const
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
BlockT * getHeader(CycleRef C) const
CycleRef getCycle(const BlockT *Block) const
Find the innermost cycle containing Block.
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
bool hasLocalLinkage() const
PointerType * getType() const
Global values are always pointers.
static LLVM_ABI bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
Value * CreatePtrAddWithSignedOffset(Value *Ptr, int64_t Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Create a ptradd with a signed constant offset.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
LLVM_ABI bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
LLVM_ABI bool mayWriteToMemory() const LLVM_READONLY
Return true if this instruction may modify memory.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
LLVM_ABI void insertBefore(InstListType::iterator InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified position.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
LLVM_ABI BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
bool isTerminator() const
LLVM_ABI bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
iterator_range< user_iterator > users()
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
LLVM_ABI bool mayReadOrWriteMemory() const LLVM_READONLY
Return true if this instruction may read or write memory.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
FPClassTest getKnownFPClasses() const
Floating-point classes the value could be one of.
This is an important class for using LLVM in a threaded context.
LLVM_ABI ConstantRange getConstantRange(Value *V, Instruction *CtxI, bool UndefAllowed)
Return the ConstantRange constraint that is known to hold for the specified value at the specified in...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static LLVM_ABI MDNode * getMostGenericRange(MDNode *A, MDNode *B)
static MemoryEffectsBase readOnly()
bool doesNotAccessMemory() const
Whether this function accesses no memory.
static MemoryEffectsBase argMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
static MemoryEffectsBase inaccessibleMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
bool onlyAccessesInaccessibleMem() const
Whether this function only (at most) accesses inaccessible memory.
ModRefInfo getModRef(Location Loc) const
Get ModRefInfo for the given Location.
bool onlyAccessesArgPointees() const
Whether this function only (at most) accesses argument memory.
bool onlyReadsMemory() const
Whether this function only (at most) reads memory.
static MemoryEffectsBase writeOnly()
static MemoryEffectsBase inaccessibleOrArgMemOnly(ModRefInfo MR=ModRefInfo::ModRef)
static MemoryEffectsBase none()
bool onlyAccessesInaccessibleOrArgMem() const
Whether this function only (at most) accesses argument and inaccessible memory.
static MemoryEffectsBase unknown()
static LLVM_ABI std::optional< MemoryLocation > getOrNone(const Instruction *Inst)
static SizeOffsetValue unknown()
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Class to represent pointers.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
LLVM_ABI SCEVUse getSCEVAtScope(const SCEV *S, const Loop *L)
Return a SCEV expression for the specified value at the specified scope in the program.
LLVM_ABI const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
LLVM_ABI unsigned getSmallConstantMaxTripCount(const Loop *L, SmallVectorImpl< const SCEVPredicate * > *Predicates=nullptr)
Returns the upper bound of the loop trip count as a normal unsigned value.
ConstantRange getUnsignedRange(const SCEV *S)
Determine the unsigned range for a particular SCEV.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
TypeSize getElementOffset(unsigned Idx) const
TypeSize getSizeInBits() const
TypeSize getElementOffsetInBits(unsigned Idx) const
Class to represent struct types.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
Analysis pass providing the TargetTransformInfo.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI unsigned getIntegerBitWidth() const
bool isPointerTy() const
True if this is an instance of PointerType.
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
static UncondBrInst * Create(BasicBlock *Target, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i=0) const
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
const Use & getOperandUse(unsigned i) const
LLVM_ABI bool isDroppable() const
A droppable user is a user for which uses can be dropped without affecting correctness and should be ...
LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
static constexpr uint64_t MaximumAlignment
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
LLVM_ABI const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr, bool LookThroughIntToPtr=false) const
Accumulate the constant offset this value has compared to a base pointer.
static constexpr unsigned MaxAlignmentExponent
The maximum alignment for instructions.
iterator_range< use_iterator > uses()
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
std::pair< iterator, bool > insert(const ValueT &V)
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
const ParentTy * getParent() const
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
SetVector< Function * >::iterator I
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Abstract Attribute helper functions.
LLVM_ABI bool isAssumedReadNone(Attributor &A, const IRPosition &IRP, const AbstractAttribute &QueryingAA, bool &IsKnown)
Return true if IRP is readnone.
LLVM_ABI bool isAssumedReadOnly(Attributor &A, const IRPosition &IRP, const AbstractAttribute &QueryingAA, bool &IsKnown)
Return true if IRP is readonly.
raw_ostream & operator<<(raw_ostream &OS, const RangeTy &R)
LLVM_ABI std::optional< Value * > combineOptionalValuesInAAValueLatice(const std::optional< Value * > &A, const std::optional< Value * > &B, Type *Ty)
Return the combination of A and B such that the result is a possible value of both.
LLVM_ABI bool isValidAtPosition(const ValueAndContext &VAC, InformationCache &InfoCache)
Return true if the value of VAC is a valid at the position of VAC, that is a constant,...
LLVM_ABI bool isAssumedThreadLocalObject(Attributor &A, Value &Obj, const AbstractAttribute &QueryingAA)
Return true if Obj is assumed to be a thread local object.
LLVM_ABI bool isGPUConstantAddressSpace(const Module &M, unsigned AS)
Check if the given address space AS corresponds to a GPU constant address space for the target triple...
LLVM_ABI bool isDynamicallyUnique(Attributor &A, const AbstractAttribute &QueryingAA, const Value &V, bool ForAnalysisOnly=true)
Return true if V is dynamically unique, that is, there are no two "instances" of V at runtime with di...
LLVM_ABI bool getPotentialCopiesOfStoredValue(Attributor &A, StoreInst &SI, SmallSetVector< Value *, 4 > &PotentialCopies, const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, bool OnlyExact=false)
Collect all potential values of the one stored by SI into PotentialCopies.
LLVM_ABI bool isGPUSharedAddressSpace(const Module &M, unsigned AS)
Check if the given address space AS corresponds to a GPU shared address space for the target triple i...
LLVM_ABI bool isGPULocalAddressSpace(const Module &M, unsigned AS)
Check if the given address space AS corresponds to a GPU local/private address space for the target t...
SmallPtrSet< Instruction *, 4 > InstExclusionSetTy
LLVM_ABI bool isGPU(const Module &M)
Return true iff M target a GPU (and we can use GPU AS reasoning).
ValueScope
Flags to distinguish intra-procedural queries from potentially inter-procedural queries.
LLVM_ABI bool isValidInScope(const Value &V, const Function *Scope)
Return true if V is a valid value in Scope, that is a constant or an instruction/argument of Scope.
LLVM_ABI bool isPotentiallyReachable(Attributor &A, const Instruction &FromI, const Instruction &ToI, const AbstractAttribute &QueryingAA, const AA::InstExclusionSetTy *ExclusionSet=nullptr, std::function< bool(const Function &F)> GoBackwardsCB=nullptr)
Return true if ToI is potentially reachable from FromI without running into any instruction in Exclus...
LLVM_ABI bool isNoSyncInst(Attributor &A, const Instruction &I, const AbstractAttribute &QueryingAA)
Return true if I is a nosync instruction.
bool hasAssumedIRAttr(Attributor &A, const AbstractAttribute *QueryingAA, const IRPosition &IRP, DepClassTy DepClass, bool &IsKnown, bool IgnoreSubsumingPositions=false, const AAType **AAPtr=nullptr)
Helper to avoid creating an AA for IR Attributes that might already be set.
LLVM_ABI bool getPotentiallyLoadedValues(Attributor &A, LoadInst &LI, SmallSetVector< Value *, 4 > &PotentialValues, SmallSetVector< Instruction *, 4 > &PotentialValueOrigins, const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation, bool OnlyExact=false)
Collect all potential values LI could read into PotentialValues.
LLVM_ABI Value * getWithType(Value &V, Type &Ty)
Try to convert V to type Ty without introducing new instructions.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
@ BasicBlock
Various leaf nodes.
@ Unsupported
This operation is completely unsupported on the target.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ Valid
The data is already valid.
unsigned combineHashValue(unsigned a, unsigned b)
Simplistic combination of 32-bit hash values into 32-bit hash values.
ElementType
The element type of an SRV or UAV resource.
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract_or_null(Y &&MD)
Extract a Value from Metadata, if any, allowing null.
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
@ User
could "use" a pointer
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< UseNode * > Use
friend class Instruction
Iterator for Instructions in a `BasicBlock.
LLVM_ABI iterator begin() const
This is an optimization pass for GlobalISel generic memory operations.
bool operator<(int64_t V1, const APSInt &V2)
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt gcd(const DynamicAPInt &A, const DynamicAPInt &B)
LLVM_ABI KnownFPClass computeKnownFPClass(const Value *V, const APInt &DemandedElts, FPClassTest InterestedClasses, const SimplifyQuery &SQ, unsigned Depth=0)
Determine which floating-point classes are valid for V, and return them in KnownFPClass bit sets.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
LLVM_ABI bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
LLVM_ABI Constant * getInitialValueOfAllocation(const Value *V, const TargetLibraryInfo *TLI, Type *Ty)
If this is a call to an allocation function that initializes memory to a fixed value,...
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
RelativeUniformCounterPtr Values
BundleAttr getBundleAttrFromOBU(OperandBundleUse OBU)
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
auto pred_end(const MachineBasicBlock *BB)
unsigned getPointerAddressSpace(const Type *T)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
auto successors(const MachineBasicBlock *BB)
LLVM_ABI bool isRemovableAlloc(const CallBase *V, const TargetLibraryInfo *TLI)
Return true if this is a call to an allocation function that does not have side effects that we are r...
APFloat abs(APFloat X)
Returns the absolute value of the argument.
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
LLVM_ABI raw_fd_ostream & outs()
This returns a reference to a raw_fd_ostream for standard output.
LLVM_ABI Value * getAllocAlignment(const CallBase *V, const TargetLibraryInfo *TLI)
Gets the alignment argument for an aligned_alloc-like function, using either built-in knowledge based...
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
LLVM_ABI Value * simplifyInstructionWithOperands(Instruction *I, ArrayRef< Value * > NewOps, const SimplifyQuery &Q)
Like simplifyInstruction but the operands of I are replaced with NewOps.
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
LLVM_ABI bool isNoAliasCall(const Value *V)
Return true if this pointer is returned by a noalias function.
MemoryEffectsBase< IRMemLocation > MemoryEffects
Summary of how a function affects memory in the program.
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
LLVM_ABI bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr, bool UseVariableInfo=true, bool IgnoreUBImplyingAttrs=true)
Return true if the instruction does not have any effects besides calculating the result and does not ...
bool isa_and_nonnull(const Y &Val)
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
LLVM_ABI ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI Value * simplifyInstruction(Instruction *I, const SimplifyQuery &Q)
See if we can compute a simplified version of this instruction.
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
PotentialValuesState< std::pair< AA::ValueAndContext, AA::ValueScope > > PotentialLLVMValuesState
bool isModSet(const ModRefInfo MRI)
void sort(IteratorTy Start, IteratorTy End)
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
LLVM_ABI bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isPointerTy(const Type *T)
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
LLVM_ABI bool wouldInstructionBeTriviallyDead(const Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction would have no side effects if it was not used.
bool set_union(S1Ty &S1, const S2Ty &S2)
set_union(A, B) - Compute A := A u B, return whether A changed.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI CallBase & promoteCall(CallBase &CB, Function *Callee, CastInst **RetBitCast=nullptr)
Promote the given indirect call site to unconditionally call Callee.
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI bool hasAssumption(const Function &F, const KnownAssumptionString &AssumptionStr)
Return true if F has the assumption AssumptionStr attached.
@ Success
The lock was released successfully.
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
AtomicOrdering
Atomic ordering for LLVM's memory model.
ModRefInfo
Flags indicating whether a memory access modifies or references memory.
PotentialValuesState< APInt > PotentialConstantIntValuesState
std::string join(IteratorT Begin, IteratorT End, StringRef Separator)
Joins the strings in the range [Begin, End), adding Separator between the elements.
InterleavedRange< Range > interleaved_array(const Range &R, StringRef Separator=", ")
Output range R as an array of interleaved elements.
ChangeStatus clampStateAndIndicateChange< DerefState >(DerefState &S, const DerefState &R)
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr, const MetadataPredicate *IdentityMD=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM.
LLVM_ABI bool isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(const CallBase *Call, bool MustPreserveOffset, bool MustPreserveProvenance=false)
launder.invariant.group and similar intrinsics return a pointer that aliases their argument,...
DWARFExpression::Operation Op
LLVM_ABI bool isGuaranteedNotToBeUndefOrPoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Return true if this function can prove that V does not have undef bits and is never poison.
ArrayRef(const T &OneElt) -> ArrayRef< T >
LLVM_ABI Value * getFreedOperand(const CallBase *CB, const TargetLibraryInfo *TLI)
If this if a call to a free function, return the freed operand.
ChangeStatus clampStateAndIndicateChange(StateType &S, const StateType &R)
Helper function to clamp a state S of type StateType with the information in R and indicate/return if...
constexpr unsigned BitWidth
ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy
auto pred_begin(const MachineBasicBlock *BB)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
iterator_range< pointer_iterator< WrappedIteratorT > > make_pointer_range(RangeT &&Range)
LLVM_ABI std::optional< APInt > getAllocSize(const CallBase *CB, const TargetLibraryInfo *TLI, function_ref< const Value *(const Value *)> Mapper=[](const Value *V) { return V;})
Return the size of the requested allocation.
LLVM_ABI DenseSet< StringRef > getAssumptions(const Function &F)
Return the set of all assumptions for the function F.
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
@ OPTIONAL
The target may be valid if the source is not.
@ NONE
Do not track a dependence between source and target.
@ REQUIRED
The target cannot be valid if the source is not.
LLVM_ABI UseCaptureInfo DetermineUseCaptureKind(const Use &U, const Value *Base)
Determine what kind of capture behaviour U may exhibit.
LLVM_ABI Value * simplifyCmpInst(CmpPredicate Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q)
Given operands for a CmpInst, fold the result or return null.
LLVM_ABI bool mayContainIrreducibleControl(const Function &F, const LoopInfo *LI)
BumpPtrAllocatorImpl<> BumpPtrAllocator
The standard BumpPtrAllocator which just uses the default template parameters.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
bool capturesNothing(CaptureComponents CC)
LLVM_ABI AssumeDereferenceableInfo getAssumeDereferenceableInfo(OperandBundleUse)
LLVM_ABI bool isIdentifiedObject(const Value *V)
Return true if this pointer refers to a distinct and identifiable object.
bool capturesAnyProvenance(CaptureComponents CC)
constexpr StringRef AssumptionAttrKey
The key we use for assumption attributes.
constexpr bool isCallableCC(CallingConv::ID CC)
bool isRefSet(const ModRefInfo MRI)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A type to track pointer/struct usage and accesses for AAPointerInfo.
bool forallInterferingAccesses(AA::RangeTy Range, F CB) const
See AAPointerInfo::forallInterferingAccesses.
AAPointerInfo::const_bin_iterator end() const
ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I, std::optional< Value * > Content, AAPointerInfo::AccessKind Kind, Type *Ty, Instruction *RemoteI=nullptr)
Add a new Access to the state at offset Offset and with size Size.
DenseMap< const Instruction *, SmallVector< unsigned > > RemoteIMap
AAPointerInfo::const_bin_iterator begin() const
AAPointerInfo::OffsetInfo ReturnedOffsets
Flag to determine if the underlying pointer is reaching a return statement in the associated function...
State & operator=(State &&R)
State(State &&SIS)=default
const AAPointerInfo::Access & getAccess(unsigned Index) const
SmallVector< AAPointerInfo::Access > AccessList
bool isAtFixpoint() const override
Return if this abstract state is fixed, thus does not need to be updated if information changes as it...
bool forallInterferingAccesses(Instruction &I, F CB, AA::RangeTy &Range) const
See AAPointerInfo::forallInterferingAccesses.
static State getWorstState(const State &SIS)
Return the worst possible representable state.
int64_t numOffsetBins() const
AAPointerInfo::OffsetBinsTy OffsetBins
ChangeStatus indicateOptimisticFixpoint() override
Indicate that the abstract state should converge to the optimistic state.
State & operator=(const State &R)
ChangeStatus indicatePessimisticFixpoint() override
Indicate that the abstract state should converge to the pessimistic state.
const State & getAssumed() const
static State getBestState(const State &SIS)
Return the best possible representable state.
bool isValidState() const override
Return if this abstract state is in a valid state.
----------------—AAIntraFnReachability Attribute-----------------------—
ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
unsigned Hash
Precomputed hash for this RQI.
const Instruction * From
Start here,.
Reachable Result
and remember if it worked:
ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To, const AA::InstExclusionSetTy *ES, bool MakeUnique)
Constructor replacement to ensure unique and stable sets are used for the cache.
const ToTy * To
reach this place,
const AA::InstExclusionSetTy * ExclusionSet
without going through any of these instructions,
unsigned computeHashValue() const
An abstract interface for address space information.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all align attributes.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
Align getKnownAlign() const
Return known alignment.
static LLVM_ABI const char ID
An abstract attribute for getting assumption information.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract state for querying live call edges.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract Attribute for specializing "dynamic" components of denormal_fpenv to a known denormal mod...
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all dereferenceable attribute.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for llvm::GlobalValue information interference.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for indirect call information interference.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface to track if a value leaves it's defining function instance.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract Attribute for computing reachability between functions.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
bool canReach(Attributor &A, const Function &Fn) const
If the function represented by this possition can reach Fn.
virtual bool instructionCanReach(Attributor &A, const Instruction &Inst, const Function &Fn, const AA::InstExclusionSetTy *ExclusionSet=nullptr) const =0
Can Inst reach Fn.
An abstract interface to determine reachability of point A to B.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for identifying pointers from which loads can be marked invariant.
static LLVM_ABI const char ID
Unique ID (due to the unique address).
An abstract interface for liveness abstract attribute.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for memory access kind related attributes (readnone/readonly/writeonly).
bool isAssumedReadOnly() const
Return true if we assume that the underlying value is not accessed (=written) in its respective scope...
bool isKnownReadNone() const
Return true if we know that the underlying value is not read or accessed in its respective scope.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
bool isAssumedReadNone() const
Return true if we assume that the underlying value is not read or accessed in its respective scope.
An abstract interface for all memory location attributes (readnone/argmemonly/inaccessiblememonly/ina...
static LLVM_ABI std::string getMemoryLocationsAsStr(MemoryLocationsKind MLK)
Return the locations encoded by MLK as a readable string.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
StateType::base_t MemoryLocationsKind
An abstract interface for all nonnull attributes.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for potential address space information.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all noalias attributes.
static LLVM_ABI bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
---------------------— NoAlias Argument Attribute ---------------------—
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all nocapture attributes.
@ NO_CAPTURE_MAYBE_RETURNED
If we do not capture the value in memory or through integers we can only communicate it back as a der...
@ NO_CAPTURE
If we do not capture the value in memory, through integers, or as a derived pointer we know it is not...
static LLVM_ABI const char ID
Unique ID (due to the unique address)
bool isAssumedNoCaptureMaybeReturned() const
Return true if we assume that the underlying value is not captured in its respective scope but we all...
static LLVM_ABI bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
--------------------— Variable Capturing ------------------------------—
static LLVM_ABI void determineFunctionCaptureCapabilities(const IRPosition &IRP, const Function &F, BitIntegerState &State)
Update State according to the capture capabilities of F for position IRP.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An AbstractAttribute for nofree.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract attribute for norecurse.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An AbstractAttribute for noreturn.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI bool isAlignedBarrier(const CallBase &CB, bool ExecutedAligned)
Helper function to determine if CB is an aligned (GPU) barrier.
static LLVM_ABI bool isNonRelaxedAtomic(const Instruction *I)
Helper function used to determine whether an instruction is non-relaxed atomic.
An abstract interface for all noundef attributes.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
---------------------— NoUndef Attribute ------------------------------—
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract Attribute for determining the necessity of the convergent attribute.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for all nonnull attributes.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI bool isImpliedByIR(Attributor &A, const IRPosition &IRP, Attribute::AttrKind ImpliedAttributeKind, bool IgnoreSubsumingPositions=false)
---------------------— NonNull Argument Attribute ---------------------—
A helper containing a list of offsets computed for a Use.
A container for a list of ranges.
static void set_difference(const RangeList &L, const RangeList &R, RangeList &D)
Copy ranges from L that are not in R, into D.
An abstract interface for struct information.
virtual bool reachesReturn() const =0
OffsetBinsTy::const_iterator const_bin_iterator
virtual const_bin_iterator begin() const =0
DenseMap< AA::RangeTy, SmallSet< unsigned, 4 > > OffsetBinsTy
static LLVM_ABI const char ID
Unique ID (due to the unique address)
virtual int64_t numOffsetBins() const =0
An abstract interface for potential values analysis.
StateWrapper< PotentialConstantIntValuesState, AbstractAttribute, bool, unsigned > Base
static LLVM_ABI const char ID
Unique ID (due to the unique address)
LLVM_ABI AAPotentialConstantValues(const IRPosition &IRP, Attributor &A)
---------------— Potential Values Attribute ----------------------—
static LLVM_ABI const char ID
Unique ID (due to the unique address)
static LLVM_ABI Value * getSingleValue(Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP, SmallVectorImpl< AA::ValueAndContext > &Values)
Extract the single value in Values if any.
An abstract interface for privatizability.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract attribute for undefined behavior.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract attribute for getting all assumption underlying objects.
virtual bool forallUnderlyingObjects(function_ref< bool(Value &)> Pred, AA::ValueScope Scope=AA::Interprocedural) const =0
Check Pred on all underlying objects in Scope collected so far.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for range value analysis.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract interface for value simplify abstract attribute.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
An abstract attribute for willreturn.
static LLVM_ABI const char ID
Unique ID (due to the unique address)
Helper to represent an access offset and size, with logic to deal with uncertainty and check for over...
static constexpr int64_t Unknown
static RangeTy getUnknown()
const Instruction * getCtxI() const
Base struct for all "concrete attribute" deductions.
void print(raw_ostream &OS) const
Helper functions, for debug purposes only.
virtual StateType & getState()=0
Return the internal abstract state for inspection.
virtual void initialize(Attributor &A)
Initialize the state with the information in the Attributor A.
An interface to query the internal state of an abstract attribute.
virtual bool isAtFixpoint() const =0
Return if this abstract state is fixed, thus does not need to be updated if information changes as it...
virtual bool isValidState() const =0
Return if this abstract state is in a valid state.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
static constexpr Align fromLog2(unsigned Log2Value)
Returns an alignment of 1 << Log2Value bytes.
std::function< void( const ArgumentReplacementInfo &, Function &, Function::arg_iterator)> CalleeRepairCBTy
Callee repair callback type.
const Argument & getReplacedArg() const
std::function< void(const ArgumentReplacementInfo &, AbstractCallSite, SmallVectorImpl< Value * > &)> ACSRepairCBTy
Abstract call site (ACS) repair callback type.
The fixpoint analysis framework that orchestrates the attribute deduction.
std::function< std::optional< Value * >( const IRPosition &, const AbstractAttribute *, bool &)> SimplifictionCallbackTy
Register CB as a simplification callback.
Specialization of the integer state for a bit-wise encoding.
BitIntegerState & addKnownBits(base_t Bits)
Add the bits in BitsEncoding to the "known bits".
Simple wrapper for a single bit (boolean) state.
static constexpr DenormalFPEnv getDefault()
static unsigned getHashValue(const Access &A)
AAPointerInfo::Access Access
static bool isEqual(const Access &LHS, const Access &RHS)
static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B)
static unsigned getHashValue(const AA::RangeTy &Range)
DenseMapInfo< std::pair< const Instruction *, const ToTy * > > PairDMI
static bool isEqual(const ReachabilityQueryInfo< ToTy > *LHS, const ReachabilityQueryInfo< ToTy > *RHS)
DenseMapInfo< const AA::InstExclusionSetTy * > InstSetDMI
static unsigned getHashValue(const ReachabilityQueryInfo< ToTy > *RQI)
An information struct used to provide DenseMap with the various necessary components for a given valu...
State for dereferenceable attribute.
IncIntegerState DerefBytesState
State representing for dereferenceable bytes.
ChangeStatus manifest(Attributor &A) override
Helper to describe and deal with positions in the LLVM-IR.
Function * getAssociatedFunction() const
Return the associated function, if any.
static const IRPosition callsite_returned(const CallBase &CB)
Create a position describing the returned value of CB.
static const IRPosition returned(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the returned value of F.
LLVM_ABI Argument * getAssociatedArgument() const
Return the associated argument, if any.
static const IRPosition value(const Value &V, const CallBaseContext *CBContext=nullptr)
Create a position describing the value of V.
int getCalleeArgNo() const
Return the callee argument number of the associated value if it is an argument or call site argument,...
static const IRPosition inst(const Instruction &I, const CallBaseContext *CBContext=nullptr)
Create a position describing the instruction I.
static const IRPosition callsite_argument(const CallBase &CB, unsigned ArgNo)
Create a position describing the argument of CB at position ArgNo.
Kind
The positions we distinguish in the IR.
@ IRP_ARGUMENT
An attribute for a function argument.
@ IRP_RETURNED
An attribute for the function return value.
@ IRP_CALL_SITE
An attribute for a call site (function scope).
@ IRP_CALL_SITE_RETURNED
An attribute for a call site return value.
@ IRP_FUNCTION
An attribute for a function (scope).
@ IRP_CALL_SITE_ARGUMENT
An attribute for a call site argument.
@ IRP_INVALID
An invalid position.
Instruction * getCtxI() const
Return the context instruction, if any.
static const IRPosition argument(const Argument &Arg, const CallBaseContext *CBContext=nullptr)
Create a position describing the argument Arg.
Type * getAssociatedType() const
Return the type this abstract attribute is associated with.
static const IRPosition function(const Function &F, const CallBaseContext *CBContext=nullptr)
Create a position describing the function scope of F.
const CallBaseContext * getCallBaseContext() const
Get the call base context from the position.
Value & getAssociatedValue() const
Return the value this abstract attribute is associated with.
Value & getAnchorValue() const
Return the value this abstract attribute is anchored with.
int getCallSiteArgNo() const
Return the call site argument number of the associated value if it is an argument or call site argume...
static const IRPosition function_scope(const IRPosition &IRP, const CallBaseContext *CBContext=nullptr)
Create a position with function scope matching the "context" of IRP.
Kind getPositionKind() const
Return the associated position kind.
bool isArgumentPosition() const
Return true if the position is an argument or call site argument.
static const IRPosition callsite_function(const CallBase &CB)
Create a position describing the function scope of CB.
Function * getAnchorScope() const
Return the Function surrounding the anchor value.
ConstantRange getKnown() const
Return the known state encoding.
ConstantRange getAssumed() const
Return the assumed state encoding.
base_t getAssumed() const
Return the assumed state encoding.
static constexpr base_t getWorstState()
Helper that allows to insert a new assumption string in the known assumption set by creating a (stati...
A "must be executed context" for a given program point PP is the set of instructions,...
LLVM_ABI bool checkForAllTerminatorsInContext(const Instruction *PP, function_ref< bool(const Instruction *)> Pred)
Check Pred on all terminators in the context of PP and return true if it holds for all of them.
LLVM_ABI bool isInContextOf(const Instruction *I, const Instruction *PP)
}
Helper to tie a abstract state implementation to an abstract attribute.
StateType & getState() override
bool isPassthrough() const
CaptureComponents ResultCC
Components captured by the return value of the user of this Use.
LLVM_ABI bool unionAssumed(std::optional< Value * > Other)
Merge Other into the currently assumed simplified value.
std::optional< Value * > SimplifiedAssociatedValue
An assumed simplified value.
Type * Ty
The type of the original value.