45 #define DEBUG_TYPE "memoryssa"
47 STATISTIC(NumClobberCacheLookups,
"Number of Memory SSA version cache lookups");
48 STATISTIC(NumClobberCacheHits,
"Number of Memory SSA version cache hits");
49 STATISTIC(NumClobberCacheInserts,
"Number of MemorySSA version cache inserts");
62 "
Memory SSA Printer", false, false)
67 "will consider trying to walk past (default = 100)"));
86 OS <<
"; " << *MA <<
"\n";
92 OS <<
"; " << *MA <<
"\n";
103 class MemoryLocOrCall {
105 MemoryLocOrCall() : IsCall(
false) {}
107 : MemoryLocOrCall(MUD->getMemoryInst()) {}
109 : MemoryLocOrCall(MUD->getMemoryInst()) {}
119 if (!isa<FenceInst>(Inst))
125 : IsCall(
false), Loc(Loc) {}
137 bool operator==(
const MemoryLocOrCall &Other)
const {
138 if (IsCall != Other.IsCall)
142 return CS.getCalledValue() == Other.CS.getCalledValue();
143 return Loc == Other.Loc;
166 MLOC.getCS().getCalledValue()));
170 static bool isEqual(
const MemoryLocOrCall &LHS,
const MemoryLocOrCall &RHS) {
186 bool VolatileClobber = MayClobber->
isVolatile();
188 if (VolatileUse && VolatileClobber)
210 if (SeqCstUse || MayClobberIsAcquire)
220 assert(DefInst &&
"Defining instruction not actually an instruction");
222 if (
const IntrinsicInst *II = dyn_cast<IntrinsicInst>(DefInst)) {
225 switch (II->getIntrinsicID()) {
226 case Intrinsic::lifetime_start:
227 case Intrinsic::lifetime_end:
228 case Intrinsic::invariant_start:
229 case Intrinsic::invariant_end:
230 case Intrinsic::assume:
243 if (
auto *DefLoad = dyn_cast<LoadInst>(DefInst)) {
244 if (
auto *UseLoad = dyn_cast<LoadInst>(UseInst)) {
260 const MemoryLocOrCall &UseMLOC,
279 struct UpwardsMemoryQuery {
291 : IsCall(
false), Inst(nullptr), OriginalAccess(nullptr) {}
304 switch (II->getIntrinsicID()) {
305 case Intrinsic::lifetime_start:
306 case Intrinsic::lifetime_end:
315 static bool isUseTriviallyOptimizableToLiveOnEntry(
AliasAnalysis &AA,
334 ++NumClobberCacheLookups;
335 MemoryAccess *R = IsCall ? Calls.lookup(MA) : Accesses.lookup({MA, Loc});
337 ++NumClobberCacheHits;
345 assert((MA != To || isa<MemoryPhi>(MA)) &&
346 "Something can't clobber itself!");
348 ++NumClobberCacheInserts;
351 Inserted = Calls.insert({MA, To}).second;
353 Inserted = Accesses.insert({{MA, Loc}, To}).second;
359 return IsCall ? Calls.erase(MA) : Accesses.erase({MA, Loc});
368 for (
auto &
P : Accesses)
369 if (
P.first.first == MA ||
P.second == MA)
371 for (
auto &
P : Calls)
372 if (
P.first == MA ||
P.second == MA)
382 struct def_chain_iterator
385 def_chain_iterator() : MA(nullptr) {}
390 def_chain_iterator &operator++() {
392 if (
auto *MUD = dyn_cast<MemoryUseOrDef>(MA))
393 MA = MUD->getDefiningAccess();
399 bool operator==(
const def_chain_iterator &O)
const {
return MA == O.MA; }
407 #ifdef EXPENSIVE_CHECKS
408 assert((!UpTo ||
find(def_chain(MA), UpTo) != def_chain_iterator()) &&
409 "UpTo isn't in the def chain!");
411 return make_range(def_chain_iterator(MA), def_chain_iterator(UpTo));
430 assert(MSSA.
dominates(ClobberAt, Start) &&
"Clobber doesn't dominate start?");
434 "liveOnEntry must clobber itself");
438 bool FoundClobber =
false;
444 while (!Worklist.
empty()) {
448 if (!VisitedPhis.
insert(MAP).second)
452 if (MA == ClobberAt) {
453 if (
auto *MD = dyn_cast<MemoryDef>(MA)) {
469 if (
auto *MD = dyn_cast<MemoryDef>(MA)) {
472 "Found clobber before reaching ClobberAt!");
476 assert(isa<MemoryPhi>(MA));
483 assert((isa<MemoryPhi>(ClobberAt) || FoundClobber) &&
484 "ClobberAt never acted as a clobber");
489 class ClobberWalker {
505 : Loc(Loc), First(First), Last(Last), Previous(Previous) {}
509 : DefPath(Loc, Init, Init, Previous) {}
516 UpwardsMemoryQuery *
Query;
524 void setUseCache(
bool Use) { UseCache = Use; }
525 bool shouldIgnoreCache()
const {
534 #ifdef EXPENSIVE_CHECKS
537 if (shouldIgnoreCache())
539 WC.insert(What, To, Loc, Query->IsCall);
543 return shouldIgnoreCache() ?
nullptr : WC.lookup(MA, Loc, Query->IsCall);
547 if (shouldIgnoreCache())
551 addCacheEntry(MA, Target, DN.Loc);
555 if (DN.Last != Target)
556 addCacheEntry(DN.Last, Target, DN.Loc);
568 auto At = WalkTargetCache.find(BB);
569 if (At != WalkTargetCache.end())
577 while ((Node = Node->
getIDom())) {
578 auto At = WalkTargetCache.find(BB);
579 if (At != WalkTargetCache.end()) {
587 return !isa<MemoryUse>(MA);
589 if (Iter != Accesses->rend()) {
595 ToCache.push_back(Node->
getBlock());
599 WalkTargetCache.insert({BB, Result});
604 struct UpwardsWalkResult {
617 UpwardsWalkResult walkToPhiOrClobber(DefPath &
Desc,
619 assert(!isa<MemoryUse>(Desc.Last) &&
"Uses don't exist in my world");
623 if (Current == StopAt)
624 return {Current,
false,
false};
626 if (
auto *MD = dyn_cast<MemoryDef>(Current))
629 return {MD,
true,
false};
634 return {MA,
true,
true};
637 assert(isa<MemoryPhi>(Desc.Last) &&
638 "Ended at a non-clobber that's not a phi?");
639 return {Desc.Last,
false,
false};
643 ListIndex PriorNode) {
648 Paths.emplace_back(
P.second,
P.first, PriorNode);
655 struct TerminatedPath {
673 assert(!PausedSearches.
empty() &&
"No searches to continue?");
677 while (!PausedSearches.
empty()) {
679 DefPath &Node = Paths[PathIndex];
699 if (!VisitedPhis.
insert({Node.Last, Node.Loc}).second)
702 UpwardsWalkResult Res = walkToPhiOrClobber(Node, StopWhere);
703 if (Res.IsKnownClobber) {
704 assert(Res.Result != StopWhere || Res.FromCache);
707 TerminatedPath Term{Res.Result, PathIndex};
708 if (!Res.FromCache || !MSSA.
dominates(Res.Result, StopWhere))
716 if (Res.Result == StopWhere) {
724 addSearches(cast<MemoryPhi>(Res.Result), PausedSearches, PathIndex);
730 template <
typename T,
typename Walker>
731 struct generic_def_path_iterator
733 std::forward_iterator_tag, T *> {
734 generic_def_path_iterator() : W(nullptr),
N(
None) {}
735 generic_def_path_iterator(Walker *W, ListIndex
N) : W(W), N(N) {}
739 generic_def_path_iterator &operator++() {
740 N = curNode().Previous;
744 bool operator==(
const generic_def_path_iterator &O)
const {
745 if (
N.hasValue() != O.N.hasValue())
747 return !
N.hasValue() || *
N == *O.N;
751 T &curNode()
const {
return W->Paths[*
N]; }
757 using def_path_iterator = generic_def_path_iterator<DefPath, ClobberWalker>;
758 using const_def_path_iterator =
759 generic_def_path_iterator<const DefPath, const ClobberWalker>;
762 return make_range(def_path_iterator(
this, From), def_path_iterator());
766 return make_range(const_def_path_iterator(
this, From),
767 const_def_path_iterator());
772 TerminatedPath PrimaryClobber;
778 ListIndex defPathIndex(
const DefPath &
N)
const {
780 const DefPath *NP = &
N;
781 assert(!Paths.empty() && NP >= &Paths.front() && NP <= &Paths.back() &&
782 "Out of bounds DefPath!");
783 return NP - &Paths.front();
802 "Reset the optimization state.");
804 Paths.emplace_back(Loc, Start, Phi,
None);
807 auto PriorPathsSize = Paths.size();
813 addSearches(Phi, PausedSearches, 0);
818 assert(!Paths.empty() &&
"Need a path to move");
819 auto Dom = Paths.begin();
820 for (
auto I = std::next(Dom),
E = Paths.end(); I !=
E; ++
I)
821 if (!MSSA.
dominates(I->Clobber, Dom->Clobber))
823 auto Last = Paths.end() - 1;
825 std::iter_swap(Last, Dom);
831 "liveOnEntry wasn't treated as a clobber?");
836 assert(
all_of(TerminatedPaths, [&](
const TerminatedPath &
P) {
837 return MSSA.
dominates(P.Clobber, Target);
844 Target, PausedSearches, NewPaused, TerminatedPaths)) {
846 cacheDefPath(Paths[Blocker->LastNode], Blocker->Clobber);
850 auto Iter =
find_if(def_path(Blocker->LastNode), [&](
const DefPath &N) {
851 return defPathIndex(N) < PriorPathsSize;
853 assert(Iter != def_path_iterator());
855 DefPath &CurNode = *Iter;
856 assert(CurNode.Last == Current);
883 TerminatedPath Result{CurNode.Last, defPathIndex(CurNode)};
890 if (NewPaused.
empty()) {
891 MoveDominatedPathToEnd(TerminatedPaths);
893 return {Result, std::move(TerminatedPaths)};
898 for (ListIndex Paused : NewPaused) {
899 UpwardsWalkResult WR = walkToPhiOrClobber(Paths[Paused]);
900 if (WR.IsKnownClobber)
904 DefChainEnd = WR.Result;
907 if (!TerminatedPaths.
empty()) {
917 for (
const TerminatedPath &TP : TerminatedPaths) {
920 if (DT.dominates(ChainBB, TP.Clobber->getBlock()))
927 if (!Clobbers.
empty()) {
928 MoveDominatedPathToEnd(Clobbers);
930 return {Result, std::move(Clobbers)};
934 [&](ListIndex I) {
return Paths[
I].Last == DefChainEnd; }));
937 auto *DefChainPhi = cast<MemoryPhi>(DefChainEnd);
939 PriorPathsSize = Paths.size();
940 PausedSearches.
clear();
941 for (ListIndex I : NewPaused)
942 addSearches(DefChainPhi, PausedSearches, I);
945 Current = DefChainPhi;
950 void cacheOptResult(
const OptznResult &R) {
951 if (R.OtherClobbers.empty()) {
954 for (
const DefPath &N : const_def_path(R.PrimaryClobber.LastNode))
955 cacheDefPath(N, R.PrimaryClobber.Clobber);
962 for (
const DefPath &N : const_def_path(R.PrimaryClobber.LastNode)) {
963 Visited[defPathIndex(N)] =
true;
964 cacheDefPath(N, R.PrimaryClobber.Clobber);
967 for (
const TerminatedPath &P : R.OtherClobbers) {
968 for (
const DefPath &N : const_def_path(P.LastNode)) {
969 ListIndex NIndex = defPathIndex(N);
972 Visited[NIndex] =
true;
973 cacheDefPath(N, P.Clobber);
978 void verifyOptResult(
const OptznResult &R)
const {
979 assert(
all_of(R.OtherClobbers, [&](
const TerminatedPath &P) {
980 return MSSA.
dominates(P.Clobber, R.PrimaryClobber.Clobber);
984 void resetPhiOptznState() {
992 : MSSA(MSSA), AA(AA), DT(DT), WC(WC), UseCache(
true) {}
994 void reset() { WalkTargetCache.clear(); }
999 bool UseWalkerCache =
true) {
1000 setUseCache(UseWalkerCache);
1006 if (
auto *MU = dyn_cast<MemoryUse>(Start))
1007 Current = MU->getDefiningAccess();
1009 DefPath FirstDesc(Q.StartingLoc, Current, Current,
None);
1012 UpwardsWalkResult WalkResult = walkToPhiOrClobber(FirstDesc);
1014 if (WalkResult.IsKnownClobber) {
1015 cacheDefPath(FirstDesc, WalkResult.Result);
1016 Result = WalkResult.Result;
1018 OptznResult OptRes = tryOptimizePhi(cast<MemoryPhi>(FirstDesc.Last),
1019 Current, Q.StartingLoc);
1020 verifyOptResult(OptRes);
1021 cacheOptResult(OptRes);
1022 resetPhiOptznState();
1023 Result = OptRes.PrimaryClobber.Clobber;
1026 #ifdef EXPENSIVE_CHECKS
1027 checkClobberSanity(Current, Result, Q.StartingLoc, MSSA, Q, AA);
1035 struct RenamePassData {
1042 : DTN(D), ChildIt(It), IncomingVal(M) {}
1043 void swap(RenamePassData &RHS) {
1046 std::swap(IncomingVal, RHS.IncomingVal);
1087 ClobberWalker Walker;
1088 bool AutoResetWalker;
1115 Walker.verify(MSSA);
1124 auto It = PerBlockAccesses.find(BB);
1126 if (It != PerBlockAccesses.end()) {
1130 if (MUD->getDefiningAccess() ==
nullptr)
1131 MUD->setDefiningAccess(IncomingVal);
1132 if (isa<MemoryDef>(&
L))
1142 auto It = PerBlockAccesses.find(S);
1144 if (It == PerBlockAccesses.end() || !isa<MemoryPhi>(It->second->front()))
1147 auto *Phi = cast<MemoryPhi>(&Accesses->front());
1161 IncomingVal = renameBlock(Root->
getBlock(), IncomingVal);
1165 while (!WorkStack.
empty()) {
1168 IncomingVal = WorkStack.
back().IncomingVal;
1170 if (ChildIt == Node->
end()) {
1174 ++WorkStack.
back().ChildIt;
1177 IncomingVal = renameBlock(BB, IncomingVal);
1187 DomLevels[*DFI] = DFI.getPathLength() - 1;
1193 void MemorySSA::markUnreachableAsLiveOnEntry(
BasicBlock *BB) {
1195 "Reachable block found while handling unreachable blocks");
1204 auto It = PerBlockAccesses.find(S);
1206 if (It == PerBlockAccesses.end() || !isa<MemoryPhi>(It->second->front()))
1209 auto *Phi = cast<MemoryPhi>(&Accesses->front());
1213 auto It = PerBlockAccesses.find(BB);
1214 if (It == PerBlockAccesses.end())
1217 auto &Accesses = It->second;
1218 for (
auto AI = Accesses->begin(), AE = Accesses->end(); AI != AE;) {
1219 auto Next = std::next(AI);
1222 if (
auto *UseOrDef = dyn_cast<MemoryUseOrDef>(AI))
1223 UseOrDef->setDefiningAccess(LiveOnEntryDef.get());
1225 Accesses->erase(AI);
1231 : AA(AA), DT(DT),
F(Func), LiveOnEntryDef(nullptr), Walker(nullptr),
1238 for (
const auto &Pair : PerBlockAccesses)
1244 auto Res = PerBlockAccesses.insert(std::make_pair(BB,
nullptr));
1247 Res.first->second = make_unique<AccessList>();
1248 return Res.first->second.get();
1262 : MSSA(MSSA), Walker(Walker), AA(AA), DT(DT) {
1270 struct MemlocStackInfo {
1273 unsigned long StackEpoch;
1274 unsigned long PopEpoch;
1279 unsigned long LowerBound;
1282 unsigned long LastKill;
1285 void optimizeUsesInBlock(
const BasicBlock *,
unsigned long &,
unsigned long &,
1306 void MemorySSA::OptimizeUses::optimizeUsesInBlock(
1307 const BasicBlock *BB,
unsigned long &StackEpoch,
unsigned long &PopEpoch,
1313 if (Accesses ==
nullptr)
1318 while (!VersionStack.
empty()) {
1334 if (isUseTriviallyOptimizableToLiveOnEntry(*AA, MU->getMemoryInst())) {
1339 MemoryLocOrCall UseMLOC(MU);
1340 auto &LocInfo = LocStackInfo[UseMLOC];
1344 if (LocInfo.PopEpoch != PopEpoch) {
1345 LocInfo.PopEpoch = PopEpoch;
1346 LocInfo.StackEpoch = StackEpoch;
1358 if (LocInfo.LowerBoundBlock && LocInfo.LowerBoundBlock != BB &&
1359 !DT->
dominates(LocInfo.LowerBoundBlock, BB)) {
1363 LocInfo.LowerBound = 0;
1364 LocInfo.LowerBoundBlock = VersionStack[0]->getBlock();
1365 LocInfo.LastKillValid =
false;
1367 }
else if (LocInfo.StackEpoch != StackEpoch) {
1371 LocInfo.PopEpoch = PopEpoch;
1372 LocInfo.StackEpoch = StackEpoch;
1374 if (!LocInfo.LastKillValid) {
1375 LocInfo.LastKill = VersionStack.
size() - 1;
1376 LocInfo.LastKillValid =
true;
1381 assert(LocInfo.LowerBound < VersionStack.
size() &&
1382 "Lower bound out of range");
1383 assert(LocInfo.LastKill < VersionStack.
size() &&
1384 "Last kill info out of range");
1386 unsigned long UpperBound = VersionStack.
size() - 1;
1389 DEBUG(
dbgs() <<
"MemorySSA skipping optimization of " << *MU <<
" ("
1390 << *(MU->getMemoryInst()) <<
")"
1391 <<
" because there are " << UpperBound - LocInfo.LowerBound
1392 <<
" stores to disambiguate\n");
1395 LocInfo.LastKillValid =
false;
1398 bool FoundClobberResult =
false;
1399 while (UpperBound > LocInfo.LowerBound) {
1400 if (isa<MemoryPhi>(VersionStack[UpperBound])) {
1405 while (VersionStack[UpperBound] != Result) {
1409 FoundClobberResult =
true;
1413 MemoryDef *MD = cast<MemoryDef>(VersionStack[UpperBound]);
1416 if (!UseMLOC.IsCall && lifetimeEndsAt(MD, UseMLOC.getLoc(), *AA)) {
1419 FoundClobberResult =
true;
1423 FoundClobberResult =
true;
1430 if (FoundClobberResult || UpperBound < LocInfo.LastKill) {
1431 MU->setDefiningAccess(VersionStack[UpperBound],
true);
1433 LocInfo.LastKill = UpperBound;
1437 MU->setDefiningAccess(VersionStack[LocInfo.LastKill],
true);
1439 LocInfo.LowerBound = VersionStack.
size() - 1;
1440 LocInfo.LowerBoundBlock = BB;
1458 unsigned long StackEpoch = 1;
1459 unsigned long PopEpoch = 1;
1461 optimizeUsesInBlock(DomNode->getBlock(), StackEpoch, PopEpoch, VersionStack,
1465 void MemorySSA::placePHINodes(
1470 IDFs.setDefiningBlocks(DefiningBlocks);
1472 IDFs.calculate(IDFBlocks);
1474 std::sort(IDFBlocks.
begin(), IDFBlocks.
end(),
1480 for (
auto &BB : IDFBlocks) {
1482 AccessList *Accesses = getOrCreateAccessList(BB);
1484 ValueToMemoryAccess[BB] = Phi;
1486 Accesses->push_front(Phi);
1490 void MemorySSA::buildMemorySSA() {
1498 LiveOnEntryDef = make_unique<MemoryDef>(F.
getContext(),
nullptr,
nullptr,
1499 &StartingPoint, NextID++);
1501 unsigned NextBBNum = 0;
1511 BBNumbers[&
B] = NextBBNum++;
1512 bool InsertIntoDef =
false;
1518 InsertIntoDef |= isa<MemoryDef>(MUD);
1521 Accesses = getOrCreateAccessList(&B);
1522 Accesses->push_back(MUD);
1525 DefiningBlocks.
insert(&B);
1529 placePHINodes(DefiningBlocks, BBNumbers);
1534 renamePass(DT->
getRootNode(), LiveOnEntryDef.get(), Visited);
1536 CachingWalker *Walker = getWalkerImpl();
1539 Walker->setAutoResetWalker(
false);
1540 OptimizeUses(
this, Walker, AA, DT).optimizeUses();
1541 Walker->setAutoResetWalker(
true);
1542 Walker->resetClobberWalker();
1547 if (!Visited.
count(&BB))
1548 markUnreachableAsLiveOnEntry(&BB);
1555 return Walker.get();
1557 Walker = make_unique<CachingWalker>(
this, AA, DT);
1558 return Walker.get();
1563 AccessList *Accesses = getOrCreateAccessList(BB);
1565 ValueToMemoryAccess[BB] = Phi;
1568 BlockNumberingValid.erase(BB);
1574 assert(!isa<PHINode>(I) &&
"Cannot create a defined access for a PHI");
1577 NewAccess !=
nullptr &&
1578 "Tried to create a memory access for a non-memory touching instruction");
1588 auto *Accesses = getOrCreateAccessList(BB);
1592 *Accesses, [](
const MemoryAccess &MA) {
return !isa<MemoryPhi>(MA); });
1594 Accesses->insert(AI, NewAccess);
1596 Accesses->push_back(NewAccess);
1598 BlockNumberingValid.erase(BB);
1606 "New and old access must be in the same block");
1608 auto *Accesses = getOrCreateAccessList(InsertPt->
getBlock());
1610 BlockNumberingValid.erase(InsertPt->
getBlock());
1618 "New and old access must be in the same block");
1620 auto *Accesses = getOrCreateAccessList(InsertPt->
getBlock());
1622 BlockNumberingValid.erase(InsertPt->
getBlock());
1630 assert(
dominates(Where, What) &&
"Only upwards splices are permitted.");
1634 if (isa<MemoryDef>(What)) {
1644 BlockNumberingValid.erase(What->
getBlock());
1646 BlockNumberingValid.erase(Where->
getBlock());
1656 if (II->getIntrinsicID() == Intrinsic::assume)
1670 "Trying to create a memory access with a non-memory instruction");
1677 ValueToMemoryAccess[
I] = MUD;
1692 CurrNode = CurrNode->
getIDom();
1695 auto It = PerBlockAccesses.find(CurrNode->
getBlock());
1696 if (It != PerBlockAccesses.end()) {
1697 auto &Accesses = It->second;
1699 if (isa<MemoryDef>(RA) || isa<MemoryPhi>(RA))
1703 CurrNode = CurrNode->
getIDom();
1705 return LiveOnEntryDef.get();
1709 bool MemorySSA::dominatesUse(
const MemoryAccess *Replacer,
1711 if (isa<MemoryUseOrDef>(Replacee))
1713 const auto *MP = cast<MemoryPhi>(Replacee);
1717 for (
const Use &Arg : MP->operands()) {
1718 if (Arg.get() != Replacee &&
1732 MA = cast<MemoryAccess>(Arg);
1745 "Trying to remove memory access that still has uses");
1746 BlockNumbering.erase(MA);
1750 if (!isa<MemoryUse>(MA))
1751 Walker->invalidateInfo(MA);
1760 auto VMA = ValueToMemoryAccess.find(MemoryInst);
1761 if (VMA->second == MA)
1762 ValueToMemoryAccess.erase(VMA);
1764 auto AccessIt = PerBlockAccesses.find(MA->
getBlock());
1765 std::unique_ptr<AccessList> &Accesses = AccessIt->second;
1766 Accesses->erase(MA);
1767 if (Accesses->empty())
1768 PerBlockAccesses.erase(AccessIt);
1776 if (
MemoryPhi *MP = dyn_cast<MemoryPhi>(MA)) {
1784 "We can't delete this memory phi");
1786 NewDefTarget = cast<MemoryUseOrDef>(MA)->getDefiningAccess();
1807 MU->resetOptimized();
1808 U.
set(NewDefTarget);
1814 removeFromLookups(MA);
1819 F.print(OS, &Writer);
1824 F.print(
dbgs(), &Writer);
1831 Walker->verify(
this);
1848 assert((!MA || AL) &&
"We have memory affecting instructions "
1849 "in this block but they are not in the "
1858 assert(AL->size() == ActualAccesses.
size() &&
1859 "We don't have the same number of accesses in the block as on the "
1861 auto ALI = AL->begin();
1862 auto AAI = ActualAccesses.
begin();
1863 while (ALI != AL->end() && AAI != ActualAccesses.
end()) {
1864 assert(&*ALI == *AAI &&
"Not the same accesses in the same order");
1868 ActualAccesses.
clear();
1879 for (
const Use &U : MP->uses())
1887 for (
const Use &U : MD->
uses())
1902 "Null def but use not point to live on entry def");
1905 "Did not find use in def's use list");
1918 "Incomplete MemoryPhi Node");
1925 verifyUseInDefs(MA->getDefiningAccess(), MA);
1932 return cast_or_null<MemoryUseOrDef>(ValueToMemoryAccess.lookup(I));
1936 return cast_or_null<MemoryPhi>(ValueToMemoryAccess.lookup(cast<Value>(BB)));
1945 void MemorySSA::renumberBlock(
const BasicBlock *B)
const {
1947 unsigned long CurrentNumber = 0;
1949 assert(AL !=
nullptr &&
"Asking to renumber an empty block");
1950 for (
const auto &I : *AL)
1951 BlockNumbering[&
I] = ++CurrentNumber;
1952 BlockNumberingValid.insert(B);
1964 "Asking for local domination when accesses are in different blocks!");
1966 if (Dominatee == Dominator)
1979 if (!BlockNumberingValid.count(DominatorBlock))
1980 renumberBlock(DominatorBlock);
1982 unsigned long DominatorNum = BlockNumbering.lookup(Dominator);
1984 assert(DominatorNum != 0 &&
"Block was not numbered properly");
1985 unsigned long DominateeNum = BlockNumbering.lookup(Dominatee);
1986 assert(DominateeNum != 0 &&
"Block was not numbered properly");
1987 return DominatorNum < DominateeNum;
1992 if (Dominator == Dominatee)
2004 const Use &Dominatee)
const {
2006 BasicBlock *UseBB = MP->getIncomingBlock(Dominatee);
2008 if (UseBB != Dominator->
getBlock())
2022 OS <<
getID() <<
" = MemoryDef(";
2023 if (UO && UO->
getID())
2032 OS <<
getID() <<
" = MemoryPhi(";
2033 for (
const auto &
Op : operands()) {
2047 if (
unsigned ID = MA->
getID())
2061 if (UO && UO->
getID())
2086 auto &MSSA = getAnalysis<MemorySSAWrapperPass>().getMSSA();
2104 OS <<
"MemorySSA for function: " << F.
getName() <<
"\n";
2132 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
2133 auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
2165 if (
MemoryUse *MU = dyn_cast<MemoryUse>(MA)) {
2166 UpwardsMemoryQuery Q(MU->getMemoryInst(), MU);
2167 Cache.remove(MU, Q.StartingLoc, Q.IsCall);
2168 MU->resetOptimized();
2174 #ifdef EXPENSIVE_CHECKS
2183 MemoryAccess *MemorySSA::CachingWalker::getClobberingMemoryAccess(
2185 MemoryAccess *New = Walker.findClobber(StartingAccess, Q);
2186 #ifdef EXPENSIVE_CHECKS
2188 Walker.findClobber(StartingAccess, Q,
false);
2189 assert(NewNoCache == New &&
"Cache made us hand back a different result?");
2191 if (AutoResetWalker)
2192 resetClobberWalker();
2198 if (isa<MemoryPhi>(StartingAccess))
2199 return StartingAccess;
2201 auto *StartingUseOrDef = cast<MemoryUseOrDef>(StartingAccess);
2203 return StartingUseOrDef;
2205 Instruction *I = StartingUseOrDef->getMemoryInst();
2210 return StartingUseOrDef;
2212 UpwardsMemoryQuery Q;
2213 Q.OriginalAccess = StartingUseOrDef;
2214 Q.StartingLoc = Loc;
2218 if (
auto *CacheResult = Cache.lookup(StartingUseOrDef, Loc, Q.IsCall))
2223 MemoryAccess *DefiningAccess = isa<MemoryUse>(StartingUseOrDef)
2224 ? StartingUseOrDef->getDefiningAccess()
2227 MemoryAccess *Clobber = getClobberingMemoryAccess(DefiningAccess, Q);
2228 DEBUG(
dbgs() <<
"Starting Memory SSA clobber for " << *I <<
" is ");
2229 DEBUG(
dbgs() << *StartingUseOrDef <<
"\n");
2230 DEBUG(
dbgs() <<
"Final Memory SSA clobber for " << *I <<
" is ");
2239 if (!StartingAccess)
2245 if (
MemoryUse *MU = dyn_cast<MemoryUse>(StartingAccess))
2246 if (MU->isOptimized())
2247 return MU->getDefiningAccess();
2249 const Instruction *I = StartingAccess->getMemoryInst();
2250 UpwardsMemoryQuery Q(I, StartingAccess);
2255 return StartingAccess;
2257 if (
auto *CacheResult = Cache.lookup(StartingAccess, Q.StartingLoc, Q.IsCall))
2260 if (isUseTriviallyOptimizableToLiveOnEntry(*MSSA->AA, I)) {
2262 Cache.insert(StartingAccess, LiveOnEntry, Q.StartingLoc, Q.IsCall);
2263 if (
MemoryUse *MU = dyn_cast<MemoryUse>(StartingAccess))
2264 MU->setDefiningAccess(LiveOnEntry,
true);
2269 MemoryAccess *DefiningAccess = StartingAccess->getDefiningAccess();
2274 return DefiningAccess;
2276 MemoryAccess *Result = getClobberingMemoryAccess(DefiningAccess, Q);
2277 DEBUG(
dbgs() <<
"Starting Memory SSA clobber for " << *I <<
" is ");
2278 DEBUG(
dbgs() << *DefiningAccess <<
"\n");
2279 DEBUG(
dbgs() <<
"Final Memory SSA clobber for " << *I <<
" is ");
2281 if (
MemoryUse *MU = dyn_cast<MemoryUse>(StartingAccess))
2282 MU->setDefiningAccess(Result,
true);
2288 void MemorySSA::CachingWalker::verifyRemoved(
MemoryAccess *MA) {
2289 assert(!Cache.contains(MA) &&
"Found removed MemoryAccess in cache.");
2294 if (
auto *Use = dyn_cast<MemoryUseOrDef>(MA))
2295 return Use->getDefiningAccess();
2301 if (
auto *Use = dyn_cast<MemoryUseOrDef>(StartingAccess))
2302 return Use->getDefiningAccess();
2303 return StartingAccess;
static Reorderability getLoadReorderability(const LoadInst *Use, const LoadInst *MayClobber)
This does one-way checks to see if Use could theoretically be hoisted above MayClobber.
void initializeMemorySSAWrapperPassPass(PassRegistry &)
void push_back(const T &Elt)
bool dominates(const MemoryAccess *A, const MemoryAccess *B) const
Given two memory accesses in potentially different blocks, determine whether MemoryAccess A dominates...
MemoryAccess * getClobberingMemoryAccess(MemoryAccess *) override
Does the same thing as getClobberingMemoryAccess(const Instruction *I), but takes a MemoryAccess inst...
iterator_range< use_iterator > uses()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
virtual void verify(const MemorySSA *MSSA)
void dropAllReferences()
Drop all references to operands.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool isFenceLike() const
Return true if this instruction behaves like a memory fence: it can load or store to memory location ...
STATISTIC(NumFunctions,"Total number of functions")
virtual void emitBasicBlockStartAnnot(const BasicBlock *BB, formatted_raw_ostream &OS)
emitBasicBlockStartAnnot - This may be implemented to emit a string right after the basic block label...
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
This is the interface for a simple mod/ref and alias analysis over globals.
A Module instance is used to store all the information related to an LLVM module. ...
#define LLVM_UNLIKELY(EXPR)
Result run(Function &F, FunctionAnalysisManager &AM)
bool defClobbersUseOrDef(MemoryDef *MD, const MemoryUseOrDef *MU, AliasAnalysis &AA)
This class provides various memory handling functions that manipulate MemoryBlock instances...
Implements a dense probed hash-table based set.
unsigned getNumOperands() const
bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB)
A trivial helper function to check to see if the specified pointers are no-alias. ...
MachineInstrBuilder MachineInstrBuilder &DefMI const MCInstrDesc & Desc
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Represents a read-write access to memory, whether it is a must-alias, or a may-alias.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
void print(raw_ostream &OS) const override
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
virtual void emitInstructionAnnot(const Instruction *I, formatted_raw_ostream &OS)
emitInstructionAnnot - This may be implemented to emit a string right before an instruction is emitte...
Analysis pass which computes a DominatorTree.
OptimizeUses(MemorySSA *MSSA, MemorySSAWalker *Walker, AliasAnalysis *AA, DominatorTree *DT)
An instruction for reading from memory.
The access modifies the value stored in memory.
INITIALIZE_PASS_BEGIN(MemorySSAWrapperPass,"memoryssa","Memory SSA", false, true) INITIALIZE_PASS_END(MemorySSAWrapperPass
MemoryUseOrDef * createMemoryAccessBefore(Instruction *I, MemoryAccess *Definition, MemoryUseOrDef *InsertPt)
Create a MemoryAccess in MemorySSA before or after an existing MemoryAccess.
return AArch64::GPR64RegClass contains(Reg)
StringRef getName() const
Return a constant reference to the value's name.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
DomTreeNodeBase< NodeT > * getIDom() const
Represents read-only accesses to memory.
virtual void dump() const
safe Safe Stack instrumentation pass
This class is a batch walker of all MemoryUse's in the program, and points their defining access at t...
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Legacy analysis pass which computes MemorySSA.
ModRefInfo
Flags indicating whether a memory access modifies or references memory.
ppc ctr loops PowerPC CTR Loops Verify
~CachingWalker() override
A MemorySSAWalker that does AA walks and caching of lookups to disambiguate accesses.
A Use represents the edge between a Value definition and its users.
AccessList * getWritableBlockAccesses(const BasicBlock *BB) const
MemorySSAAnnotatedWriter(const MemorySSA *M)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Encapsulates MemorySSA, including all data associated with memory accesses.
MemoryUseOrDef * getMemoryAccess(const Instruction *) const
Given a memory Mod/Ref'ing instruction, get the MemorySSA access associated with it.
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
DomTreeNodeBase< NodeT > * getRootNode()
getRootNode - This returns the entry node for the CFG of the function.
MemoryAccess * getDefiningAccess() const
Get the access that produces the memory state used by this Use.
The access references the value stored in memory.
static const uint16_t * lookup(unsigned opcode, unsigned domain, ArrayRef< uint16_t[3]> Table)
LLVM_NODISCARD bool empty() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
BasicBlock * getBlock() const
CachingWalker(MemorySSA *, AliasAnalysis *, DominatorTree *)
static int getID(struct InternalInstruction *insn, const void *miiArg)
upward_defs_iterator upward_defs_end()
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
void removeMemoryAccess(MemoryAccess *)
Remove a MemoryAccess from MemorySSA, including updating all definitions and uses.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Base class for the actual dominator tree node.
bool runOnFunction(Function &) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass...
virtual unsigned getID() const =0
Used for debugging and tracking things about MemoryAccesses.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
iplist< MemoryAccess > AccessList
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
static void ValueIsRAUWd(Value *Old, Value *New)
This is the generic walker interface for walkers of MemorySSA.
CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
static MemoryAccess * onlySingleValue(MemoryPhi *MP)
If all arguments of a MemoryPHI are defined by the same incoming argument, return that argument...
static bool instructionClobbersQuery(MemoryDef *MD, const MemoryLocation &UseLoc, const Instruction *UseInst, AliasAnalysis &AA)
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
An assembly annotator class to print Memory SSA information in comments.
void setAutoResetWalker(bool AutoReset)
Whether we call resetClobberWalker() after each time we actually walk to answer a clobber query...
The access neither references nor modifies the value stored in memory.
bool verify(const TargetRegisterInfo &TRI) const
Check that information hold by this instance make sense for the given TRI.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
initializer< Ty > init(const Ty &Val)
void verifyAnalysis() const override
verifyAnalysis() - This member can be implemented by a analysis pass to check state of analysis infor...
A set of analyses that are preserved following a run of a transformation pass.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
void optimizeUses()
Optimize uses to point to their actual clobbering definitions.
df_iterator< T > df_end(const T &G)
void verifyOrdering(Function &F) const
Verify that the order and existence of MemoryAccesses matches the order and existence of memory affec...
static bool isAtLeastOrStrongerThan(AtomicOrdering ao, AtomicOrdering other)
void push_front(pointer val)
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
A manager for alias analyses.
std::pair< iterator, bool > insert(const ValueT &V)
void addIncoming(MemoryAccess *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static bool isEqual(const MemoryLocOrCall &LHS, const MemoryLocOrCall &RHS)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
early cse Early CSE w MemorySSA
MemorySSA(Function &, AliasAnalysis *, DominatorTree *)
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
bool runOnFunction(Function &) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass...
MemorySSAWalker(MemorySSA *)
Represent the analysis usage information of a pass.
bool locallyDominates(const MemoryAccess *A, const MemoryAccess *B) const
Given two memory accesses in the same basic block, determine whether MemoryAccess A dominates MemoryA...
void splice(iterator where, iplist_impl &L2)
void setDefiningAccess(MemoryAccess *DMA)
#define LLVM_ATTRIBUTE_UNUSED
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
User * getUser() const
Returns the User that contains this Use.
FunctionPass class - This class is used to implement most global optimizations.
Interval::pred_iterator pred_end(Interval *I)
Memory true print Memory SSA Printer
bool hasValueHandle() const
Return true if there is a value handle associated with this value.
void verifyDefUses(Function &F) const
Verify the immediate use information, by walking all the memory accesses and verifying that...
void print(raw_ostream &OS) const override
MemoryAccess * getIncomingValue(unsigned I) const
Return incoming value number x.
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
void print(raw_ostream &OS, const Module *M=nullptr) const override
print - Print out the internal state of the pass.
BlockMass operator*(BlockMass L, BranchProbability R)
static const char LiveOnEntryStr[]
void verifyDomination(Function &F) const
Verify the domination properties of MemorySSA by checking that each definition dominates all of its u...
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
LLVMContext & getContext() const
All values hold a context through their type.
An intrusive list with ownership and callbacks specified/controlled by ilist_traits, only with API safe for polymorphic types.
MemoryAccess * getLiveOnEntryDef() const
MemorySSAWalker * getWalker()
MemoryAccess * createMemoryAccessInBB(Instruction *I, MemoryAccess *Definition, const BasicBlock *BB, InsertionPlace Point)
Create a MemoryAccess in MemorySSA at a specified point in a block, with a specified clobbering defin...
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
Representation for a specific memory location.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Iterator for intrusive lists based on ilist_node.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
bool isVolatile() const
Return true if this is a load from a volatile memory location.
An analysis that produces MemorySSA for a function.
LLVM_NODISCARD T pop_back_val()
std::vector< DomTreeNodeBase< NodeT > * >::const_iterator const_iterator
AtomicOrdering getOrdering() const
Returns the ordering effect of this fence.
void verify(const MemorySSA *MSSA) override
const BasicBlock & getEntryBlock() const
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void print(raw_ostream &OS) const override
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
df_iterator< T > df_begin(const T &G)
A range adaptor for a pair of iterators.
Target - Wrapper for Target specific information.
hash_code hash_combine(const Ts &...args)
Combine values into a single hash_code.
void verifyMemorySSA() const
Verify that MemorySSA is self consistent (IE definitions dominate all uses, uses appear in the right ...
Class that has the common methods + fields of memory uses/defs.
ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc)
getModRefInfo (for call sites) - Return information about whether a particular call site modifies or ...
void setPreservesAll()
Set by analyses that do not transform their input at all.
iterator_range< user_iterator > users()
bool isMustAlias(const MemoryLocation &LocA, const MemoryLocation &LocB)
A trivial helper function to check to see if the specified pointers are must-alias.
static void clear(coro::Shape &Shape)
Memory true print Memory SSA static false cl::opt< unsigned > MaxCheckLimit("memssa-check-limit", cl::Hidden, cl::init(100), cl::desc("The maximum number of stores/phis MemorySSA""will consider trying to walk past (default = 100)"))
void resetClobberWalker()
Drop the walker's persistent data structures.
void initializeMemorySSAPrinterLegacyPassPass(PassRegistry &)
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
iterator insert(iterator where, pointer New)
void emplace_back(ArgTypes &&...Args)
This file provides utility analysis objects describing memory locations.
ImmutableCallSite - establish a view to a call site for examination.
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
static void Query(const MachineInstr &MI, AliasAnalysis &AA, bool &Read, bool &Write, bool &Effects, bool &StackPointer)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
MemoryPhi * createMemoryPhi(BasicBlock *BB)
Create an empty MemoryPhi in MemorySSA for a given basic block.
AnalysisUsage & addRequiredTransitive()
MemorySSAPrinterLegacyPass()
iterator_range< df_iterator< T > > depth_first(const T &G)
MemoryAccess * getClobberingMemoryAccess(const Instruction *I)
Given a memory Mod/Ref/ModRef'ing instruction, calling this will give you the nearest dominating Memo...
Determine the iterated dominance frontier, given a set of defining blocks, and optionally, a set of live-in blocks.
Instruction * getMemoryInst() const
Get the instruction that this MemoryUse represents.
void spliceMemoryAccessAbove(MemoryDef *Where, MemoryUseOrDef *What)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static cl::opt< bool > VerifyMemorySSA("verify-memoryssa", cl::init(false), cl::Hidden, cl::desc("Verify MemorySSA in legacy printer pass."))
LLVMContext & getContext() const
Get the context in which this basic block lives.
LLVM Value Representation.
static MemoryLocOrCall getTombstoneKey()
succ_range successors(BasicBlock *BB)
upward_defs_iterator upward_defs_begin(const MemoryAccessPair &Pair)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
This class implements an extremely fast bulk output stream that can only output to a stream...
static unsigned getHashValue(const MemoryLocOrCall &MLOC)
void invalidateInfo(MemoryAccess *) override
Given a memory access, invalidate anything this walker knows about that access.
void print(raw_ostream &) const
A container for analyses that lazily runs them and caches their results.
Legacy analysis pass which computes a DominatorTree.
bool operator==(uint64_t V1, const APInt &V2)
DomTreeNodeBase< NodeT > * getNode(NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
Represents phi nodes for memory accesses.
auto find_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly...
iterator insertAfter(iterator where, pointer New)
const AccessList * getBlockAccesses(const BasicBlock *BB) const
Return the list of MemoryAccess's for a given basic block.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
MemoryUseOrDef * createMemoryAccessAfter(Instruction *I, MemoryAccess *Definition, MemoryAccess *InsertPt)
void releaseMemory() override
releaseMemory() - This member can be implemented by a pass if it wants to be able to release its memo...
static MemoryLocOrCall getEmptyKey()
A special type used by analysis passes to provide an address that identifies that particular analysis...
const BasicBlock * getParent() const
std::pair< MemoryAccess *, MemoryLocation > MemoryAccessPair
bool isLiveOnEntryDef(const MemoryAccess *MA) const
Return true if MA represents the live on entry value.
A wrapper class for inspecting calls to intrinsic functions.
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.