33 #define DEBUG_TYPE "gvn-hoist"
35 STATISTIC(NumHoisted,
"Number of instructions hoisted");
36 STATISTIC(NumRemoved,
"Number of instructions removed");
37 STATISTIC(NumLoadsHoisted,
"Number of loads hoisted");
38 STATISTIC(NumLoadsRemoved,
"Number of loads removed");
39 STATISTIC(NumStoresHoisted,
"Number of stores hoisted");
40 STATISTIC(NumStoresRemoved,
"Number of stores removed");
41 STATISTIC(NumCallsHoisted,
"Number of calls hoisted");
42 STATISTIC(NumCallsRemoved,
"Number of calls removed");
46 cl::desc(
"Max number of instructions to hoist "
47 "(default unlimited = -1)"));
50 cl::desc(
"Max number of basic blocks on the path between "
51 "hoisting locations (default = 4, unlimited = -1)"));
55 cl::desc(
"Hoist instructions from the beginning of the BB up to the "
56 "maximum specified depth (default = 100, unlimited = -1)"));
60 cl::desc(
"Maximum length of dependent chains to hoist "
61 "(default = 10, unlimited = -1)"));
86 ADFS = DFSNumber.lookup(A);
87 BDFS = DFSNumber.lookup(B);
89 ADFS = DFSNumber.lookup(BA);
90 BDFS = DFSNumber.lookup(BB);
102 enum :
unsigned { InvalidVN = ~2U };
106 VNtoInsns VNtoScalars;
113 VNtoScalars[{V, InvalidVN}].push_back(I);
116 const VNtoInsns &getVNTable()
const {
return VNtoScalars; }
128 VNtoLoads[{V, InvalidVN}].push_back(Load);
132 const VNtoInsns &getVNTable()
const {
return VNtoLoads; }
137 VNtoInsns VNtoStores;
151 const VNtoInsns &getVNTable()
const {
return VNtoStores; }
156 VNtoInsns VNtoCallsScalars;
157 VNtoInsns VNtoCallsLoads;
158 VNtoInsns VNtoCallsStores;
167 auto Entry = std::make_pair(V, InvalidVN);
170 VNtoCallsScalars[Entry].push_back(Call);
172 VNtoCallsLoads[Entry].push_back(Call);
174 VNtoCallsStores[Entry].push_back(Call);
177 const VNtoInsns &getScalarVNTable()
const {
return VNtoCallsScalars; }
179 const VNtoInsns &getLoadVNTable()
const {
return VNtoCallsLoads; }
181 const VNtoInsns &getStoreVNTable()
const {
return VNtoCallsStores; }
189 static const unsigned KnownIDs[] = {
204 : DT(DT), AA(AA), MD(MD), MSSA(MSSA),
217 DFSNumber[BB] = ++BBI;
219 for (
auto &Inst : *BB)
220 DFSNumber[&Inst] = ++
I;
230 auto HoistStat = hoistExpressions(F);
231 if (HoistStat.first + HoistStat.second == 0)
234 if (HoistStat.second > 0)
252 const bool HoistingGeps;
254 BBSideEffectsSet BBSideEffects;
261 auto It = BBSideEffects.find(BB);
262 if (It != BBSideEffects.end())
266 BBSideEffects[BB] =
true;
271 BBSideEffects[BB] =
true;
275 BBSideEffects[BB] =
false;
282 if (DT->dominates(Succ, A))
290 bool hoistingFromAllPaths(
const BasicBlock *HoistBB,
300 if (WorkList.empty())
304 if (WorkList.erase(BB)) {
316 if (successorDominate(BB, HoistBB))
329 unsigned I1DFS = DFSNumber.lookup(I1);
330 unsigned I2DFS = DFSNumber.lookup(I2);
332 return I1DFS < I2DFS;
345 bool ReachedNewPt =
false;
348 if (
const MemoryUse *MU = dyn_cast<MemoryUse>(&MA)) {
352 if (BB == OldBB && firstInBB(OldPt, Insn))
358 if (firstInBB(Insn, NewPt))
378 int &NBBsOnAllPaths) {
381 assert(DT->dominates(NewBB, OldBB) &&
"invalid path");
383 "def does not dominate new hoisting point");
397 if (NBBsOnAllPaths == 0)
405 if (hasMemoryUse(NewPt, Def, *I))
409 if (NBBsOnAllPaths != -1)
423 int &NBBsOnAllPaths) {
424 assert(DT->dominates(HoistPt, BB) &&
"Invalid path");
439 if (NBBsOnAllPaths == 0)
447 if (NBBsOnAllPaths != -1)
472 if (DT->properlyDominates(NewBB, DBB))
476 if (NewBB == DBB && !MSSA->isLiveOnEntryDef(D))
477 if (
auto *UD = dyn_cast<MemoryUseOrDef>(D))
478 if (firstInBB(NewPt, UD->getMemoryInst()))
484 if (hasEHOrLoadsOnPath(NewPt, dyn_cast<MemoryDef>(U), NBBsOnAllPaths))
486 }
else if (hasEHOnPath(NewBB, OldBB, NBBsOnAllPaths))
490 if (DT->properlyDominates(DBB, NewBB))
493 assert(MSSA->locallyDominates(D, U));
502 bool safeToHoistScalar(
const BasicBlock *HoistBB,
504 int &NBBsOnAllPaths) {
506 if (!hoistingFromAllPaths(HoistBB, WL))
510 if (hasEHOnPath(HoistBB, BB, NBBsOnAllPaths))
518 typedef std::pair<BasicBlock *, SmallVecInsn> HoistingPointInfo;
525 void partitionCandidates(SmallVecImplInsn &InstructionsToHoist,
526 HoistingPointList &HPL, InsKind K) {
528 if (InstructionsToHoist.size() > 2) {
529 SortByDFSIn Pred(DFSNumber);
530 std::sort(InstructionsToHoist.begin(), InstructionsToHoist.end(), Pred);
535 SmallVecImplInsn::iterator II = InstructionsToHoist.begin();
536 SmallVecImplInsn::iterator Start = II;
541 UD = MSSA->getMemoryAccess(HoistPt);
543 for (++II; II != InstructionsToHoist.end(); ++II) {
550 NewHoistBB = HoistBB;
551 NewHoistPt = firstInBB(Insn, HoistPt) ? Insn : HoistPt;
555 NewHoistBB = DT->findNearestCommonDominator(HoistBB, BB);
556 if (NewHoistBB == BB)
558 else if (NewHoistBB == HoistBB)
559 NewHoistPt = HoistPt;
569 if (safeToHoistScalar(NewHoistBB, WL, NumBBsOnAllPaths)) {
571 HoistPt = NewHoistPt;
572 HoistBB = NewHoistBB;
582 if ((HoistBB == NewHoistBB || BB == NewHoistBB ||
583 hoistingFromAllPaths(NewHoistBB, WL)) &&
586 safeToHoistLdSt(NewHoistPt, HoistPt, UD, K, NumBBsOnAllPaths) &&
587 safeToHoistLdSt(NewHoistPt, Insn, MSSA->getMemoryAccess(Insn),
588 K, NumBBsOnAllPaths)) {
590 HoistPt = NewHoistPt;
591 HoistBB = NewHoistBB;
598 if (std::distance(Start, II) > 1)
599 HPL.push_back({HoistBB, SmallVecInsn(Start, II)});
604 UD = MSSA->getMemoryAccess(*Start);
611 if (std::distance(Start, II) > 1)
612 HPL.push_back({HoistBB, SmallVecInsn(Start, II)});
616 void computeInsertionPoints(
const VNtoInsns &Map, HoistingPointList &HPL,
618 for (
const auto &Entry : Map) {
622 const SmallVecInsn &V = Entry.second;
627 SmallVecInsn InstructionsToHoist;
629 if (!hasEH(I->getParent()))
630 InstructionsToHoist.push_back(I);
632 if (!InstructionsToHoist.empty())
633 partitionCandidates(InstructionsToHoist, HPL, K);
644 if (
const auto *Inst = dyn_cast<Instruction>(&
Op))
645 if (!DT->dominates(Inst->getParent(), HoistPt))
655 if (
const auto *Inst = dyn_cast<Instruction>(&
Op))
656 if (!DT->dominates(Inst->getParent(), HoistPt)) {
658 dyn_cast<GetElementPtrInst>(Inst)) {
659 if (!allGepOperandsAvailable(GepOp, HoistPt))
673 const SmallVecInsn &InstructionsToHoist,
675 assert(allGepOperandsAvailable(Gep, HoistPt) &&
676 "GEP operands not available");
683 if (DT->dominates(
Op->getParent(), HoistPt))
689 makeGepsAvailable(ClonedGep, HoistPt, InstructionsToHoist, GepOp);
701 for (
const Instruction *OtherInst : InstructionsToHoist) {
703 if (
auto *OtherLd = dyn_cast<LoadInst>(OtherInst))
704 OtherGep = cast<GetElementPtrInst>(OtherLd->getPointerOperand());
706 OtherGep = cast<GetElementPtrInst>(
719 const SmallVecInsn &InstructionsToHoist)
const {
723 if (
auto *Ld = dyn_cast<LoadInst>(Repl)) {
725 }
else if (
auto *St = dyn_cast<StoreInst>(Repl)) {
730 if (isa<GetElementPtrInst>(Val)) {
732 if (!allGepOperandsAvailable(Val, HoistPt))
734 }
else if (!DT->dominates(Val->getParent(), HoistPt))
740 if (!Gep || !allGepOperandsAvailable(Gep, HoistPt))
743 makeGepsAvailable(Repl, HoistPt, InstructionsToHoist, Gep);
745 if (Val && isa<GetElementPtrInst>(Val))
746 makeGepsAvailable(Repl, HoistPt, InstructionsToHoist, Val);
751 std::pair<unsigned, unsigned>
hoist(HoistingPointList &HPL) {
752 unsigned NI = 0, NL = 0, NS = 0,
NC = 0, NR = 0;
753 for (
const HoistingPointInfo &HP : HPL) {
757 const SmallVecInsn &InstructionsToHoist = HP.second;
764 if (!Repl || firstInBB(I, Repl))
769 bool MoveAccess =
true;
772 assert(allOperandsAvailable(Repl, HoistPt) &&
773 "instruction depends on operands that are not available");
778 Repl = InstructionsToHoist.front();
783 if (!allOperandsAvailable(Repl, HoistPt)) {
791 if (!makeGepOperandsAvailable(Repl, HoistPt, InstructionsToHoist))
797 MD->removeInstruction(Repl);
800 DFSNumber[Repl] = DFSNumber[Last]++;
807 dyn_cast_or_null<MemoryUseOrDef>(NewMemAcc)) {
814 MSSA->removeMemoryAccess(OldMemAcc);
818 if (isa<LoadInst>(Repl))
820 else if (isa<StoreInst>(Repl))
822 else if (isa<CallInst>(Repl))
831 if (
auto *ReplacementLoad = dyn_cast<LoadInst>(Repl)) {
832 ReplacementLoad->setAlignment(
833 std::min(ReplacementLoad->getAlignment(),
836 }
else if (
auto *ReplacementStore = dyn_cast<StoreInst>(Repl)) {
837 ReplacementStore->setAlignment(
838 std::min(ReplacementStore->getAlignment(),
841 }
else if (
auto *ReplacementAlloca = dyn_cast<AllocaInst>(Repl)) {
842 ReplacementAlloca->setAlignment(
843 std::max(ReplacementAlloca->getAlignment(),
845 }
else if (isa<CallInst>(Repl)) {
853 MSSA->removeMemoryAccess(OldMA);
857 combineKnownMetadata(Repl, I);
860 MD->removeInstruction(I);
868 if (
MemoryPhi *Phi = dyn_cast<MemoryPhi>(U))
871 for (
auto *Phi : UsePhis) {
872 auto In = Phi->incoming_values();
873 if (
all_of(
In, [&](
Use &U) {
return U == NewMemAcc; })) {
874 Phi->replaceAllUsesWith(NewMemAcc);
875 MSSA->removeMemoryAccess(Phi);
881 NumHoisted += NL + NS +
NC + NI;
883 NumLoadsHoisted += NL;
884 NumStoresHoisted += NS;
885 NumCallsHoisted +=
NC;
886 return {NI, NL +
NC + NS};
891 std::pair<unsigned, unsigned> hoistExpressions(
Function &F) {
897 int InstructionNb = 0;
905 if (isa<TerminatorInst>(&I1))
908 if (
auto *Load = dyn_cast<LoadInst>(&I1))
910 else if (
auto *Store = dyn_cast<StoreInst>(&I1))
911 SI.insert(Store, VN);
912 else if (
auto *Call = dyn_cast<CallInst>(&I1)) {
913 if (
auto *Intr = dyn_cast<IntrinsicInst>(Call)) {
914 if (isa<DbgInfoIntrinsic>(Intr) ||
915 Intr->getIntrinsicID() == Intrinsic::assume)
925 }
else if (HoistingGeps || !isa<GetElementPtrInst>(&I1))
934 HoistingPointList HPL;
939 computeInsertionPoints(CI.getLoadVNTable(), HPL,
InsKind::Load);
940 computeInsertionPoints(CI.getStoreVNTable(), HPL,
InsKind::Store);
953 bool runOnFunction(
Function &F)
override {
956 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
957 auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
958 auto &MD = getAnalysis<MemoryDependenceWrapperPass>().getMemDep();
959 auto &MSSA = getAnalysis<MemorySSAWrapperPass>().getMSSA();
961 GVNHoist
G(&DT, &AA, &MD, &MSSA);
981 GVNHoist
G(&DT, &AA, &MD, &MSSA);
993 "Early GVN Hoisting of Expressions",
false,
false)
void setDomTree(DominatorTree *D)
Value * getValueOperand()
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
static cl::opt< int > MaxDepthInBB("gvn-hoist-max-depth", cl::Hidden, cl::init(100), cl::desc("Hoist instructions from the beginning of the BB up to the ""maximum specified depth (default = 100, unlimited = -1)"))
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Provides a lazy, caching interface for making common memory aliasing information queries, backed by LLVM's alias analysis passes.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
gvn Early GVN Hoisting of false
STATISTIC(NumFunctions,"Total number of functions")
void dropUnknownNonDebugMetadata(ArrayRef< unsigned > KnownIDs)
Drop all unknown metadata except for debug locations.
bool defClobbersUseOrDef(MemoryDef *MD, const MemoryUseOrDef *MU, AliasAnalysis &AA)
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Run the pass over the function.
unsigned getNumOperands() const
This class represents a function call, abstracting a target machine's calling convention.
bool mayHaveSideEffects() const
Return true if the instruction may have side effects.
Represents a read-write access to memory, whether it is a must-alias, or a may-alias.
void setAliasAnalysis(AliasAnalysis *A)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
bool isEHPad() const
Return true if this basic block is an exception handling block.
Analysis pass which computes a DominatorTree.
FunctionPass * createGVNHoistPass()
An instruction for reading from memory.
gvn Early GVN Hoisting of Expressions
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Represents read-only accesses to memory.
bool onlyReadsMemory() const
Determine if the call does not access or only reads memory.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Legacy analysis pass which computes MemorySSA.
static Value * getPointerOperand(Instruction &Inst)
void clear()
Remove all entries from the ValueTable.
A Use represents the edge between a Value definition and its users.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Encapsulates MemorySSA, including all data associated with memory accesses.
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches, switches, etc.
MemoryAccess * getDefiningAccess() const
Get the access that produces the memory state used by this Use.
An analysis that produces MemoryDependenceResults for a function.
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following: ...
BasicBlock * getBlock() const
static unsigned getAlignment(GlobalVariable *GV)
The core GVN pass object.
void andIRFlags(const Value *V)
Logical 'and' of any supported wrapping, exact, and fast-math flags of V and this instruction...
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
An instruction for storing to memory.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
idf_iterator< T > idf_begin(const T &G)
idf_iterator< T > idf_end(const T &G)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
initializer< Ty > init(const Ty &Val)
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
A set of analyses that are preserved following a run of a transformation pass.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
LLVM Basic Block Representation.
This class holds the mapping between values and value numbers.
df_iterator< T > df_end(const T &G)
This file provides the interface for LLVM's Global Value Numbering pass which eliminates fully redund...
A manager for alias analyses.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Represent the analysis usage information of a pass.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
bool doesNotAccessMemory() const
Determine if the call does not access memory.
FunctionPass class - This class is used to implement most global optimizations.
Value * getOperand(unsigned i) const
Value * getPointerOperand()
static cl::opt< int > MaxHoistedThreshold("gvn-max-hoisted", cl::Hidden, cl::init(-1), cl::desc("Max number of instructions to hoist ""(default unlimited = -1)"))
INITIALIZE_PASS_BEGIN(GVNHoistLegacyPass,"gvn-hoist","Early GVN Hoisting of Expressions", false, false) INITIALIZE_PASS_END(GVNHoistLegacyPass
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
static cl::opt< int > MaxChainLength("gvn-hoist-max-chain-length", cl::Hidden, cl::init(10), cl::desc("Maximum length of dependent chains to hoist ""(default = 10, unlimited = -1)"))
A wrapper analysis pass for the legacy pass manager that exposes a MemoryDepnedenceResults instance...
An intrusive list with ownership and callbacks specified/controlled by ilist_traits, only with API safe for polymorphic types.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
An analysis that produces MemorySSA for a function.
static cl::opt< int > MaxNumberOfBBSInPath("gvn-hoist-max-bbs", cl::Hidden, cl::init(4), cl::desc("Max number of basic blocks on the path between ""hoisting locations (default = 4, unlimited = -1)"))
const BasicBlock & getEntryBlock() const
bool isConvergent() const
Determine if the call is convergent.
gvn hoist
When an instruction is found to only use loop invariant operands that is safe to hoist, this instruction is called to do the dirty work.
df_iterator< T > df_begin(const T &G)
Class that has the common methods + fields of memory uses/defs.
iterator_range< user_iterator > users()
bool mayThrow() const
Return true if this instruction may throw an exception.
void initializeGVNHoistLegacyPassPass(PassRegistry &)
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
void setMemDep(MemoryDependenceResults *M)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
void preserve()
Mark an analysis as preserved.
uint32_t lookupOrAdd(Value *V)
lookup_or_add - Returns the value number for the specified value, assigning it a new number if it did...
iterator_range< df_iterator< T > > depth_first(const T &G)
Instruction * getMemoryInst() const
Get the instruction that this MemoryUse represents.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
A container for analyses that lazily runs them and caches their results.
Legacy analysis pass which computes a DominatorTree.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object...
Represents phi nodes for memory accesses.
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
Value * getPointerOperand()
void combineMetadata(Instruction *K, const Instruction *J, ArrayRef< unsigned > KnownIDs)
Combine the metadata of two instructions so that K can replace J.
const BasicBlock * getParent() const