30 cl::desc(
"Add no-alias annotation for instructions that "
31 "are disambiguated by memchecks"));
36 : VersionedLoop(L), NonVersionedLoop(nullptr), LAI(LAI), LI(LI), DT(DT),
48 AliasChecks = std::move(Checks);
52 Preds = std::move(Check);
59 Value *SCEVRuntimeCheck;
60 Value *RuntimeCheck =
nullptr;
64 std::tie(FirstCheckInst, MemRuntimeCheck) =
71 Exp.expandCodeForPredicate(&Pred, RuntimeCheckBB->
getTerminator());
75 if (CI && CI->isZero())
76 SCEVRuntimeCheck =
nullptr;
78 if (MemRuntimeCheck && SCEVRuntimeCheck) {
80 SCEVRuntimeCheck,
"lver.safe");
81 if (
auto *
I = dyn_cast<Instruction>(RuntimeCheck))
84 RuntimeCheck = MemRuntimeCheck ? MemRuntimeCheck : SCEVRuntimeCheck;
86 assert(RuntimeCheck &&
"called even though we don't need "
87 "any runtime checks");
106 ".lver.orig", LI, DT, NonVersionedLoopBlocks);
121 addPHINodes(DefsUsedOutside);
124 void LoopVersioning::addPHINodes(
127 assert(PHIBlock &&
"No single successor to loop exit block");
132 for (
auto *Inst : DefsUsedOutside) {
144 if (!VersionedLoop->
contains(cast<Instruction>(
User)->getParent()))
153 "Exit block should only have on predecessor");
157 auto Mapped = VMap.
find(ClonedValue);
158 if (Mapped != VMap.
end())
159 ClonedValue = Mapped->
second;
187 for (
unsigned PtrIdx : Group.Members)
195 GroupToNonAliasingScopes;
197 for (
const auto &
Check : AliasChecks)
203 for (
auto Pair : GroupToNonAliasingScopes)
204 GroupToNonAliasingScopeList[Pair.first] =
MDNode::get(Context, Pair.second);
226 const Value *
Ptr = isa<LoadInst>(OrigInst)
227 ? cast<LoadInst>(OrigInst)->getPointerOperand()
231 auto Group = PtrToGroup.find(Ptr);
232 if (Group != PtrToGroup.end()) {
237 MDNode::get(Context, GroupToScope[Group->second])));
240 auto NonAliasingScopeList = GroupToNonAliasingScopeList.find(Group->second);
241 if (NonAliasingScopeList != GroupToNonAliasingScopeList.end())
246 NonAliasingScopeList->second));
260 bool runOnFunction(
Function &
F)
override {
261 auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
262 auto *LAA = &getAnalysis<LoopAccessLegacyAnalysis>();
263 auto *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
264 auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
271 for (
Loop *TopLevelLoop : *LI)
278 bool Changed =
false;
279 for (
Loop *
L : Worklist) {
285 LVer.annotateLoopWithNoAlias();
306 #define LVER_OPTION "loop-versioning"
307 #define DEBUG_TYPE LVER_OPTION
321 return new LoopVersioningPass();
static bool Check(DecodeStatus &Out, DecodeStatus In)
TrackingVH< Value > PointerValue
Holds the pointer value that we need to check.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void push_back(const T &Elt)
SmallVector< CheckingPtrGroup, 2 > CheckingGroups
Holds a partitioning of pointers into "check groups".
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the specified block at the specified instruction - everything before SplitPt stays in Old and e...
const SmallVectorImpl< Instruction * > & getMemoryInstructions() const
The vector of memory access instructions.
MDNode * createAnonymousAliasScope(MDNode *Domain, StringRef Name=StringRef())
Return metadata appropriate for an alias scope root node.
unsigned getNumOperands() const
The main scalar evolution driver.
std::pair< Instruction *, Instruction * > addRuntimeChecks(Instruction *Loc) const
Add code that checks at runtime if the accessed arrays overlap.
const Instruction & front() const
BlockT * getExitBlock() const
If getExitBlocks would return exactly one block, return that block.
BlockT * getHeader() const
void remapInstructionsInBlocks(const SmallVectorImpl< BasicBlock * > &Blocks, ValueToValueMapTy &VMap)
Remaps instructions in Blocks using the mapping in VMap.
static cl::opt< bool > AnnotateNoAlias("loop-version-annotate-no-alias", cl::init(true), cl::Hidden, cl::desc("Add no-alias annotation for instructions that ""are disambiguated by memchecks"))
iterator begin()
Instruction iterator methods.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
static Value * getPointerOperand(Instruction &Inst)
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr it the function does no...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
const PredicatedScalarEvolution & getPSE() const
Used to add runtime SCEV checks.
void setName(const Twine &Name)
Change the name of the value.
bool isLoopSimplifyForm() const
Return true if the Loop is in the form that the LoopSimplify form transforms loops to...
static const char LVer_name[]
const RuntimePointerChecking * getRuntimePointerChecking() const
Function Alias Analysis false
iterator find(const KeyT &Val)
void versionLoop()
Performs the CFG manipulation part of versioning the loop including the DominatorTree and LoopInfo up...
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
initializer< Ty > init(const Ty &Val)
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
LLVM Basic Block Representation.
This is an important class for using LLVM in a threaded context.
This analysis provides dependence information for the memory accesses of a loop.
const PointerInfo & getPointerInfo(unsigned PtrIdx) const
Return PointerInfo for pointer at index PtrIdx.
APInt Or(const APInt &LHS, const APInt &RHS)
Bitwise OR function for APInt.
Represent the analysis usage information of a pass.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
const SmallVector< PointerCheck, 4 > & getChecks() const
Returns the checks that generateChecks created.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
void annotateLoopWithNoAlias()
Annotate memory instructions in the versioned loop with no-alias metadata based on the memchecks issu...
FunctionPass class - This class is used to implement most global optimizations.
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
void setAliasChecks(SmallVector< RuntimePointerChecking::PointerCheck, 4 > Checks)
Sets the runtime alias checks for versioning the loop.
void setSCEVChecks(SCEVUnionPredicate Check)
Sets the runtime SCEV checks for versioning the loop.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
MDNode * createAnonymousAliasScopeDomain(StringRef Name=StringRef())
Return metadata appropriate for an alias scope domain node.
FunctionPass * createLoopVersioningPass()
This is the shared class of boolean and integer constants.
bool isAlwaysTrue() const override
Implementation of the SCEVPredicate interface.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
const MemoryDepChecker & getDepChecker() const
the Memory Dependence Checker which can determine the loop-independent and loop-carried dependences b...
Drive the analysis of memory accesses in the loop.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
void annotateInstWithNoAlias(Instruction *VersionedInst, const Instruction *OrigInst)
Add the noalias annotations to VersionedInst.
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), Instruction *InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
iterator_range< user_iterator > users()
This class uses information about analyze scalars to rewrite expressions in canonical form...
Loop * cloneLoopWithPreheader(BasicBlock *Before, BasicBlock *LoopDomBB, Loop *OrigLoop, ValueToValueMapTy &VMap, const Twine &NameSuffix, LoopInfo *LI, DominatorTree *DT, SmallVectorImpl< BasicBlock * > &Blocks)
Clones a loop OrigLoop.
Holds information about the memory runtime legality checks to verify that a group of pointers do not ...
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
const SCEVUnionPredicate & getUnionPredicate() const
void initializeLoopVersioningPassPass(PassRegistry &)
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void prepareNoAliasMetadata()
Set up the aliasing scopes based on the memchecks.
Represents a single loop in the control flow graph.
static MDNode * concatenate(MDNode *A, MDNode *B)
Methods for metadata merging.
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
iterator_range< df_iterator< T > > depth_first(const T &G)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This class represents a composition of other SCEV predicates, and is the class that most clients will...
LLVM Value Representation.
The legacy pass manager's analysis pass to compute loop information.
Legacy analysis pass which computes a DominatorTree.
LoopVersioning(const LoopAccessInfo &LAI, Loop *L, LoopInfo *LI, DominatorTree *DT, ScalarEvolution *SE, bool UseLAIChecks=true)
Expects LoopAccessInfo, Loop, LoopInfo, DominatorTree as input.