36 #define DEBUG_TYPE "memcpyopt"
38 STATISTIC(NumMemCpyInstr,
"Number of memcpy instructions deleted");
39 STATISTIC(NumMemSetInfer,
"Number of memsets inferred");
40 STATISTIC(NumMoveToCpy,
"Number of memmoves converted to memcpy");
41 STATISTIC(NumCpyToSet,
"Number of memcpys converted to memset");
44 bool &VariableIdxFound,
48 for (
unsigned i = 1; i != Idx; ++i, ++GTI)
53 for (
unsigned i = Idx, e = GEP->
getNumOperands(); i != e; ++i, ++GTI) {
56 return VariableIdxFound =
true;
57 if (OpC->
isZero())
continue;
60 if (
StructType *STy = dyn_cast<StructType>(*GTI)) {
91 bool VariableIdxFound =
false;
97 return !VariableIdxFound;
102 return !VariableIdxFound;
110 if (!GEP1 || !GEP2 || GEP1->
getOperand(0) != GEP2->getOperand(0))
115 for (; Idx != GEP1->
getNumOperands() && Idx != GEP2->getNumOperands(); ++Idx)
116 if (GEP1->
getOperand(Idx) != GEP2->getOperand(Idx))
121 if (VariableIdxFound)
return false;
123 Offset = Offset2-Offset1;
154 bool isProfitableToUseMemset(
const DataLayout &
DL)
const;
158 bool MemsetRange::isProfitableToUseMemset(
const DataLayout &DL)
const {
160 if (TheStores.size() >= 4 || End-Start >= 16)
return true;
163 if (TheStores.size() < 2)
return false;
167 for (
unsigned i = 0, e = TheStores.size(); i != e; ++i)
168 if (!isa<StoreInst>(TheStores[i]))
173 if (TheStores.size() == 2)
return false;
185 unsigned Bytes =
unsigned(End-Start);
189 unsigned NumPointerStores = Bytes / MaxIntSize;
192 unsigned NumByteStores = Bytes - NumPointerStores * MaxIntSize;
197 return TheStores.size() > NumPointerStores+NumByteStores;
205 std::list<MemsetRange>
Ranges;
206 typedef std::list<MemsetRange>::iterator range_iterator;
209 MemsetRanges(
const DataLayout &DL) : DL(DL) {}
211 typedef std::list<MemsetRange>::const_iterator const_iterator;
212 const_iterator
begin()
const {
return Ranges.begin(); }
213 const_iterator
end()
const {
return Ranges.end(); }
214 bool empty()
const {
return Ranges.empty(); }
216 void addInst(int64_t OffsetFromFirst,
Instruction *Inst) {
218 addStore(OffsetFromFirst,
SI);
220 addMemSet(OffsetFromFirst, cast<MemSetInst>(Inst));
223 void addStore(int64_t OffsetFromFirst,
StoreInst *
SI) {
226 addRange(OffsetFromFirst, StoreSize,
230 void addMemSet(int64_t OffsetFromFirst,
MemSetInst *MSI) {
231 int64_t Size = cast<ConstantInt>(MSI->
getLength())->getZExtValue();
253 int64_t End = Start+Size;
256 while (I != E && Start > I->End)
262 if (I == E || End < I->Start) {
263 MemsetRange &R = *
Ranges.insert(I, MemsetRange());
267 R.Alignment = Alignment;
268 R.TheStores.push_back(Inst);
273 I->TheStores.push_back(Inst);
277 if (I->Start <= Start && I->End >= End)
286 if (Start < I->Start) {
289 I->Alignment = Alignment;
297 range_iterator NextI =
I;
298 while (++NextI != E && End >= NextI->Start) {
300 I->TheStores.append(NextI->TheStores.begin(), NextI->TheStores.end());
301 if (NextI->End > I->End)
325 bool runOnFunction(
Function &
F)
override;
346 uint64_t cpyLen,
unsigned cpyAlign,
CallInst *
C);
350 bool processByValArgument(
CallSite CS,
unsigned ArgNo);
379 const DataLayout &DL = StartInst->getModule()->getDataLayout();
388 for (++BI; !isa<TerminatorInst>(BI); ++BI) {
389 if (!isa<StoreInst>(BI) && !isa<MemSetInst>(BI)) {
393 if (BI->mayWriteToMemory() || BI->mayReadFromMemory())
398 if (
StoreInst *NextStore = dyn_cast<StoreInst>(BI)) {
400 if (!NextStore->isSimple())
break;
412 Ranges.addStore(Offset, NextStore);
425 Ranges.addMemSet(Offset, MSI);
437 Ranges.addInst(0, StartInst);
447 for (MemsetRanges::const_iterator I = Ranges.begin(), E = Ranges.end();
449 const MemsetRange &Range = *
I;
451 if (Range.TheStores.size() == 1)
continue;
454 if (!Range.isProfitableToUseMemset(DL))
459 StartPtr = Range.StartPtr;
462 unsigned Alignment = Range.Alignment;
463 if (Alignment == 0) {
465 cast<PointerType>(StartPtr->getType())->getElementType();
470 Builder.
CreateMemSet(StartPtr, ByteVal, Range.End-Range.Start, Alignment);
473 for (
unsigned i = 0, e = Range.TheStores.size(); i != e; ++i)
474 dbgs() << *Range.TheStores[i] <<
'\n';
475 dbgs() <<
"With: " << *AMemSet <<
'\n');
477 if (!Range.TheStores.empty())
478 AMemSet->
setDebugLoc(Range.TheStores[0]->getDebugLoc());
482 SI = Range.TheStores.begin(),
483 SE = Range.TheStores.end(); SI != SE; ++SI) {
484 MD->removeInstruction(*SI);
485 (*SI)->eraseFromParent();
502 if (LI->isSimple() && LI->hasOneUse() &&
515 E = C; I != E; --
I) {
527 unsigned loadAlign = LI->getAlignment();
531 bool changed = performCallSlotOptzn(
535 std::min(storeAlign, loadAlign), C);
537 MD->removeInstruction(SI);
539 MD->removeInstruction(LI);
540 LI->eraseFromParent();
580 bool MemCpyOpt::performCallSlotOptzn(
Instruction *cpy,
582 uint64_t cpyLen,
unsigned cpyAlign,
615 if (cpyLen < srcSize)
621 if (
AllocaInst *
A = dyn_cast<AllocaInst>(cpyDest)) {
630 if (destSize < srcSize)
632 }
else if (
Argument *
A = dyn_cast<Argument>(cpyDest)) {
633 if (
A->getDereferenceableBytes() < srcSize) {
636 if (!
A->hasStructRetAttr())
639 Type *StructTy = cast<PointerType>(
A->getType())->getElementType();
648 if (destSize < srcSize)
659 bool isDestSufficientlyAligned = srcAlign <= cpyAlign;
662 if (!isDestSufficientlyAligned && !isa<AllocaInst>(cpyDest))
671 while (!srcUseList.empty()) {
672 User *U = srcUseList.pop_back_val();
674 if (isa<BitCastInst>(U) || isa<AddrSpaceCastInst>(U)) {
676 srcUseList.push_back(UU);
680 if (!
G->hasAllZeroIndices())
684 srcUseList.push_back(UU);
688 if (
IT->getIntrinsicID() == Intrinsic::lifetime_start ||
689 IT->getIntrinsicID() == Intrinsic::lifetime_end)
692 if (U != C && U != cpy)
698 for (
unsigned i = 0, e =
CS.arg_size(); i != e; ++i)
699 if (
CS.getArgument(i) == cpySrc && !
CS.doesNotCapture(i))
704 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
705 if (
Instruction *cpyDestInst = dyn_cast<Instruction>(cpyDest))
722 bool changedArgument =
false;
723 for (
unsigned i = 0; i <
CS.arg_size(); ++i)
724 if (
CS.getArgument(i)->stripPointerCasts() == cpySrc) {
728 changedArgument =
true;
729 if (
CS.getArgument(i)->getType() == Dest->
getType())
730 CS.setArgument(i, Dest);
733 CS.getArgument(i)->getType(), Dest->
getName(), C));
736 if (!changedArgument)
740 if (!isDestSufficientlyAligned) {
741 assert(isa<AllocaInst>(cpyDest) &&
"Can only increase alloca alignment!");
742 cast<AllocaInst>(cpyDest)->setAlignment(srcAlign);
747 MD->removeInstruction(C);
752 unsigned KnownIDs[] = {
760 MD->removeInstruction(cpy);
788 if (!MDepLen || !MLen || MDepLen->
getZExtValue() < MLen->getZExtValue())
807 if (!SourceDep.isClobber() || SourceDep.getInst() != MDep)
813 bool UseMemMove =
false;
836 MD->removeInstruction(M);
856 bool MemCpyOpt::processMemSetMemCpyDependence(
MemCpyInst *MemCpy,
865 if (DstDepInfo.
getInst() != MemSet)
877 const unsigned DestAlign =
880 if (
ConstantInt *SrcSizeC = dyn_cast<ConstantInt>(SrcSize))
881 Align =
MinAlign(SrcSizeC->getZExtValue(), DestAlign);
889 SrcSize = Builder.CreateZExt(SrcSize, DestSize->
getType());
891 DestSize = Builder.CreateZExt(DestSize, SrcSize->
getType());
895 Builder.CreateSelect(Builder.CreateICmpULE(DestSize, SrcSize),
897 Builder.CreateSub(DestSize, SrcSize));
898 Builder.CreateMemSet(Builder.CreateGEP(Dest, SrcSize), MemSet->
getOperand(1),
901 MD->removeInstruction(MemSet);
920 bool MemCpyOpt::performMemCpyToMemSetOptzn(
MemCpyInst *MemCpy,
930 if (!MemSetSize || CopySize->
getZExtValue() > MemSetSize->getZExtValue())
944 bool MemCpyOpt::processMemCpy(
MemCpyInst *M) {
950 MD->removeInstruction(M);
957 if (GV->isConstant() && GV->hasDefinitiveInitializer())
962 MD->removeInstruction(M);
974 if (processMemSetMemCpyDependence(M, MDep))
979 if (!CopySize)
return false;
993 MD->removeInstruction(M);
1001 MemDepResult SrcDepInfo = MD->getPointerDependencyFrom(SrcLoc,
true,
1006 return processMemCpyMemCpyDependence(M, MDep);
1007 }
else if (SrcDepInfo.
isDef()) {
1009 bool hasUndefContents =
false;
1011 if (isa<AllocaInst>(I)) {
1012 hasUndefContents =
true;
1013 }
else if (
IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
1014 if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1015 if (
ConstantInt *LTSize = dyn_cast<ConstantInt>(II->getArgOperand(0)))
1016 if (LTSize->getZExtValue() >= CopySize->
getZExtValue())
1017 hasUndefContents =
true;
1020 if (hasUndefContents) {
1021 MD->removeInstruction(M);
1030 if (performMemCpyToMemSetOptzn(M, MDep)) {
1031 MD->removeInstruction(M);
1045 if (!TLI->has(LibFunc::memmove))
1053 DEBUG(
dbgs() <<
"MemCpyOpt: Optimizing memmove -> memcpy: " << *M <<
"\n");
1065 MD->removeInstruction(M);
1072 bool MemCpyOpt::processByValArgument(
CallSite CS,
unsigned ArgNo) {
1076 Type *ByValTy = cast<PointerType>(ByValArg->
getType())->getElementType();
1100 if (ByValAlign == 0)
return false;
1105 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
1107 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1133 DEBUG(
dbgs() <<
"MemCpyOpt: Forwarding memcpy to byval:\n"
1134 <<
" " << *MDep <<
"\n"
1144 bool MemCpyOpt::iterateOnFunction(
Function &
F) {
1145 bool MadeChange =
false;
1153 bool RepeatInstruction =
false;
1155 if (
StoreInst *SI = dyn_cast<StoreInst>(I))
1156 MadeChange |= processStore(SI, BI);
1157 else if (
MemSetInst *M = dyn_cast<MemSetInst>(I))
1158 RepeatInstruction = processMemSet(M, BI);
1159 else if (
MemCpyInst *M = dyn_cast<MemCpyInst>(I))
1160 RepeatInstruction = processMemCpy(M);
1161 else if (
MemMoveInst *M = dyn_cast<MemMoveInst>(I))
1162 RepeatInstruction = processMemMove(M);
1164 for (
unsigned i = 0, e = CS.
arg_size(); i != e; ++i)
1166 MadeChange |= processByValArgument(CS, i);
1170 if (RepeatInstruction) {
1171 if (BI != BB->begin()) --BI;
1183 bool MemCpyOpt::runOnFunction(
Function &F) {
1184 if (skipOptnoneFunction(F))
1187 bool MadeChange =
false;
1188 MD = &getAnalysis<MemoryDependenceAnalysis>();
1189 TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
1194 if (!TLI->has(LibFunc::memset) || !TLI->has(LibFunc::memcpy))
1198 if (!iterateOnFunction(F))
unsigned getAlignment() const
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
A parsed version of the target data layout string in and methods for querying it. ...
Type * getIndexedType() const
const_iterator end(StringRef path)
Get end iterator over path.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
unsigned getOrEnforceKnownAlignment(Value *V, unsigned PrefAlign, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
getOrEnforceKnownAlignment - If the specified pointer has an alignment that we can determine...
LLVM Argument representation.
uint64_t getZExtValue() const
Get zero extended value.
bool isDef() const
isDef - Return true if this MemDepResult represents a query that is an instruction definition depende...
STATISTIC(NumFunctions,"Total number of functions")
Value * isBytewiseValue(Value *V)
isBytewiseValue - If the specified value can be set by repeating the same byte in memory...
ValTy * getArgument(unsigned ArgNo) const
A Module instance is used to store all the information related to an LLVM module. ...
ModRefResult callCapturesBefore(const Instruction *I, const MemoryLocation &MemLoc, DominatorTree *DT)
callCapturesBefore - Return information about whether a particular call site modifies or reads the sp...
InstrTy * getInstruction() const
Value * getValue() const
get* - Return the arguments to the instruction.
unsigned getNumOperands() const
CallInst - This class represents a function call, abstracting a target machine's calling convention...
An immutable pass that tracks lazily created AssumptionCache objects.
BBTy * getParent() const
Get the basic block containing the call site.
A cache of .assume calls within a function.
static bool IsPointerOffset(Value *Ptr1, Value *Ptr2, int64_t &Offset, const DataLayout &DL)
IsPointerOffset - Return true if Ptr1 is provably equal to Ptr2 plus a constant offset, and return that constant offset.
const_iterator begin(StringRef path)
Get begin iterator over path.
MemSetInst - This class wraps the llvm.memset intrinsic.
const Function * getParent() const
Return the enclosing method, or null if none.
FunTy * getCaller() const
getCaller - Return the caller function for this call site
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, unsigned Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
LoadInst - an instruction for reading from memory.
bool isClobber() const
isClobber - Return true if this MemDepResult represents a query that is an instruction clobber depend...
static Constant * getNullValue(Type *Ty)
StringRef getName() const
Return a constant reference to the value's name.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
static int64_t GetOffsetFromIndex(const GEPOperator *GEP, unsigned Idx, bool &VariableIdxFound, const DataLayout &DL)
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::ZeroOrMore, cl::values(clEnumValN(DefaultIT,"arm-default-it","Generate IT block based on arch"), clEnumValN(RestrictedIT,"arm-restrict-it","Disallow deprecated IT based on ARMv8"), clEnumValN(NoRestrictedIT,"arm-no-restrict-it","Allow IT blocks based on ARMv7"), clEnumValEnd))
const APInt & getValue() const
Return the constant as an APInt value reference.
MemMoveInst - This class wraps the llvm.memmove intrinsic.
StructType - Class to represent struct types.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
bool isSized(SmallPtrSetImpl< const Type * > *Visited=nullptr) const
isSized - Return true if it makes sense to take the size of this type.
MemoryDependenceAnalysis - This is an analysis that determines, for a given memory operation...
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
uint16_t getParamAlignment(uint16_t i) const
Extract the alignment for a call or parameter (0=unknown).
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static MemoryLocation getForDest(const MemIntrinsic *MI)
Return a location representing the destination of a memory set or transfer.
This class represents a no-op cast from one type to another.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
StoreInst - an instruction for storing to memory.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
uint64_t getElementOffset(unsigned Idx) const
GetElementPtrInst - an instruction for type-safe pointer arithmetic to access elements of arrays and ...
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
void setDebugLoc(DebugLoc Loc)
setDebugLoc - Set the debug location information for this instruction.
The instances of the Type class are immutable: once they are created, they are never changed...
bool isByValArgument(unsigned ArgNo) const
Determine whether this argument is passed by value.
unsigned getAlignment() const
getAlignment - Return the alignment of the memory that is being allocated by the instruction.
Value * getRawDest() const
Represent the analysis usage information of a pass.
FunctionPass * createMemCpyOptPass()
FunctionPass class - This class is used to implement most global optimizations.
Value * getOperand(unsigned i) const
static CastInst * CreatePointerCast(Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd)
Create a BitCast AddrSpaceCast, or a PtrToInt cast instruction.
#define INITIALIZE_AG_DEPENDENCY(depName)
INITIALIZE_PASS_BEGIN(MemCpyOpt,"memcpyopt","MemCpy Optimization", false, false) INITIALIZE_PASS_END(MemCpyOpt
MemDepResult - A memory dependence query can return one of three different answers, described below.
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
Representation for a specific memory location.
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
unsigned getIntegerBitWidth() const
This is the shared class of boolean and integer constants.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Value * getDest() const
getDest - This is just like getRawDest, but it strips off any cast instructions that feed it...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
ModRefResult getModRefInfo(const Instruction *I)
getModRefInfo - Return information about whether or not an instruction may read or write memory (with...
Value * getLength() const
unsigned arg_size() const
Value * stripPointerCasts()
Strip off pointer casts, all-zero GEPs, and aliases.
MemCpyInst - This class wraps the llvm.memcpy intrinsic.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
iterator_range< user_iterator > users()
unsigned getLargestLegalIntTypeSize() const
Returns the size of largest legal integer type size, or 0 if none are set.
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Value * getSource() const
getSource - This is just like getRawSource, but it strips off any cast instructions that feed it...
uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
void setCalledFunction(Value *Fn)
setCalledFunction - Set the function called.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Instruction * getInst() const
getInst() - If this is a normal dependency, return the instruction that is depended on...
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Value * getRawSource() const
get* - Return the arguments to the instruction.
void setArgument(unsigned ArgNo, Value *newVal)
user_iterator user_begin()
void initializeMemCpyOptPass(PassRegistry &)
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
ModRefResult
Simple mod/ref information...
const Value * getArraySize() const
getArraySize - Get the number of elements allocated.
C - The default llvm calling convention, compatible with C.
bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB)
isNoAlias - A trivial helper function to check to see if the specified pointers are no-alias...
Legacy analysis pass which computes a DominatorTree.
Type * getAllocatedType() const
getAllocatedType - Return the type that is being allocated by the instruction.
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Value * getPointerOperand()
void combineMetadata(Instruction *K, const Instruction *J, ArrayRef< unsigned > KnownIDs)
Combine the metadata of two instructions so that K can replace J.
const BasicBlock * getParent() const
IntrinsicInst - A useful wrapper class for inspecting calls to intrinsic functions.
AllocaInst - an instruction to allocate memory on the stack.
gep_type_iterator gep_type_begin(const User *GEP)
static MemoryLocation getForSource(const MemTransferInst *MTI)
Return a location representing the source of a memory transfer.