63 #define DEBUG_TYPE "loop-idiom"
65 STATISTIC(NumMemSet,
"Number of memset's formed from loop stores");
66 STATISTIC(NumMemCpy,
"Number of memcpy's formed from loop load+stores");
70 class LoopIdiomRecognize;
93 class NclPopcountRecognize {
94 LoopIdiomRecognize &LIR;
101 explicit NclPopcountRecognize(LoopIdiomRecognize &TheLIR);
107 bool preliminaryScreen();
131 class LoopIdiomRecognize :
public LoopPass {
139 explicit LoopIdiomRecognize() :
LoopPass(
ID) {
154 bool processLoopStridedStore(
Value *DestPtr,
unsigned StoreSize,
155 unsigned StoreAlignment,
158 const SCEV *BECount);
159 bool processLoopStoreOfLoopLoad(
StoreInst *
SI,
unsigned StoreSize,
162 const SCEV *BECount);
186 : (DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree());
190 return SE ? SE : (SE = &getAnalysis<ScalarEvolution>());
195 TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
202 : (TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
203 *CurLoop->getHeader()->getParent()));
206 Loop *getLoop()
const {
return CurLoop; }
209 bool runOnNoncountableLoop();
210 bool runOnCountableLoop();
256 return Br->isUnconditional() && Br == BB->
begin();
275 NclPopcountRecognize::NclPopcountRecognize(LoopIdiomRecognize &TheLIR):
276 LIR(TheLIR), CurLoop(TheLIR.getLoop()), PreCondBB(nullptr) {
279 bool NclPopcountRecognize::preliminaryScreen() {
290 if (CurLoop->getNumBackEdges() != 1 || CurLoop->getNumBlocks() != 1)
293 BasicBlock *LoopBody = *(CurLoop->block_begin());
294 if (LoopBody->
size() >= 20) {
300 BasicBlock *PreHead = CurLoop->getLoopPreheader();
301 if (!PreHead || !LIRUtil::isAlmostEmpty(PreHead))
306 PreCondBB = LIRUtil::getPrecondBb(PreHead);
323 if (!CmpZero || !CmpZero->
isZero())
334 bool NclPopcountRecognize::detectIdiom(
Instruction *&CntInst,
359 Value *VarX1, *VarX0;
362 DefX2 = CountInst =
nullptr;
363 VarX1 = VarX0 =
nullptr;
364 PhiX = CountPhi =
nullptr;
365 LoopEntry = *(CurLoop->block_begin());
369 if (
Value *
T = matchCondition (LIRUtil::getBranch(LoopEntry), LoopEntry))
370 DefX2 = dyn_cast<Instruction>(
T);
382 if ((SubOneOp = dyn_cast<BinaryOperator>(DefX2->
getOperand(0))))
391 Instruction *SubInst = cast<Instruction>(SubOneOp);
394 !((SubInst->
getOpcode() == Instruction::Sub && Dec->isOne()) ||
395 (SubInst->
getOpcode() == Instruction::Add && Dec->isAllOnesValue()))) {
413 IterE = LoopEntry->
end(); Iter != IterE; Iter++) {
415 if (Inst->
getOpcode() != Instruction::Add)
419 if (!Inc || !Inc->
isOne())
423 if (!Phi || Phi->
getParent() != LoopEntry)
427 bool LiveOutLoop =
false;
429 if ((cast<Instruction>(U))->getParent() != LoopEntry) {
430 LiveOutLoop =
true;
break;
448 BranchInst *PreCondBr = LIRUtil::getBranch(PreCondBB);
449 Value *
T = matchCondition (PreCondBr, CurLoop->getLoopPreheader());
461 void NclPopcountRecognize::transform(
Instruction *CntInst,
466 BasicBlock *PreHead = CurLoop->getLoopPreheader();
467 BranchInst *PreCondBr = LIRUtil::getBranch(PreCondBB);
475 IRBuilderTy Builder(PreCondBr);
476 Value *PopCnt, *PopCntZext, *NewCount, *TripCnt;
478 PopCnt = createPopcntIntrinsic(Builder, Var, DL);
479 NewCount = PopCntZext =
480 Builder.CreateZExtOrTrunc(PopCnt, cast<IntegerType>(CntPhi->
getType()));
482 if (NewCount != PopCnt)
483 (cast<Instruction>(NewCount))->setDebugLoc(DL);
491 if (!InitConst || !InitConst->
isZero()) {
492 NewCount = Builder.CreateAdd(NewCount, CntInitVal);
493 (cast<Instruction>(NewCount))->setDebugLoc(DL);
504 Value *Opnd0 = PopCntZext;
505 Value *Opnd1 = ConstantInt::get(PopCntZext->
getType(), 0);
510 cast<ICmpInst>(Builder.CreateICmp(PreCond->
getPredicate(), Opnd0, Opnd1));
542 PHINode *TcPhi = PHINode::Create(Ty, 2,
"tcphi", Body->
begin());
544 Builder.SetInsertPoint(LbCond);
545 Value *Opnd1 = cast<Value>(TcPhi);
546 Value *Opnd2 = cast<Value>(ConstantInt::get(Ty, 1));
548 cast<Instruction>(Builder.CreateSub(Opnd1, Opnd2,
"tcdec",
false,
true));
557 LbCond->
setOperand(1, cast<Value>(ConstantInt::get(Ty, 0)));
572 Value *Ops[] = { Val };
575 Module *M = (*(CurLoop->block_begin()))->getParent()->getParent();
577 CallInst *CI = IRBuilder.CreateCall(Func, Ops);
586 bool NclPopcountRecognize::recognize() {
588 if (!LIR.getTargetTransformInfo())
591 LIR.getScalarEvolution();
593 if (!preliminaryScreen())
599 if (!detectIdiom(CntInst, CntPhi, Val))
602 transform(CntInst, CntPhi, Val);
612 bool LoopIdiomRecognize::runOnCountableLoop() {
614 assert(!isa<SCEVCouldNotCompute>(BECount) &&
615 "runOnCountableLoop() called on a loop without a predictable"
616 "backedge-taken count");
620 if (
const SCEVConstant *BECst = dyn_cast<SCEVConstant>(BECount))
621 if (BECst->getValue()->getValue() == 0)
625 (void)getDominatorTree();
627 LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
628 TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
631 (void)getTargetLibraryInfo();
634 CurLoop->getUniqueExitBlocks(ExitBlocks);
636 DEBUG(
dbgs() <<
"loop-idiom Scanning: F["
637 << CurLoop->getHeader()->getParent()->getName()
638 <<
"] Loop %" << CurLoop->getHeader()->getName() <<
"\n");
640 bool MadeChange =
false;
642 for (
auto *BB : CurLoop->getBlocks()) {
647 MadeChange |= runOnLoopBlock(BB, BECount, ExitBlocks);
652 bool LoopIdiomRecognize::runOnNoncountableLoop() {
653 NclPopcountRecognize Popcount(*
this);
654 if (Popcount.recognize())
661 if (skipOptnoneFunction(L))
673 if (Name ==
"memset" || Name ==
"memcpy")
676 SE = &getAnalysis<ScalarEvolution>();
678 return runOnCountableLoop();
679 return runOnNoncountableLoop();
685 bool LoopIdiomRecognize::runOnLoopBlock(
BasicBlock *BB,
const SCEV *BECount,
690 for (
unsigned i = 0, e = ExitBlocks.
size(); i != e; ++i)
691 if (!DT->dominates(BB, ExitBlocks[i]))
694 bool MadeChange =
false;
700 if (!processLoopStore(
SI, BECount))
continue;
711 if (
MemSetInst *MSI = dyn_cast<MemSetInst>(Inst)) {
713 if (!processLoopMemSet(MSI, BECount))
continue;
729 bool LoopIdiomRecognize::processLoopStore(
StoreInst *
SI,
const SCEV *BECount) {
736 auto &DL = CurLoop->getHeader()->getModule()->getDataLayout();
737 uint64_t SizeInBits = DL.getTypeSizeInBits(StoredVal->
getType());
738 if ((SizeInBits & 7) || (SizeInBits >> 32) != 0)
751 unsigned StoreSize = (
unsigned)SizeInBits >> 3;
759 dbgs() <<
"NEGATIVE STRIDE: " << *SI <<
"\n";
767 if (processLoopStridedStore(StorePtr, StoreSize, SI->
getAlignment(),
768 StoredVal,
SI, StoreEv, BECount))
774 if (
LoadInst *LI = dyn_cast<LoadInst>(StoredVal)) {
778 StoreEv->getOperand(1) == LoadEv->
getOperand(1) && LI->isSimple())
779 if (processLoopStoreOfLoopLoad(SI, StoreSize, StoreEv, LoadEv, BECount))
788 bool LoopIdiomRecognize::
794 if (!TLI->
has(LibFunc::memset))
807 uint64_t SizeInBytes = cast<ConstantInt>(MSI->
getLength())->getZExtValue();
808 if ((SizeInBytes >> 32) != 0)
820 return processLoopStridedStore(Pointer, (
unsigned)SizeInBytes,
836 uint64_t AccessSize = MemoryLocation::UnknownSize;
840 if (
const SCEVConstant *BECst = dyn_cast<SCEVConstant>(BECount))
841 AccessSize = (BECst->getValue()->getZExtValue()+1)*StoreSize;
852 if (&*
I != IgnoredStore &&
870 if (!C)
return nullptr;
874 if (Size == 0 || (Size & 7) || (Size & (Size-1)))
886 if (Size > 16)
return nullptr;
889 if (Size == 16)
return C;
892 unsigned ArraySize = 16/Size;
894 return ConstantArray::get(AT, std::vector<Constant*>(ArraySize, C));
900 bool LoopIdiomRecognize::
901 processLoopStridedStore(
Value *DestPtr,
unsigned StoreSize,
902 unsigned StoreAlignment,
Value *StoredVal,
904 const SCEV *BECount) {
912 auto &DL = CurLoop->getHeader()->getModule()->getDataLayout();
917 if (SplatValue && TLI->
has(LibFunc::memset) &&
920 CurLoop->isLoopInvariant(SplatValue)) {
922 PatternValue =
nullptr;
923 }
else if (DestAS == 0 && TLI->
has(LibFunc::memset_pattern16) &&
927 SplatValue =
nullptr;
937 BasicBlock *Preheader = CurLoop->getLoopPreheader();
949 Expander.expandCodeFor(Ev->
getStart(), DestInt8PtrTy,
954 StoreSize, getAnalysis<AliasAnalysis>(), TheStore)) {
965 Type *IntPtr = Builder.getIntPtrTy(DL, DestAS);
970 if (StoreSize != 1) {
976 Expander.expandCodeFor(NumBytesS, IntPtr, Preheader->
getTerminator());
980 NewCall = Builder.CreateMemSet(BasePtr,
986 Type *Int8PtrTy = DestInt8PtrTy;
999 GlobalValue::PrivateLinkage,
1000 PatternValue,
".memset_pattern");
1003 Value *PatternPtr = ConstantExpr::getBitCast(GV, Int8PtrTy);
1004 NewCall = Builder.CreateCall(MSP, {BasePtr, PatternPtr, NumBytes});
1007 DEBUG(
dbgs() <<
" Formed memset: " << *NewCall <<
"\n"
1008 <<
" from store to: " << *Ev <<
" at: " << *TheStore <<
"\n");
1020 bool LoopIdiomRecognize::
1021 processLoopStoreOfLoopLoad(
StoreInst *SI,
unsigned StoreSize,
1024 const SCEV *BECount) {
1026 if (!TLI->
has(LibFunc::memcpy))
1034 BasicBlock *Preheader = CurLoop->getLoopPreheader();
1036 const DataLayout &DL = Preheader->getModule()->getDataLayout();
1045 Value *StoreBasePtr =
1046 Expander.expandCodeFor(StoreEv->
getStart(),
1048 Preheader->getTerminator());
1051 CurLoop, BECount, StoreSize,
1052 getAnalysis<AliasAnalysis>(), SI)) {
1061 Value *LoadBasePtr =
1062 Expander.expandCodeFor(LoadEv->
getStart(),
1064 Preheader->getTerminator());
1067 StoreSize, getAnalysis<AliasAnalysis>(), SI)) {
1090 Expander.expandCodeFor(NumBytesS, IntPtrTy, Preheader->getTerminator());
1093 Builder.CreateMemCpy(StoreBasePtr, LoadBasePtr, NumBytes,
1097 DEBUG(
dbgs() <<
" Formed memcpy: " << *NewCall <<
"\n"
1098 <<
" from load ptr=" << *LoadEv <<
" at: " << *LI <<
"\n"
1099 <<
" from store ptr=" << *StoreEv <<
" at: " << *SI <<
"\n");
unsigned getAlignment() const
Pass interface - Implemented by all 'passes'.
Value * getValueOperand()
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
A parsed version of the target data layout string in and methods for querying it. ...
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
const SCEV * getConstant(ConstantInt *V)
STATISTIC(NumFunctions,"Total number of functions")
Value * isBytewiseValue(Value *V)
isBytewiseValue - If the specified value can be set by repeating the same byte in memory...
A Module instance is used to store all the information related to an LLVM module. ...
static void deleteDeadInstruction(Instruction *I, const TargetLibraryInfo *TLI)
deleteDeadInstruction - Delete this instruction.
Value * getValue() const
get* - Return the arguments to the instruction.
value_op_iterator value_op_begin()
ScalarEvolution - This class is the main scalar evolution driver.
CallInst - This class represents a function call, abstracting a target machine's calling convention...
MemSetInst - This class wraps the llvm.memset intrinsic.
value_op_iterator value_op_end()
const Function * getParent() const
Return the enclosing method, or null if none.
LoadInst - an instruction for reading from memory.
LoopT * getLoopFor(const BlockT *BB) const
getLoopFor - Return the inner most loop that BB lives in.
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
void setAlignment(unsigned Align)
BlockT * getHeader() const
const SCEV * getStart() const
iterator begin()
Instruction iterator methods.
static bool mayLoopAccessLocation(Value *Ptr, AliasAnalysis::ModRefResult Access, Loop *L, const SCEV *BECount, unsigned StoreSize, AliasAnalysis &AA, Instruction *IgnoredStore)
mayLoopAccessLocation - Return true if the specified loop might access the specified pointer location...
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
const APInt & getValue() const
Return the constant as an APInt value reference.
bool has(LibFunc::Func F) const
Tests whether a library function is available.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
SCEVAddRecExpr - This node represents a polynomial recurrence on the trip count of the specified loop...
Value handle that is nullable, but tries to track the Value.
ArrayType - Class to represent array types.
BasicBlock * getSuccessor(unsigned i) const
AnalysisUsage & addPreservedID(const void *ID)
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
StoreInst - an instruction for storing to memory.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
static Constant * getMemSetPatternValue(Value *V, const DataLayout &DL)
getMemSetPatternValue - If a strided store of the specified value is safe to turn into a memset_patte...
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
bool isAffine() const
isAffine - Return true if this represents an expression A + B*x where A and B are loop invariant valu...
void setDebugLoc(DebugLoc Loc)
setDebugLoc - Set the debug location information for this instruction.
BlockT * getLoopPreheader() const
getLoopPreheader - If there is a preheader for this loop, return it.
Constant * getOrInsertFunction(StringRef Name, FunctionType *T, AttributeSet AttributeList)
Look up the specified function in the module symbol table.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
BranchInst - Conditional or Unconditional Branch instruction.
This is an important base class in LLVM.
const SCEV * getOperand(unsigned i) const
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
Represent the analysis usage information of a pass.
This instruction compares its operands according to the predicate given to the constructor.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Value * getOperand(unsigned i) const
Predicate getPredicate() const
Return the predicate for this instruction.
#define INITIALIZE_AG_DEPENDENCY(depName)
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr)
RecursivelyDeleteTriviallyDeadInstructions - If the specified value is a trivially dead instruction...
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
bool isConditional() const
Representation for a specific memory location.
loop Recognize loop false
This is the shared class of boolean and integer constants.
Value * getDest() const
getDest - This is just like getRawDest, but it strips off any cast instructions that feed it...
AnalysisUsage & addRequiredID(const void *ID)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
void setUnnamedAddr(bool Val)
ModRefResult getModRefInfo(const Instruction *I)
getModRefInfo - Return information about whether or not an instruction may read or write memory (with...
Value * getLength() const
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
ConstantInt * getValue() const
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
void setPredicate(Predicate P)
Set the predicate for this instruction to the specified value.
void setOperand(unsigned i, Value *Val)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
loop Recognize loop idioms
Value * getIncomingValueForBlock(const BasicBlock *BB) const
iterator_range< user_iterator > users()
BasicBlock * getSinglePredecessor()
Return the predecessor of this block if it has a single predecessor block.
This class uses information about analyze scalars to rewrite expressions in canonical form...
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
getAddExpr - Get a canonical add expression, or something simpler if possible.
Pass * createLoopIdiomPass()
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
block_iterator block_end() const
Value * getCondition() const
void forgetLoop(const Loop *L)
forgetLoop - This method should be called by the client when it has changed a loop in a way that may ...
SCEV - This class represents an analyzed expression in the program.
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Loop * getLoop() const
void initializeLoopIdiomRecognizePass(PassRegistry &)
void setCondition(Value *V)
const SCEV * getBackedgeTakenCount(const Loop *L)
getBackedgeTakenCount - If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCouldNotCompute object.
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
ModRefResult
Simple mod/ref information...
const SCEV * getSCEV(Value *V)
getSCEV - Return a SCEV expression for the full generality of the specified expression.
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
block_iterator block_begin() const
The legacy pass manager's analysis pass to compute loop information.
C - The default llvm calling convention, compatible with C.
StringRef - Represent a constant reference to a string, i.e.
void replaceUsesOutsideBlock(Value *V, BasicBlock *BB)
replaceUsesOutsideBlock - Go through the uses list for this definition and make each use point to "V"...
Legacy analysis pass which computes a DominatorTree.
const SCEV * getTruncateOrZeroExtend(const SCEV *V, Type *Ty)
getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the input value to the speci...
INITIALIZE_PASS_BEGIN(LoopIdiomRecognize,"loop-idiom","Recognize loop idioms", false, false) INITIALIZE_PASS_END(LoopIdiomRecognize
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
getMulExpr - Get a canonical multiply expression, or something simpler if possible.
bool hasLoopInvariantBackedgeTakenCount(const Loop *L)
hasLoopInvariantBackedgeTakenCount - Return true if the specified loop has an analyzable loop-invaria...
Value * getPointerOperand()
const BasicBlock * getParent() const
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
SCEVConstant - This class represents a constant integer value.