LLVM 19.0.0git
Classes | Macros | Functions | Variables
LoopIdiomRecognize.cpp File Reference
#include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/CmpInstAnalysis.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/MustExecute.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BuildLibCalls.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <utility>
#include <vector>

Go to the source code of this file.

Classes

struct  match_LoopInvariant< SubPattern_t >
 Match loop-invariant value. More...
 

Macros

#define DEBUG_TYPE   "loop-idiom"
 

Functions

 STATISTIC (NumMemSet, "Number of memset's formed from loop stores")
 
 STATISTIC (NumMemCpy, "Number of memcpy's formed from loop load+stores")
 
 STATISTIC (NumMemMove, "Number of memmove's formed from loop load+stores")
 
 STATISTIC (NumShiftUntilBitTest, "Number of uncountable loops recognized as 'shift until bitttest' idiom")
 
 STATISTIC (NumShiftUntilZero, "Number of uncountable loops recognized as 'shift until zero' idiom")
 
static void deleteDeadInstruction (Instruction *I)
 
static APInt getStoreStride (const SCEVAddRecExpr *StoreEv)
 
static ConstantgetMemSetPatternValue (Value *V, const DataLayout *DL)
 getMemSetPatternValue - If a strided store of the specified value is safe to turn into a memset_pattern16, return a ConstantArray of 16 bytes that should be passed in.
 
static bool mayLoopAccessLocation (Value *Ptr, ModRefInfo Access, Loop *L, const SCEV *BECount, const SCEV *StoreSizeSCEV, AliasAnalysis &AA, SmallPtrSetImpl< Instruction * > &IgnoredInsts)
 mayLoopAccessLocation - Return true if the specified loop might access the specified pointer location, which is a loop-strided access.
 
static const SCEVgetStartForNegStride (const SCEV *Start, const SCEV *BECount, Type *IntPtr, const SCEV *StoreSizeSCEV, ScalarEvolution *SE)
 
static const SCEVgetNumBytes (const SCEV *BECount, Type *IntPtr, const SCEV *StoreSizeSCEV, Loop *CurLoop, const DataLayout *DL, ScalarEvolution *SE)
 Compute the number of bytes as a SCEV from the backedge taken count.
 
static ValuematchCondition (BranchInst *BI, BasicBlock *LoopEntry, bool JmpOnZero=false)
 Check if the given conditional branch is based on the comparison between a variable and zero, and if the variable is non-zero or zero (JmpOnZero is true), the control yields to the loop entry.
 
static PHINodegetRecurrenceVar (Value *VarX, Instruction *DefX, BasicBlock *LoopEntry)
 
static bool detectPopcountIdiom (Loop *CurLoop, BasicBlock *PreCondBB, Instruction *&CntInst, PHINode *&CntPhi, Value *&Var)
 Return true iff the idiom is detected in the loop.
 
static bool detectShiftUntilZeroIdiom (Loop *CurLoop, const DataLayout &DL, Intrinsic::ID &IntrinID, Value *&InitX, Instruction *&CntInst, PHINode *&CntPhi, Instruction *&DefX)
 Return true if the idiom is detected in the loop.
 
static CallInstcreatePopcntIntrinsic (IRBuilder<> &IRBuilder, Value *Val, const DebugLoc &DL)
 
static CallInstcreateFFSIntrinsic (IRBuilder<> &IRBuilder, Value *Val, const DebugLoc &DL, bool ZeroCheck, Intrinsic::ID IID)
 
template<typename Ty >
match_LoopInvariant< Ty > m_LoopInvariant (const Ty &M, const Loop *L)
 Matches if the value is loop-invariant.
 
static bool detectShiftUntilBitTestIdiom (Loop *CurLoop, Value *&BaseX, Value *&BitMask, Value *&BitPos, Value *&CurrX, Instruction *&NextX)
 Return true if the idiom is detected in the loop.
 
static bool detectShiftUntilZeroIdiom (Loop *CurLoop, ScalarEvolution *SE, Instruction *&ValShiftedIsZero, Intrinsic::ID &IntrinID, Instruction *&IV, Value *&Start, Value *&Val, const SCEV *&ExtraOffsetExpr, bool &InvertedCond)
 Return true if the idiom is detected in the loop.
 

Variables

static cl::opt< bool, trueDisableLIRPAll ("disable-" DEBUG_TYPE "-all", cl::desc("Options to disable Loop Idiom Recognize Pass."), cl::location(DisableLIRP::All), cl::init(false), cl::ReallyHidden)
 
static cl::opt< bool, trueDisableLIRPMemset ("disable-" DEBUG_TYPE "-memset", cl::desc("Proceed with loop idiom recognize pass, but do " "not convert loop(s) to memset."), cl::location(DisableLIRP::Memset), cl::init(false), cl::ReallyHidden)
 
static cl::opt< bool, trueDisableLIRPMemcpy ("disable-" DEBUG_TYPE "-memcpy", cl::desc("Proceed with loop idiom recognize pass, but do " "not convert loop(s) to memcpy."), cl::location(DisableLIRP::Memcpy), cl::init(false), cl::ReallyHidden)
 
static cl::opt< boolUseLIRCodeSizeHeurs ("use-lir-code-size-heurs", cl::desc("Use loop idiom recognition code size heuristics when compiling" "with -Os/-Oz"), cl::init(true), cl::Hidden)
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "loop-idiom"

Definition at line 98 of file LoopIdiomRecognize.cpp.

Function Documentation

◆ createFFSIntrinsic()

static CallInst * createFFSIntrinsic ( IRBuilder<> &  IRBuilder,
Value Val,
const DebugLoc DL,
bool  ZeroCheck,
Intrinsic::ID  IID 
)
static

◆ createPopcntIntrinsic()

static CallInst * createPopcntIntrinsic ( IRBuilder<> &  IRBuilder,
Value Val,
const DebugLoc DL 
)
static

◆ deleteDeadInstruction()

static void deleteDeadInstruction ( Instruction I)
static

Definition at line 274 of file LoopIdiomRecognize.cpp.

References llvm::PoisonValue::get(), and I.

◆ detectPopcountIdiom()

static bool detectPopcountIdiom ( Loop CurLoop,
BasicBlock PreCondBB,
Instruction *&  CntInst,
PHINode *&  CntPhi,
Value *&  Var 
)
static

Return true iff the idiom is detected in the loop.

Additionally: 1) CntInst is set to the instruction counting the population bit. 2) CntPhi is set to the corresponding phi node. 3) Var is set to the value whose population bits are being counted.

The core idiom we are trying to detect is:

if (x0 != 0)
goto loop-exit // the precondition of the loop
cnt0 = init-val;
do {
x1 = phi (x0, x2);
cnt1 = phi(cnt0, cnt2);
cnt2 = cnt1 + 1;
...
x2 = x1 & (x1 - 1);
...
} while(x != 0);
loop-exit:

Definition at line 1557 of file LoopIdiomRecognize.cpp.

References llvm::LoopBase< BlockT, LoopT >::block_begin(), llvm::BasicBlock::end(), llvm::BasicBlock::getFirstNonPHI(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::BinaryOperator::getOpcode(), llvm::Instruction::getOpcode(), llvm::User::getOperand(), getParent(), getRecurrenceVar(), llvm::BasicBlock::getTerminator(), llvm::ConstantInt::isMinusOne(), llvm::ConstantInt::isOne(), llvm::make_range(), matchCondition(), and llvm::Value::users().

◆ detectShiftUntilBitTestIdiom()

static bool detectShiftUntilBitTestIdiom ( Loop CurLoop,
Value *&  BaseX,
Value *&  BitMask,
Value *&  BitPos,
Value *&  CurrX,
Instruction *&  NextX 
)
static

Return true if the idiom is detected in the loop.

The core idiom we are trying to detect is:

entry:
<...>
%bitmask = shl i32 1, %bitpos
br label %loop
loop:
%x.curr = phi i32 [ %x, %entry ], [ %x.next, %loop ]
%x.curr.bitmasked = and i32 %x.curr, %bitmask
%x.curr.isbitunset = icmp eq i32 %x.curr.bitmasked, 0
%x.next = shl i32 %x.curr, 1
<...>
br i1 %x.curr.isbitunset, label %loop, label %end
end:
%x.curr.res = phi i32 [ %x.curr, %loop ] <...>
%x.next.res = phi i32 [ %x.next, %loop ] <...>
<...>

Definition at line 2203 of file LoopIdiomRecognize.cpp.

References assert(), llvm::dbgs(), DEBUG_TYPE, llvm::decomposeBitTestICmp(), llvm::ConstantExpr::getExactLogBase2(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::CmpInst::getInversePredicate(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::LoopBase< BlockT, LoopT >::getNumBackEdges(), llvm::LoopBase< BlockT, LoopT >::getNumBlocks(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::CmpInst::ICMP_EQ, llvm::ICmpInst::isEquality(), llvm::Loop::isLoopInvariant(), LLVM_DEBUG, llvm::PatternMatch::m_And(), llvm::PatternMatch::m_BasicBlock(), llvm::PatternMatch::m_Br(), llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_CombineAnd(), llvm::PatternMatch::m_ICmp(), m_LoopInvariant(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Power2(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::match(), and std::swap().

◆ detectShiftUntilZeroIdiom() [1/2]

static bool detectShiftUntilZeroIdiom ( Loop CurLoop,
const DataLayout DL,
Intrinsic::ID IntrinID,
Value *&  InitX,
Instruction *&  CntInst,
PHINode *&  CntPhi,
Instruction *&  DefX 
)
static

Return true if the idiom is detected in the loop.

Additionally: 1) CntInst is set to the instruction Counting Leading Zeros (CTLZ) or nullptr if there is no such. 2) CntPhi is set to the corresponding phi node or nullptr if there is no such. 3) Var is set to the value whose CTLZ could be used. 4) DefX is set to the instruction calculating Loop exit condition.

The core idiom we are trying to detect is:

if (x0 == 0)
goto loop-exit // the precondition of the loop
cnt0 = init-val;
do {
x = phi (x0, x.next); //PhiX
cnt = phi(cnt0, cnt.next);
cnt.next = cnt + 1;
...
x.next = x >> 1; // DefX
...
} while(x.next != 0);
loop-exit:

Definition at line 1690 of file LoopIdiomRecognize.cpp.

References llvm::LoopBase< BlockT, LoopT >::block_begin(), DL, llvm::BasicBlock::end(), llvm::BasicBlock::getFirstNonPHI(), llvm::PHINode::getIncomingValueForBlock(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::Instruction::getOpcode(), llvm::User::getOperand(), getRecurrenceVar(), llvm::BasicBlock::getTerminator(), llvm::isKnownNonNegative(), llvm::ConstantInt::isMinusOne(), llvm::ConstantInt::isOne(), llvm::Instruction::isShift(), llvm::make_range(), and matchCondition().

◆ detectShiftUntilZeroIdiom() [2/2]

static bool detectShiftUntilZeroIdiom ( Loop CurLoop,
ScalarEvolution SE,
Instruction *&  ValShiftedIsZero,
Intrinsic::ID IntrinID,
Instruction *&  IV,
Value *&  Start,
Value *&  Val,
const SCEV *&  ExtraOffsetExpr,
bool InvertedCond 
)
static

Return true if the idiom is detected in the loop.

The core idiom we are trying to detect is:

entry:
<...>
%start = <...>
%extraoffset = <...>
<...>
br label %for.cond
loop:
%iv = phi i8 [ %start, %entry ], [ %iv.next, %for.cond ]
%nbits = add nsw i8 %iv, %extraoffset
%val.shifted = {{l,a}shr,shl} i8 %val, %nbits
%val.shifted.iszero = icmp eq i8 %val.shifted, 0
%iv.next = add i8 %iv, 1
<...>
br i1 %val.shifted.iszero, label %end, label %loop
%iv.res = phi i8 [ %iv, %loop ] <...>
%nbits.res = phi i8 [ %nbits, %loop ] <...>
%val.shifted.res = phi i8 [ %val.shifted, %loop ] <...>
%val.shifted.iszero.res = phi i1 [ %val.shifted.iszero, %loop ] <...>
%iv.next.res = phi i8 [ %iv.next, %loop ] <...>
<...>
constexpr double phi
Definition: MathExtras.h:45
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:236

Definition at line 2557 of file LoopIdiomRecognize.cpp.

References assert(), llvm::dbgs(), DEBUG_TYPE, llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::CmpInst::getInversePredicate(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::ScalarEvolution::getNegativeSCEV(), llvm::LoopBase< BlockT, LoopT >::getNumBackEdges(), llvm::LoopBase< BlockT, LoopT >::getNumBlocks(), llvm::Instruction::getOpcode(), llvm::ScalarEvolution::getSCEV(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::ScalarEvolution::getZero(), llvm::Instruction::hasNoSignedWrap(), llvm::Instruction::hasNoUnsignedWrap(), llvm::CmpInst::ICMP_EQ, llvm::ICmpInst::isEquality(), llvm::ScalarEvolution::isKnownNonNegative(), llvm::isMustProgress(), IV, LLVM_DEBUG, llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_BasicBlock(), llvm::PatternMatch::m_Br(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_Instruction(), m_LoopInvariant(), llvm::PatternMatch::m_One(), llvm::PatternMatch::m_Shift(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), llvm::PatternMatch::match(), and std::swap().

◆ getMemSetPatternValue()

static Constant * getMemSetPatternValue ( Value V,
const DataLayout DL 
)
static

getMemSetPatternValue - If a strided store of the specified value is safe to turn into a memset_pattern16, return a ConstantArray of 16 bytes that should be passed in.

Otherwise, return null.

Note that we don't ever attempt to use memset_pattern8 or 4, because these just replicate their input array and then pass on to memset_pattern16.

Definition at line 363 of file LoopIdiomRecognize.cpp.

References llvm::CallingConv::C, DL, llvm::ConstantArray::get(), llvm::ArrayType::get(), and Size.

◆ getNumBytes()

static const SCEV * getNumBytes ( const SCEV BECount,
Type IntPtr,
const SCEV StoreSizeSCEV,
Loop CurLoop,
const DataLayout DL,
ScalarEvolution SE 
)
static

Compute the number of bytes as a SCEV from the backedge taken count.

This also maps the SCEV into the provided type and tries to handle the computation in a way that will fold cleanly.

Definition at line 988 of file LoopIdiomRecognize.cpp.

References llvm::SCEV::FlagNUW, llvm::ScalarEvolution::getMulExpr(), llvm::ScalarEvolution::getTripCountFromExitCount(), and llvm::ScalarEvolution::getTruncateOrZeroExtend().

Referenced by llvm::DebugLocStream::getBytes().

◆ getRecurrenceVar()

static PHINode * getRecurrenceVar ( Value VarX,
Instruction DefX,
BasicBlock LoopEntry 
)
static

Definition at line 1524 of file LoopIdiomRecognize.cpp.

References llvm::User::getOperand().

Referenced by detectPopcountIdiom(), and detectShiftUntilZeroIdiom().

◆ getStartForNegStride()

static const SCEV * getStartForNegStride ( const SCEV Start,
const SCEV BECount,
Type IntPtr,
const SCEV StoreSizeSCEV,
ScalarEvolution SE 
)
static

◆ getStoreStride()

static APInt getStoreStride ( const SCEVAddRecExpr StoreEv)
static

◆ m_LoopInvariant()

template<typename Ty >
match_LoopInvariant< Ty > m_LoopInvariant ( const Ty &  M,
const Loop L 
)
inline

Matches if the value is loop-invariant.

Definition at line 2177 of file LoopIdiomRecognize.cpp.

Referenced by detectShiftUntilBitTestIdiom(), and detectShiftUntilZeroIdiom().

◆ matchCondition()

static Value * matchCondition ( BranchInst BI,
BasicBlock LoopEntry,
bool  JmpOnZero = false 
)
static

Check if the given conditional branch is based on the comparison between a variable and zero, and if the variable is non-zero or zero (JmpOnZero is true), the control yields to the loop entry.

If the branch matches the behavior, the variable involved in the comparison is returned. This function will be called to see if the precondition and postcondition of the loop are in desirable form.

Definition at line 1496 of file LoopIdiomRecognize.cpp.

References Cond, llvm::BranchInst::getCondition(), llvm::BranchInst::getSuccessor(), llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::BranchInst::isConditional(), llvm::ConstantInt::isZero(), and std::swap().

Referenced by detectPopcountIdiom(), and detectShiftUntilZeroIdiom().

◆ mayLoopAccessLocation()

static bool mayLoopAccessLocation ( Value Ptr,
ModRefInfo  Access,
Loop L,
const SCEV BECount,
const SCEV StoreSizeSCEV,
AliasAnalysis AA,
SmallPtrSetImpl< Instruction * > &  IgnoredInsts 
)
static

mayLoopAccessLocation - Return true if the specified loop might access the specified pointer location, which is a loop-strided access.

The 'Access' argument specifies what the verboten forms of access are (read or write).

Definition at line 932 of file LoopIdiomRecognize.cpp.

References llvm::LocationSize::afterPointer(), B, llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::SCEVConstant::getAPInt(), llvm::AAResults::getModRefInfo(), I, llvm::isModOrRefSet(), llvm::LocationSize::precise(), Ptr, and llvm::APInt::tryZExtValue().

◆ STATISTIC() [1/5]

STATISTIC ( NumMemCpy  ,
"Number of memcpy's formed from loop load+stores"   
)

◆ STATISTIC() [2/5]

STATISTIC ( NumMemMove  ,
"Number of memmove's formed from loop load+stores"   
)

◆ STATISTIC() [3/5]

STATISTIC ( NumMemSet  ,
"Number of memset's formed from loop stores"   
)

◆ STATISTIC() [4/5]

STATISTIC ( NumShiftUntilBitTest  ,
"Number of uncountable loops recognized as 'shift until bitttest' idiom"   
)

◆ STATISTIC() [5/5]

STATISTIC ( NumShiftUntilZero  ,
"Number of uncountable loops recognized as 'shift until zero' idiom"   
)

Variable Documentation

◆ DisableLIRPAll

cl::opt< bool, true > DisableLIRPAll("disable-" DEBUG_TYPE "-all", cl::desc("Options to disable Loop Idiom Recognize Pass."), cl::location(DisableLIRP::All), cl::init(false), cl::ReallyHidden) ( "disable-" DEBUG_TYPE "-all"  ,
cl::desc("Options to disable Loop Idiom Recognize Pass.")  ,
cl::location(DisableLIRP::All ,
cl::init(false)  ,
cl::ReallyHidden   
)
static

◆ DisableLIRPMemcpy

cl::opt< bool, true > DisableLIRPMemcpy("disable-" DEBUG_TYPE "-memcpy", cl::desc("Proceed with loop idiom recognize pass, but do " "not convert loop(s) to memcpy."), cl::location(DisableLIRP::Memcpy), cl::init(false), cl::ReallyHidden) ( "disable-" DEBUG_TYPE "-memcpy"  ,
cl::desc("Proceed with loop idiom recognize pass, but do " "not convert loop(s) to memcpy.")  ,
cl::location(DisableLIRP::Memcpy)  ,
cl::init(false)  ,
cl::ReallyHidden   
)
static

◆ DisableLIRPMemset

cl::opt< bool, true > DisableLIRPMemset("disable-" DEBUG_TYPE "-memset", cl::desc("Proceed with loop idiom recognize pass, but do " "not convert loop(s) to memset."), cl::location(DisableLIRP::Memset), cl::init(false), cl::ReallyHidden) ( "disable-" DEBUG_TYPE "-memset"  ,
cl::desc("Proceed with loop idiom recognize pass, but do " "not convert loop(s) to memset.")  ,
cl::location(DisableLIRP::Memset)  ,
cl::init(false)  ,
cl::ReallyHidden   
)
static

◆ UseLIRCodeSizeHeurs

cl::opt< bool > UseLIRCodeSizeHeurs("use-lir-code-size-heurs", cl::desc("Use loop idiom recognition code size heuristics when compiling" "with -Os/-Oz"), cl::init(true), cl::Hidden) ( "use-lir-code-size-heurs"  ,
cl::desc("Use loop idiom recognition code size heuristics when compiling" "with -Os/-Oz")  ,
cl::init(true ,
cl::Hidden   
)
static