#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/IVUsers.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionNormalization.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/OperandTraits.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <map>
#include <numeric>
#include <utility>

Macros
#define	DEBUG_TYPE "loop-reduce"

Typedefs
using	EqualValues = SmallVector< std::tuple< WeakVH, int64_t >, 4 >

using	EqualValuesMap = DenseMap< DbgValueInst *, SmallVector< std::pair< unsigned, EqualValues > >>

using	LocationMap = DenseMap< DbgValueInst , std::pair< DIExpression , Metadata * > >

Functions
static void	DoInitialMatch (const SCEV S, Loop L, SmallVectorImpl< const SCEV * > &Good, SmallVectorImpl< const SCEV * > &Bad, ScalarEvolution &SE)
	Recursion helper for initialMatch. More...

static bool	isAddRecSExtable (const SCEVAddRecExpr *AR, ScalarEvolution &SE)
	Return true if the given addrec can be sign-extended without changing its value. More...

static bool	isAddSExtable (const SCEVAddExpr *A, ScalarEvolution &SE)
	Return true if the given add can be sign-extended without changing its value. More...

static bool	isMulSExtable (const SCEVMulExpr *M, ScalarEvolution &SE)
	Return true if the given mul can be sign-extended without changing its value. More...

static const SCEV *	getExactSDiv (const SCEV LHS, const SCEV RHS, ScalarEvolution &SE, bool IgnoreSignificantBits=false)
	Return an expression for LHS /s RHS, if it can be determined and if the remainder is known to be zero, or null otherwise. More...

static int64_t	ExtractImmediate (const SCEV *&S, ScalarEvolution &SE)
	If S involves the addition of a constant integer value, return that integer value, and mutate S to point to a new SCEV with that value excluded. More...

static GlobalValue *	ExtractSymbol (const SCEV *&S, ScalarEvolution &SE)
	If S involves the addition of a GlobalValue address, return that symbol, and mutate S to point to a new SCEV with that value excluded. More...

static bool	isAddressUse (const TargetTransformInfo &TTI, Instruction Inst, Value OperandVal)
	Returns true if the specified instruction is using the specified value as an address. More...

static MemAccessTy	getAccessType (const TargetTransformInfo &TTI, Instruction Inst, Value OperandVal)
	Return the type of the memory being accessed. More...

static bool	isExistingPhi (const SCEVAddRecExpr *AR, ScalarEvolution &SE)
	Return true if this AddRec is already a phi in its loop. More...

static bool	isHighCostExpansion (const SCEV S, SmallPtrSetImpl< const SCEV > &Processed, ScalarEvolution &SE)
	Check if expanding this expression is likely to incur significant cost. More...

static bool	isAMCompletelyFolded (const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F)
	Check if the addressing mode defined by `F` is completely folded in `LU` at isel time. More...

static InstructionCost	getScalingFactorCost (const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F, const Loop &L)

static bool	isAMCompletelyFolded (const TargetTransformInfo &TTI, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, Instruction Fixup=nullptr)

static unsigned	getSetupCost (const SCEV *Reg, unsigned Depth)

static bool	isAMCompletelyFolded (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale)

static bool	isAMCompletelyFolded (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, const Formula &F, const Loop &L)

static bool	isLegalUse (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale)
	Test whether we know how to expand the current formula. More...

static bool	isLegalUse (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, const Formula &F)

static bool	isAlwaysFoldable (const TargetTransformInfo &TTI, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg)

static bool	isAlwaysFoldable (const TargetTransformInfo &TTI, ScalarEvolution &SE, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, const SCEV *S, bool HasBaseReg)

static User::op_iterator	findIVOperand (User::op_iterator OI, User::op_iterator OE, Loop *L, ScalarEvolution &SE)
	Helper for CollectChains that finds an IV operand (computed by an AddRec in this loop) within [OI,OE) or returns OE. More...

static Value *	getWideOperand (Value *Oper)
	IVChain logic must consistently peek base TruncInst operands, so wrap it in a convenient helper. More...

static bool	isCompatibleIVType (Value LVal, Value RVal)
	Return true if we allow an IV chain to include both types. More...

static const SCEV *	getExprBase (const SCEV *S)
	Return an approximation of this SCEV expression's "base", or NULL for any constant. More...

static bool	isProfitableChain (IVChain &Chain, SmallPtrSetImpl< Instruction * > &Users, ScalarEvolution &SE, const TargetTransformInfo &TTI)
	Return true if the number of registers needed for the chain is estimated to be less than the number required for the individual IV users. More...

static bool	canFoldIVIncExpr (const SCEV IncExpr, Instruction UserInst, Value *Operand, const TargetTransformInfo &TTI)
	Return true if the IVInc can be folded into an addressing mode. More...

static const SCEV *	CollectSubexprs (const SCEV S, const SCEVConstant C, SmallVectorImpl< const SCEV * > &Ops, const Loop *L, ScalarEvolution &SE, unsigned Depth=0)
	Split S into subexpressions which can be pulled out into separate registers. More...

static bool	mayUsePostIncMode (const TargetTransformInfo &TTI, LSRUse &LU, const SCEV S, const Loop L, ScalarEvolution &SE)
	Return true if the SCEV represents a value that may end up as a post-increment operation. More...

static void	DbgGatherEqualValues (Loop *L, ScalarEvolution &SE, EqualValuesMap &DbgValueToEqualSet, LocationMap &DbgValueToLocation)

static void	DbgApplyEqualValues (EqualValuesMap &DbgValueToEqualSet, LocationMap &DbgValueToLocation)

static bool	ReduceLoopStrength (Loop L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI, const TargetTransformInfo &TTI, AssumptionCache &AC, TargetLibraryInfo &TLI, MemorySSA MSSA)

	INITIALIZE_PASS_BEGIN (LoopStrengthReduce, "loop-reduce", "Loop Strength Reduction", false, false) INITIALIZE_PASS_END(LoopStrengthReduce

Variables
static const unsigned	MaxIVUsers = 200
	MaxIVUsers is an arbitrary threshold that provides an early opportunity for bail out. More...

static cl::opt< bool >	EnablePhiElim ("enable-lsr-phielim", cl::Hidden, cl::init(true), cl::desc("Enable LSR phi elimination"))

static cl::opt< bool >	InsnsCost ("lsr-insns-cost", cl::Hidden, cl::init(true), cl::desc("Add instruction count to a LSR cost model"))

static cl::opt< bool >	LSRExpNarrow ("lsr-exp-narrow", cl::Hidden, cl::init(false), cl::desc("Narrow LSR complex solution using" " expectation of registers number"))

static cl::opt< bool >	FilterSameScaledReg ("lsr-filter-same-scaled-reg", cl::Hidden, cl::init(true), cl::desc("Narrow LSR search space by filtering non-optimal formulae" " with the same ScaledReg and Scale"))

static cl::opt< TTI::AddressingModeKind >	PreferredAddresingMode ("lsr-preferred-addressing-mode", cl::Hidden, cl::init(TTI::AMK_None), cl::desc("A flag that overrides the target's preferred addressing mode."), cl::values(clEnumValN(TTI::AMK_None, "none", "Don't prefer any addressing mode"), clEnumValN(TTI::AMK_PreIndexed, "preindexed", "Prefer pre-indexed addressing mode"), clEnumValN(TTI::AMK_PostIndexed, "postindexed", "Prefer post-indexed addressing mode")))

static cl::opt< unsigned >	ComplexityLimit ("lsr-complexity-limit", cl::Hidden, cl::init(std::numeric_limits< uint16_t >::max()), cl::desc("LSR search space complexity limit"))

static cl::opt< unsigned >	SetupCostDepthLimit ("lsr-setupcost-depth-limit", cl::Hidden, cl::init(7), cl::desc("The limit on recursion depth for LSRs setup cost"))

static cl::opt< bool >	StressIVChain ("stress-ivchain", cl::Hidden, cl::init(false), cl::desc("Stress test LSR IV chains"))

loop	reduce

loop Loop Strength	Reduction

loop Loop Strength	false

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "loop-reduce"

Definition at line 131 of file LoopStrengthReduce.cpp.

Typedef Documentation

◆ EqualValues

using EqualValues = SmallVector<std::tuple<WeakVH, int64_t>, 4>

Definition at line 5832 of file LoopStrengthReduce.cpp.

◆ EqualValuesMap

using EqualValuesMap = DenseMap<DbgValueInst *, SmallVector<std::pair<unsigned, EqualValues> >>

Definition at line 5834 of file LoopStrengthReduce.cpp.

◆ LocationMap

using LocationMap = DenseMap<DbgValueInst *, std::pair<DIExpression *, Metadata *> >

Definition at line 5836 of file LoopStrengthReduce.cpp.

Function Documentation

◆ canFoldIVIncExpr()

static bool canFoldIVIncExpr	(	const SCEV *	IncExpr,
		Instruction *	UserInst,
		Value *	Operand,
		const TargetTransformInfo &	TTI
	)

static

Return true if the IVInc can be folded into an addressing mode.

Definition at line 3131 of file LoopStrengthReduce.cpp.

References getAccessType(), llvm::SCEVConstant::getAPInt(), llvm::APInt::getMinSignedBits(), llvm::ConstantInt::getSExtValue(), llvm::SCEVConstant::getValue(), isAddressUse(), and isAlwaysFoldable().

◆ CollectSubexprs()

static const SCEV* CollectSubexprs	(	const SCEV *	S,
		const SCEVConstant *	C,
		SmallVectorImpl< const SCEV * > &	Ops,
		const Loop *	L,
		ScalarEvolution &	SE,
		unsigned	Depth = `0`
	)

static

Split S into subexpressions which can be pulled out into separate registers.

If C is non-null, multiply each subexpression by C.

Return remainder expression after factoring the subexpressions captured by Ops. If Ops is complete, return NULL.

Definition at line 3502 of file LoopStrengthReduce.cpp.

References llvm::Depth, llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddRecExpr(), llvm::ScalarEvolution::getConstant(), llvm::SCEVAddRecExpr::getLoop(), llvm::ScalarEvolution::getMulExpr(), llvm::SCEVAddRecExpr::getStart(), llvm::SCEVAddRecExpr::getStepRecurrence(), llvm::SCEVNAryExpr::getType(), llvm::SCEVAddRecExpr::isAffine(), llvm::SCEV::isZero(), llvm::Mul, and S.

◆ DbgApplyEqualValues()

static void DbgApplyEqualValues	(	EqualValuesMap &	DbgValueToEqualSet,
		LocationMap &	DbgValueToLocation
	)

static

Definition at line 5881 of file LoopStrengthReduce.cpp.

References A, assert(), and Offset.

Referenced by ReduceLoopStrength().

◆ DbgGatherEqualValues()

static void DbgGatherEqualValues	(	Loop *	L,
		ScalarEvolution &	SE,
		EqualValuesMap &	DbgValueToEqualSet,
		LocationMap &	DbgValueToLocation
	)

static

Definition at line 5838 of file LoopStrengthReduce.cpp.

References B, llvm::ScalarEvolution::computeConstantDifference(), llvm::SmallVectorImpl< T >::emplace_back(), get, llvm::LoopBase< BlockT, LoopT >::getBlocks(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::ScalarEvolution::getSCEV(), llvm::Value::getType(), I, llvm::ScalarEvolution::isSCEVable(), move, Offset, and llvm::BasicBlock::phis().

Referenced by ReduceLoopStrength().

◆ DoInitialMatch()

static void DoInitialMatch	(	const SCEV *	S,
		Loop *	L,
		SmallVectorImpl< const SCEV * > &	Good,
		SmallVectorImpl< const SCEV * > &	Bad,
		ScalarEvolution &	SE
	)

static

Recursion helper for initialMatch.

Definition at line 405 of file LoopStrengthReduce.cpp.

References llvm::drop_begin(), llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddRecExpr(), llvm::Constant::getAllOnesValue(), llvm::ScalarEvolution::getConstant(), llvm::ScalarEvolution::getEffectiveSCEVType(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::ScalarEvolution::getMulExpr(), llvm::ScalarEvolution::getSCEV(), llvm::SCEV::getType(), llvm::Mul, llvm::ScalarEvolution::properlyDominates(), and S.

◆ ExtractImmediate()

static int64_t ExtractImmediate	(	const SCEV *&	S,
		ScalarEvolution &	SE
	)

static

If S involves the addition of a constant integer value, return that integer value, and mutate S to point to a new SCEV with that value excluded.

Definition at line 758 of file LoopStrengthReduce.cpp.

References llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getAddRecExpr(), llvm::ScalarEvolution::getConstant(), and S.

Referenced by isAlwaysFoldable().

◆ ExtractSymbol()

static GlobalValue* ExtractSymbol	(	const SCEV *&	S,
		ScalarEvolution &	SE
	)

static

If S involves the addition of a GlobalValue address, return that symbol, and mutate S to point to a new SCEV with that value excluded.

Definition at line 784 of file LoopStrengthReduce.cpp.

References llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getAddRecExpr(), llvm::ScalarEvolution::getConstant(), and S.

Referenced by isAlwaysFoldable().

◆ findIVOperand()

static User::op_iterator findIVOperand	(	User::op_iterator	OI,
		User::op_iterator	OE,
		Loop *	L,
		ScalarEvolution &	SE
	)

static

Helper for CollectChains that finds an IV operand (computed by an AddRec in this loop) within [OI,OE) or returns OE.

If IVUsers mapped Instructions to IVStrideUses, we could partially skip this.

Definition at line 2723 of file LoopStrengthReduce.cpp.

References llvm::SCEVAddRecExpr::getLoop(), llvm::ScalarEvolution::getSCEV(), and llvm::ScalarEvolution::isSCEVable().

◆ getAccessType()

static MemAccessTy getAccessType	(	const TargetTransformInfo &	TTI,
		Instruction *	Inst,
		Value *	OperandVal
	)

static

Return the type of the memory being accessed.

Definition at line 855 of file LoopStrengthReduce.cpp.

References llvm::IntegerType::get(), llvm::PointerType::get(), llvm::Type::getPointerAddressSpace(), llvm::TargetTransformInfo::getTgtMemIntrinsic(), llvm::Value::getType(), memcpy(), prefetch, llvm::MemIntrinsicInfo::PtrVal, SI, and UnknownAddressSpace.

Referenced by canFoldIVIncExpr().

◆ getExactSDiv()

static const SCEV* getExactSDiv	(	const SCEV *	LHS,
		const SCEV *	RHS,
		ScalarEvolution &	SE,
		bool	IgnoreSignificantBits = `false`
	)

static

Return an expression for LHS /s RHS, if it can be determined and if the remainder is known to be zero, or null otherwise.

If IgnoreSignificantBits is true, expressions like (X * Y) /s Y are simplified to Y, ignoring that the multiplication may overflow, which is useful when the result will be used in a context where the most significant bits are ignored.

Definition at line 671 of file LoopStrengthReduce.cpp.

References llvm::SCEVConstant::getAPInt(), llvm::ScalarEvolution::getConstant(), llvm::ScalarEvolution::getMulExpr(), llvm::SCEV::getType(), isAddRecSExtable(), RA, llvm::APInt::sdiv(), and llvm::APInt::srem().

◆ getExprBase()

static const SCEV* getExprBase ( const SCEV * S )

static

Return an approximation of this SCEV expression's "base", or NULL for any constant.

Returning the expression itself is conservative. Returning a deeper subexpression is more precise and valid as long as it isn't less complex than another subexpression. For expressions involving multiple unscaled values, we need to return the pointer-type SCEVUnknown. This avoids forming chains across objects, such as: PrevOper==a[i], IVOper==b[i], IVInc==b-a.

Since SCEVUnknown is the rightmost type, and pointers are the rightmost SCEVUnknown, we simply return the rightmost SCEV operand.

Definition at line 2770 of file LoopStrengthReduce.cpp.

References E, llvm::SCEV::getSCEVType(), I, llvm_unreachable, S, llvm::scAddExpr, llvm::scAddRecExpr, llvm::scConstant, llvm::scMulExpr, llvm::scSignExtend, llvm::scTruncate, and llvm::scZeroExtend.

◆ getScalingFactorCost()

static InstructionCost getScalingFactorCost	(	const TargetTransformInfo &	TTI,
		const LSRUse &	LU,
		const Formula &	F,
		const Loop &	L
	)

static

Definition at line 1760 of file LoopStrengthReduce.cpp.

References assert(), llvm::codeview::Basic, F, llvm::TargetTransformInfo::getScalingFactorCost(), isAMCompletelyFolded(), llvm::InstructionCost::isValid(), llvm_unreachable, and llvm::max().

◆ getSetupCost()

static unsigned getSetupCost	(	const SCEV *	Reg,
		unsigned	Depth
	)

static

Definition at line 1219 of file LoopStrengthReduce.cpp.

References llvm::Depth, i, Reg, and S.

◆ getWideOperand()

static Value* getWideOperand ( Value * Oper )

static

IVChain logic must consistently peek base TruncInst operands, so wrap it in a convenient helper.

Definition at line 2742 of file LoopStrengthReduce.cpp.

◆ INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	LoopStrengthReduce	,
		"loop-reduce"	,
		"Loop Strength Reduction"	,
		false	,
		false
	)

◆ isAddRecSExtable()

static bool isAddRecSExtable	(	const SCEVAddRecExpr *	AR,
		ScalarEvolution &	SE
	)

static

Return true if the given addrec can be sign-extended without changing its value.

Definition at line 643 of file LoopStrengthReduce.cpp.

References llvm::IntegerType::get(), llvm::ScalarEvolution::getContext(), llvm::ScalarEvolution::getSignExtendExpr(), llvm::SCEVNAryExpr::getType(), and llvm::ScalarEvolution::getTypeSizeInBits().

Referenced by getExactSDiv().

◆ isAddressUse()

static bool isAddressUse	(	const TargetTransformInfo &	TTI,
		Instruction *	Inst,
		Value *	OperandVal
	)

static

Returns true if the specified instruction is using the specified value as an address.

Definition at line 810 of file LoopStrengthReduce.cpp.

References llvm::TargetTransformInfo::getTgtMemIntrinsic(), memcpy(), prefetch, llvm::MemIntrinsicInfo::PtrVal, and SI.

Referenced by canFoldIVIncExpr().

◆ isAddSExtable()

static bool isAddSExtable	(	const SCEVAddExpr *	A,
		ScalarEvolution &	SE
	)

static

Return true if the given add can be sign-extended without changing its value.

Definition at line 651 of file LoopStrengthReduce.cpp.

References llvm::IntegerType::get(), llvm::ScalarEvolution::getContext(), llvm::ScalarEvolution::getSignExtendExpr(), and llvm::ScalarEvolution::getTypeSizeInBits().

◆ isAlwaysFoldable() [1/2]

static bool isAlwaysFoldable	(	const TargetTransformInfo &	TTI,
		LSRUse::KindType	Kind,
		MemAccessTy	AccessTy,
		GlobalValue *	BaseGV,
		int64_t	BaseOffset,
		bool	HasBaseReg
	)

static

Definition at line 1797 of file LoopStrengthReduce.cpp.

References isAMCompletelyFolded().

Referenced by canFoldIVIncExpr().

◆ isAlwaysFoldable() [2/2]

static bool isAlwaysFoldable	(	const TargetTransformInfo &	TTI,
		ScalarEvolution &	SE,
		int64_t	MinOffset,
		int64_t	MaxOffset,
		LSRUse::KindType	Kind,
		MemAccessTy	AccessTy,
		const SCEV *	S,
		bool	HasBaseReg
	)

static

Definition at line 1819 of file LoopStrengthReduce.cpp.

References ExtractImmediate(), ExtractSymbol(), isAMCompletelyFolded(), and S.

◆ isAMCompletelyFolded() [1/4]

static bool isAMCompletelyFolded	(	const TargetTransformInfo &	TTI,
		const LSRUse &	LU,
		const Formula &	F
	)

static

Check if the addressing mode defined by F is completely folded in LU at isel time.

This includes address-mode folding and special icmp tricks. This function returns true if LU can accommodate what F defines and up to 1 base + 1 scaled + offset. In other words, if F has several base registers, this function may still return true. Therefore, users still need to account for additional base registers and/or unfolded offsets to derive an accurate cost model.

Definition at line 1743 of file LoopStrengthReduce.cpp.

References F, Fixup, and llvm::TargetTransformInfo::LSRWithInstrQueries().

Referenced by getScalingFactorCost(), isAlwaysFoldable(), isAMCompletelyFolded(), and isLegalUse().

◆ isAMCompletelyFolded() [2/4]

static bool isAMCompletelyFolded	(	const TargetTransformInfo &	TTI,
		int64_t	MinOffset,
		int64_t	MaxOffset,
		LSRUse::KindType	Kind,
		MemAccessTy	AccessTy,
		const Formula &	F,
		const Loop &	L
	)

static

Definition at line 1705 of file LoopStrengthReduce.cpp.

References assert(), F, and isAMCompletelyFolded().

◆ isAMCompletelyFolded() [3/4]

static bool isAMCompletelyFolded	(	const TargetTransformInfo &	TTI,
		int64_t	MinOffset,
		int64_t	MaxOffset,
		LSRUse::KindType	Kind,
		MemAccessTy	AccessTy,
		GlobalValue *	BaseGV,
		int64_t	BaseOffset,
		bool	HasBaseReg,
		int64_t	Scale
	)

static

Definition at line 1684 of file LoopStrengthReduce.cpp.

References isAMCompletelyFolded().

◆ isAMCompletelyFolded() [4/4]

static bool isAMCompletelyFolded	(	const TargetTransformInfo &	TTI,
		LSRUse::KindType	Kind,
		MemAccessTy	AccessTy,
		GlobalValue *	BaseGV,
		int64_t	BaseOffset,
		bool	HasBaseReg,
		int64_t	Scale,
		Instruction *	Fixup = `nullptr`
	)

static

Definition at line 1630 of file LoopStrengthReduce.cpp.

References llvm::codeview::Basic, Fixup, llvm::TargetTransformInfo::isLegalAddressingMode(), llvm::TargetTransformInfo::isLegalICmpImmediate(), and llvm_unreachable.

◆ isCompatibleIVType()

static bool isCompatibleIVType	(	Value *	LVal,
		Value *	RVal
	)

static

Return true if we allow an IV chain to include both types.

Definition at line 2749 of file LoopStrengthReduce.cpp.

References llvm::Type::getPointerAddressSpace(), llvm::Value::getType(), and llvm::Type::isPointerTy().

◆ isExistingPhi()

static bool isExistingPhi	(	const SCEVAddRecExpr *	AR,
		ScalarEvolution &	SE
	)

static

Return true if this AddRec is already a phi in its loop.

Definition at line 910 of file LoopStrengthReduce.cpp.

References llvm::ScalarEvolution::getEffectiveSCEVType(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::SCEVAddRecExpr::getLoop(), llvm::ScalarEvolution::getSCEV(), llvm::SCEVNAryExpr::getType(), llvm::ScalarEvolution::isSCEVable(), and llvm::BasicBlock::phis().

Referenced by isHighCostExpansion().

◆ isHighCostExpansion()

static bool isHighCostExpansion	(	const SCEV *	S,
		SmallPtrSetImpl< const SCEV * > &	Processed,
		ScalarEvolution &	SE
	)

static

Check if expanding this expression is likely to incur significant cost.

This is tricky because SCEV doesn't track which expressions are actually computed by the current IR.

We currently allow expansion of IV increments that involve adds, multiplication by constants, and AddRecs from existing phis.

TODO: Allow UDivExpr if we can find an existing IV increment that is an obvious multiple of the UDivExpr.

Definition at line 930 of file LoopStrengthReduce.cpp.

References llvm::Instruction::getOpcode(), llvm::ScalarEvolution::getSCEV(), llvm::Value::getType(), llvm::SmallPtrSetImpl< PtrType >::insert(), isExistingPhi(), llvm::ScalarEvolution::isSCEVable(), llvm::Mul, S, llvm::scConstant, llvm::scSignExtend, llvm::scTruncate, llvm::scUnknown, llvm::scZeroExtend, and llvm::Value::users().

◆ isLegalUse() [1/2]

static bool isLegalUse	(	const TargetTransformInfo &	TTI,
		int64_t	MinOffset,
		int64_t	MaxOffset,
		LSRUse::KindType	Kind,
		MemAccessTy	AccessTy,
		const Formula &	F
	)

static

Definition at line 1736 of file LoopStrengthReduce.cpp.

References F, and isLegalUse().

◆ isLegalUse() [2/2]

static bool isLegalUse	(	const TargetTransformInfo &	TTI,
		int64_t	MinOffset,
		int64_t	MaxOffset,
		LSRUse::KindType	Kind,
		MemAccessTy	AccessTy,
		GlobalValue *	BaseGV,
		int64_t	BaseOffset,
		bool	HasBaseReg,
		int64_t	Scale
	)

static

Test whether we know how to expand the current formula.

Definition at line 1722 of file LoopStrengthReduce.cpp.

References isAMCompletelyFolded().

Referenced by isLegalUse().

◆ isMulSExtable()

static bool isMulSExtable	(	const SCEVMulExpr *	M,
		ScalarEvolution &	SE
	)

static

Return true if the given mul can be sign-extended without changing its value.

Definition at line 659 of file LoopStrengthReduce.cpp.

References llvm::IntegerType::get(), llvm::ScalarEvolution::getContext(), llvm::ScalarEvolution::getSignExtendExpr(), llvm::ScalarEvolution::getTypeSizeInBits(), and M.

◆ isProfitableChain()

static bool isProfitableChain	(	IVChain &	Chain,
		SmallPtrSetImpl< Instruction * > &	Users,
		ScalarEvolution &	SE,
		const TargetTransformInfo &	TTI
	)

static

Return true if the number of registers needed for the chain is estimated to be less than the number required for the individual IV users.

First prohibit any IV users that keep the IV live across increments (the Users set should be empty). Next count the number and type of increments in the chain.

Chaining IVs can lead to considerable code bloat if ISEL doesn't effectively use postinc addressing modes. Only consider it profitable it the increments can be computed in fewer registers when chained.

TODO: Consider IVInc free if it's already used in another chains.

Definition at line 2838 of file LoopStrengthReduce.cpp.

References assert(), llvm::dbgs(), for, llvm::ScalarEvolution::getSCEV(), llvm::TargetTransformInfo::isProfitableLSRChainElement(), LLVM_DEBUG, StressIVChain, and Users.

◆ mayUsePostIncMode()

static bool mayUsePostIncMode	(	const TargetTransformInfo &	TTI,
		LSRUse &	LU,
		const SCEV *	S,
		const Loop *	L,
		ScalarEvolution &	SE
	)

static

Return true if the SCEV represents a value that may end up as a post-increment operation.

Definition at line 3560 of file LoopStrengthReduce.cpp.

References llvm::SCEVAddRecExpr::getStart(), llvm::SCEVAddRecExpr::getStepRecurrence(), llvm::SCEVNAryExpr::getType(), llvm::TargetTransformInfo::isIndexedLoadLegal(), llvm::TargetTransformInfo::isIndexedStoreLegal(), llvm::ScalarEvolution::isLoopInvariant(), llvm::TargetTransformInfo::MIM_PostInc, and S.

◆ ReduceLoopStrength()

static bool ReduceLoopStrength	(	Loop *	L,
		IVUsers &	IU,
		ScalarEvolution &	SE,
		DominatorTree &	DT,
		LoopInfo &	LI,
		const TargetTransformInfo &	TTI,
		AssumptionCache &	AC,
		TargetLibraryInfo &	TLI,
		MemorySSA *	MSSA
	)

static

Definition at line 5915 of file LoopStrengthReduce.cpp.

References DbgApplyEqualValues(), DbgGatherEqualValues(), DEBUG_TYPE, llvm::DeleteDeadPHIs(), DL, EnablePhiElim, llvm::Module::getDataLayout(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::BasicBlock::getModule(), llvm::Loop::isLoopSimplifyForm(), llvm::RecursivelyDeleteTriviallyDeadInstructionsPermissive(), and Rewriter.

Referenced by llvm::LoopStrengthReducePass::run().

Variable Documentation

◆ ComplexityLimit

cl::opt<unsigned> ComplexityLimit("lsr-complexity-limit", cl::Hidden, cl::init(std::numeric_limits< uint16_t >::max()), cl::desc("LSR search space complexity limit"))

static

◆ EnablePhiElim

cl::opt<bool> EnablePhiElim("enable-lsr-phielim", cl::Hidden, cl::init(true), cl::desc("Enable LSR phi elimination"))

static

Referenced by ReduceLoopStrength().

◆ false

loop Loop Strength false

Definition at line 6004 of file LoopStrengthReduce.cpp.

◆ FilterSameScaledReg

cl::opt<bool> FilterSameScaledReg("lsr-filter-same-scaled-reg", cl::Hidden, cl::init(true), cl::desc("Narrow LSR search space by filtering non-optimal formulae" " with the same ScaledReg and Scale"))

static

◆ InsnsCost

cl::opt<bool> InsnsCost("lsr-insns-cost", cl::Hidden, cl::init(true), cl::desc("Add instruction count to a LSR cost model"))

static

◆ LSRExpNarrow

cl::opt<bool> LSRExpNarrow("lsr-exp-narrow", cl::Hidden, cl::init(false), cl::desc("Narrow LSR complex solution using" " expectation of registers number"))

static

◆ MaxIVUsers

const unsigned MaxIVUsers = 200

static

MaxIVUsers is an arbitrary threshold that provides an early opportunity for bail out.

This threshold is far beyond the number of users that LSR can conceivably solve, so it should not affect generated code, but catches the worst cases before LSR burns too much compile time and stack space.

Definition at line 137 of file LoopStrengthReduce.cpp.

◆ PreferredAddresingMode

cl::opt<TTI::AddressingModeKind> PreferredAddresingMode("lsr-preferred-addressing-mode", cl::Hidden, cl::init(TTI::AMK_None), cl::desc("A flag that overrides the target's preferred addressing mode."), cl::values(clEnumValN(TTI::AMK_None, "none", "Don't prefer any addressing mode"), clEnumValN(TTI::AMK_PreIndexed, "preindexed", "Prefer pre-indexed addressing mode"), clEnumValN(TTI::AMK_PostIndexed, "postindexed", "Prefer post-indexed addressing mode")))

static

◆ reduce

loop reduce

Definition at line 6003 of file LoopStrengthReduce.cpp.

◆ Reduction

loop Loop Strength Reduction

Definition at line 6004 of file LoopStrengthReduce.cpp.

Referenced by llvm::LoopVectorizationLegality::canVectorizeFPMath(), llvm::LoopVectorizationCostModel::canVectorizeReductions(), llvm::LoopVectorizationCostModel::collectInLoopReductions(), llvm::LoopVectorizationCostModel::collectValuesToIgnore(), PerformSELECTCombine(), and llvm::LoopVectorizationLegality::prepareToFoldTailByMasking().

◆ SetupCostDepthLimit

cl::opt<unsigned> SetupCostDepthLimit("lsr-setupcost-depth-limit", cl::Hidden, cl::init(7), cl::desc("The limit on recursion depth for LSRs setup cost"))

static

◆ StressIVChain

cl::opt<bool> StressIVChain("stress-ivchain", cl::Hidden, cl::init(false), cl::desc("Stress test LSR IV chains"))

static

Referenced by isProfitableChain().

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

◆ DEBUG_TYPE

Typedef Documentation

◆ EqualValues

◆ EqualValuesMap

◆ LocationMap

Function Documentation

◆ canFoldIVIncExpr()

◆ CollectSubexprs()

◆ DbgApplyEqualValues()

◆ DbgGatherEqualValues()

◆ DoInitialMatch()

◆ ExtractImmediate()

◆ ExtractSymbol()

◆ findIVOperand()

◆ getAccessType()

◆ getExactSDiv()

◆ getExprBase()

◆ getScalingFactorCost()

◆ getSetupCost()

◆ getWideOperand()

◆ INITIALIZE_PASS_BEGIN()

◆ isAddRecSExtable()

◆ isAddressUse()

◆ isAddSExtable()

◆ isAlwaysFoldable() [1/2]

◆ isAlwaysFoldable() [2/2]

◆ isAMCompletelyFolded() [1/4]

◆ isAMCompletelyFolded() [2/4]

◆ isAMCompletelyFolded() [3/4]

◆ isAMCompletelyFolded() [4/4]

◆ isCompatibleIVType()

◆ isExistingPhi()

◆ isHighCostExpansion()

◆ isLegalUse() [1/2]

◆ isLegalUse() [2/2]

◆ isMulSExtable()

◆ isProfitableChain()

◆ mayUsePostIncMode()

◆ ReduceLoopStrength()

Variable Documentation

◆ ComplexityLimit

◆ EnablePhiElim

◆ false

◆ FilterSameScaledReg

◆ InsnsCost

◆ LSRExpNarrow

◆ MaxIVUsers

◆ PreferredAddresingMode

◆ reduce

◆ Reduction

◆ SetupCostDepthLimit

◆ StressIVChain