LLVM  6.0.0svn
Macros | Functions | Variables
LoopStrengthReduce.cpp File Reference
#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/PointerIntPair.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/IVUsers.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpander.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionNormalization.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/OperandTraits.h"
#include "llvm/IR/Operator.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <map>
#include <utility>

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "loop-reduce"
 

Functions

static void DoInitialMatch (const SCEV *S, Loop *L, SmallVectorImpl< const SCEV *> &Good, SmallVectorImpl< const SCEV *> &Bad, ScalarEvolution &SE)
 Recursion helper for initialMatch. More...
 
static bool isAddRecSExtable (const SCEVAddRecExpr *AR, ScalarEvolution &SE)
 Return true if the given addrec can be sign-extended without changing its value. More...
 
static bool isAddSExtable (const SCEVAddExpr *A, ScalarEvolution &SE)
 Return true if the given add can be sign-extended without changing its value. More...
 
static bool isMulSExtable (const SCEVMulExpr *M, ScalarEvolution &SE)
 Return true if the given mul can be sign-extended without changing its value. More...
 
static const SCEVgetExactSDiv (const SCEV *LHS, const SCEV *RHS, ScalarEvolution &SE, bool IgnoreSignificantBits=false)
 Return an expression for LHS /s RHS, if it can be determined and if the remainder is known to be zero, or null otherwise. More...
 
static int64_t ExtractImmediate (const SCEV *&S, ScalarEvolution &SE)
 If S involves the addition of a constant integer value, return that integer value, and mutate S to point to a new SCEV with that value excluded. More...
 
static GlobalValueExtractSymbol (const SCEV *&S, ScalarEvolution &SE)
 If S involves the addition of a GlobalValue address, return that symbol, and mutate S to point to a new SCEV with that value excluded. More...
 
static bool isAddressUse (Instruction *Inst, Value *OperandVal)
 Returns true if the specified instruction is using the specified value as an address. More...
 
static MemAccessTy getAccessType (const Instruction *Inst)
 Return the type of the memory being accessed. More...
 
static bool isExistingPhi (const SCEVAddRecExpr *AR, ScalarEvolution &SE)
 Return true if this AddRec is already a phi in its loop. More...
 
static bool isHighCostExpansion (const SCEV *S, SmallPtrSetImpl< const SCEV *> &Processed, ScalarEvolution &SE)
 Check if expanding this expression is likely to incur significant cost. More...
 
static bool DeleteTriviallyDeadInstructions (SmallVectorImpl< WeakTrackingVH > &DeadInsts)
 If any of the instructions is the specified set are trivially dead, delete them and see if this makes any of their operands subsequently dead. More...
 
static bool isAMCompletelyFolded (const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F)
 Check if the addressing mode defined by F is completely folded in LU at isel time. More...
 
static unsigned getScalingFactorCost (const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F, const Loop &L)
 
static bool isAMCompletelyFolded (const TargetTransformInfo &TTI, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, Instruction *Fixup=nullptr)
 
static bool isAMCompletelyFolded (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale)
 
static bool isAMCompletelyFolded (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, const Formula &F, const Loop &L)
 
static bool isLegalUse (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale)
 Test whether we know how to expand the current formula. More...
 
static bool isLegalUse (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, const Formula &F)
 
static bool isAlwaysFoldable (const TargetTransformInfo &TTI, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg)
 
static bool isAlwaysFoldable (const TargetTransformInfo &TTI, ScalarEvolution &SE, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, const SCEV *S, bool HasBaseReg)
 
static User::op_iterator findIVOperand (User::op_iterator OI, User::op_iterator OE, Loop *L, ScalarEvolution &SE)
 Helper for CollectChains that finds an IV operand (computed by an AddRec in this loop) within [OI,OE) or returns OE. More...
 
static ValuegetWideOperand (Value *Oper)
 IVChain logic must consistenctly peek base TruncInst operands, so wrap it in a convenient helper. More...
 
static bool isCompatibleIVType (Value *LVal, Value *RVal)
 Return true if we allow an IV chain to include both types. More...
 
static const SCEVgetExprBase (const SCEV *S)
 Return an approximation of this SCEV expression's "base", or NULL for any constant. More...
 
static bool isProfitableChain (IVChain &Chain, SmallPtrSetImpl< Instruction *> &Users, ScalarEvolution &SE, const TargetTransformInfo &TTI)
 Return true if the number of registers needed for the chain is estimated to be less than the number required for the individual IV users. More...
 
static bool canFoldIVIncExpr (const SCEV *IncExpr, Instruction *UserInst, Value *Operand, const TargetTransformInfo &TTI)
 Return true if the IVInc can be folded into an addressing mode. More...
 
static const SCEVCollectSubexprs (const SCEV *S, const SCEVConstant *C, SmallVectorImpl< const SCEV *> &Ops, const Loop *L, ScalarEvolution &SE, unsigned Depth=0)
 Split S into subexpressions which can be pulled out into separate registers. More...
 
static bool ReduceLoopStrength (Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI, const TargetTransformInfo &TTI)
 
 INITIALIZE_PASS_BEGIN (LoopStrengthReduce, "loop-reduce", "Loop Strength Reduction", false, false) INITIALIZE_PASS_END(LoopStrengthReduce
 

Variables

static const unsigned MaxIVUsers = 200
 MaxIVUsers is an arbitrary threshold that provides an early opportunitiy for bail out. More...
 
static cl::opt< boolEnablePhiElim ("enable-lsr-phielim", cl::Hidden, cl::init(true), cl::desc("Enable LSR phi elimination"))
 
static cl::opt< boolInsnsCost ("lsr-insns-cost", cl::Hidden, cl::init(true), cl::desc("Add instruction count to a LSR cost model"))
 
static cl::opt< boolLSRExpNarrow ("lsr-exp-narrow", cl::Hidden, cl::init(false), cl::desc("Narrow LSR complex solution using" " expectation of registers number"))
 
static cl::opt< boolFilterSameScaledReg ("lsr-filter-same-scaled-reg", cl::Hidden, cl::init(true), cl::desc("Narrow LSR search space by filtering non-optimal formulae" " with the same ScaledReg and Scale"))
 
static cl::opt< boolStressIVChain ("stress-ivchain", cl::Hidden, cl::init(false), cl::desc("Stress test LSR IV chains"))
 
static const size_t ComplexityLimit = std::numeric_limits<uint16_t>::max()
 
loop reduce
 
loop Loop Strength Reduction
 
loop Loop Strength false
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "loop-reduce"

Definition at line 122 of file LoopStrengthReduce.cpp.

Referenced by ReduceLoopStrength().

Function Documentation

◆ canFoldIVIncExpr()

static bool canFoldIVIncExpr ( const SCEV IncExpr,
Instruction UserInst,
Value Operand,
const TargetTransformInfo TTI 
)
static

Return true if the IVInc can be folded into an addressing mode.

Definition at line 3032 of file LoopStrengthReduce.cpp.

References llvm::dwarf::syntax::Address, assert(), llvm::codeview::Basic, C, llvm::SCEVExpander::clearPostInc(), llvm::LoopBase< BlockT, LoopT >::contains(), D, llvm::dbgs(), DEBUG, llvm::DominatorTree::dominates(), llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::SCEVExpander::expandCodeFor(), F(), llvm::find(), findIVOperand(), getAccessType(), llvm::ScalarEvolution::getAddExpr(), llvm::SCEVConstant::getAPInt(), llvm::Instruction::getDebugLoc(), llvm::ScalarEvolution::getEffectiveSCEVType(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::PHINode::getIncomingValueForBlock(), llvm::PHINode::getIncomingValueNumForOperand(), llvm::APInt::getMinSignedBits(), llvm::ScalarEvolution::getMinusSCEV(), llvm::ScalarEvolution::getNoopOrSignExtend(), llvm::User::getOperand(), llvm::Instruction::getParent(), llvm::BasicBlock::getParent(), llvm::ScalarEvolution::getSCEV(), llvm::ConstantInt::getSExtValue(), llvm::BasicBlock::getTerminator(), llvm::Value::getType(), llvm::ScalarEvolution::getTypeSizeInBits(), llvm::ScalarEvolution::getUnknown(), llvm::SCEVConstant::getValue(), getWideOperand(), llvm::ScalarEvolution::hasComputableLoopEvolution(), I, llvm::SmallVectorImpl< T >::insert(), isAddressUse(), isAlwaysFoldable(), isCompatibleIVType(), llvm::Instruction::isEHPad(), isLegalUse(), llvm::ScalarEvolution::isLoopInvariant(), llvm::Type::isPointerTy(), llvm::isSafeToExpand(), llvm::ScalarEvolution::isSCEVable(), llvm::SCEV::isZero(), N, llvm::normalizeForPostIncUse(), llvm::User::op_end(), llvm::User::operands(), P, llvm::User::replaceUsesOfWith(), Rewriter, llvm::IRBuilderBase::SetCurrentDebugLocation(), and llvm::Value::uses().

◆ CollectSubexprs()

static const SCEV* CollectSubexprs ( const SCEV S,
const SCEVConstant C,
SmallVectorImpl< const SCEV *> &  Ops,
const Loop L,
ScalarEvolution SE,
unsigned  Depth = 0 
)
static

Split S into subexpressions which can be pulled out into separate registers.

If C is non-null, multiply each subexpression by C.

Return remainder expression after factoring the subexpressions captured by Ops. If Ops is complete, return NULL.

Definition at line 3391 of file LoopStrengthReduce.cpp.

References llvm::abs(), llvm::MCID::Add, llvm::SystemZICMP::Any, llvm::SmallVectorImpl< T >::append(), llvm::array_lengthof(), assert(), llvm::codeview::Basic, llvm::sys::path::begin(), llvm::SmallVectorTemplateCommon< T >::begin(), C, llvm::SmallPtrSetImplBase::clear(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::clear(), llvm::SmallSet< T, N, C >::clear(), llvm::count(), llvm::countTrailingZeros(), llvm::dbgs(), DEBUG, llvm::Depth, dump(), llvm::dump(), llvm::dyn_cast(), llvm::sys::path::end(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::errs(), ExtractImmediate(), ExtractSymbol(), F(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::find(), first, llvm::SCEV::FlagAnyWrap, for(), G, llvm::ConstantInt::get(), llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getAnyExtendExpr(), llvm::ScalarEvolution::getConstant(), llvm::ScalarEvolution::getEffectiveSCEVType(), getExactSDiv(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::SCEVAddRecExpr::getLoop(), llvm::ScalarEvolution::getMulExpr(), llvm::ScalarEvolution::getSCEV(), llvm::SCEVConstant::getType(), llvm::SCEV::getType(), llvm::ScalarEvolution::getTypeSizeInBits(), llvm::SCEVConstant::getValue(), llvm::ConstantInt::getZExtValue(), llvm::ScalarEvolution::hasComputableLoopEvolution(), if(), llvm::SmallSet< T, N, C >::insert(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::insert(), isAlwaysFoldable(), llvm::TargetTransformInfo::isLegalAddImmediate(), isLegalUse(), llvm::ScalarEvolution::isLoopInvariant(), llvm::Type::isPointerTy(), llvm::TargetTransformInfo::isTruncateFree(), llvm::ConstantInt::isValueValidForType(), llvm::SCEV::isZero(), llvm::AArch64CC::LE, LLVM_DUMP_METHOD, N, llvm::AArch64CC::NE, P, llvm::ScalarEvolution::properlyDominates(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), Regs, llvm::PPCISD::SC, llvm::SmallBitVector::set_bits(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::parallel::sort(), and std::swap().

◆ DeleteTriviallyDeadInstructions()

static bool DeleteTriviallyDeadInstructions ( SmallVectorImpl< WeakTrackingVH > &  DeadInsts)
static

If any of the instructions is the specified set are trivially dead, delete them and see if this makes any of their operands subsequently dead.

Definition at line 921 of file LoopStrengthReduce.cpp.

References llvm::TargetTransformInfo::LSRCost::AddRecCost, llvm::dwarf::syntax::Address, assert(), llvm::SmallVectorTemplateCommon< T >::back(), llvm::codeview::Basic, llvm::SmallVectorTemplateCommon< T >::begin(), C, llvm::SmallPtrSetImplBase::clear(), llvm::LoopBase< BlockT, LoopT >::contains(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::count(), llvm::SmallPtrSetImpl< PtrType >::count(), dump(), llvm::dump(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorBase::empty(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::Instruction::eraseFromParent(), llvm::errs(), F(), Fixup, llvm::APInt::getMinSignedBits(), llvm::TargetTransformInfo::getNumberOfRegisters(), getScalingFactorCost(), llvm::ScalarEvolution::hasComputableLoopEvolution(), llvm::hash_combine_range(), if(), llvm::TargetTransformInfo::LSRCost::ImmCost, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetTransformInfo::LSRCost::Insns, InsnsCost, isAMCompletelyFolded(), isEqual(), isExistingPhi(), llvm::isInstructionTriviallyDead(), llvm::TargetTransformInfo::isLSRCostLess(), llvm::SCEV::isZero(), Kind, LLVM_DUMP_METHOD, llvm::max(), llvm::TargetTransformInfo::LSRCost::NumBaseAdds, llvm::TargetTransformInfo::LSRCost::NumIVMuls, llvm::TargetTransformInfo::LSRCost::NumRegs, llvm::RISCVFenceField::O, llvm::User::operands(), Other, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), Regs, llvm::TargetTransformInfo::LSRCost::ScaleCost, llvm::TargetTransformInfo::LSRCost::SetupCost, llvm::parallel::sort(), llvm::SPII::Store, and std::swap().

Referenced by ReduceLoopStrength().

◆ DoInitialMatch()

static void DoInitialMatch ( const SCEV S,
Loop L,
SmallVectorImpl< const SCEV *> &  Good,
SmallVectorImpl< const SCEV *> &  Bad,
ScalarEvolution SE 
)
static

◆ ExtractImmediate()

static int64_t ExtractImmediate ( const SCEV *&  S,
ScalarEvolution SE 
)
static

If S involves the addition of a constant integer value, return that integer value, and mutate S to point to a new SCEV with that value excluded.

Definition at line 728 of file LoopStrengthReduce.cpp.

References llvm::MCID::Add, C, llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getAddRecExpr(), llvm::ScalarEvolution::getConstant(), and llvm::Value::getType().

Referenced by CollectSubexprs(), and isAlwaysFoldable().

◆ ExtractSymbol()

static GlobalValue* ExtractSymbol ( const SCEV *&  S,
ScalarEvolution SE 
)
static

If S involves the addition of a GlobalValue address, return that symbol, and mutate S to point to a new SCEV with that value excluded.

Definition at line 754 of file LoopStrengthReduce.cpp.

References llvm::MCID::Add, llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getAddRecExpr(), and llvm::ScalarEvolution::getConstant().

Referenced by CollectSubexprs(), and isAlwaysFoldable().

◆ findIVOperand()

static User::op_iterator findIVOperand ( User::op_iterator  OI,
User::op_iterator  OE,
Loop L,
ScalarEvolution SE 
)
static

Helper for CollectChains that finds an IV operand (computed by an AddRec in this loop) within [OI,OE) or returns OE.

If IVUsers mapped Instructions to IVStrideUses, we could partially skip this.

Definition at line 2630 of file LoopStrengthReduce.cpp.

References llvm::ScalarEvolution::getSCEV(), and llvm::ScalarEvolution::isSCEVable().

Referenced by canFoldIVIncExpr(), and isProfitableChain().

◆ getAccessType()

static MemAccessTy getAccessType ( const Instruction Inst)
static

Return the type of the memory being accessed.

Definition at line 813 of file LoopStrengthReduce.cpp.

References llvm::IntegerType::get(), llvm::PointerType::get(), llvm::Value::getType(), and SI.

Referenced by canFoldIVIncExpr(), and isAlwaysFoldable().

◆ getExactSDiv()

static const SCEV* getExactSDiv ( const SCEV LHS,
const SCEV RHS,
ScalarEvolution SE,
bool  IgnoreSignificantBits = false 
)
static

Return an expression for LHS /s RHS, if it can be determined and if the remainder is known to be zero, or null otherwise.

If IgnoreSignificantBits is true, expressions like (X * Y) /s Y are simplified to Y, ignoring that the multiplication may overflow, which is useful when the result will be used in a context where the most significant bits are ignored.

Definition at line 641 of file LoopStrengthReduce.cpp.

References llvm::MCID::Add, C, llvm::dyn_cast(), llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddExpr(), llvm::SCEVConstant::getAPInt(), llvm::ScalarEvolution::getConstant(), llvm::ScalarEvolution::getMulExpr(), llvm::SCEV::getType(), isAddRecSExtable(), isAddSExtable(), llvm::APInt::isAllOnesValue(), isMulSExtable(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), RA, llvm::APInt::sdiv(), and llvm::APInt::srem().

Referenced by CollectSubexprs(), and isAlwaysFoldable().

◆ getExprBase()

static const SCEV* getExprBase ( const SCEV S)
static

Return an approximation of this SCEV expression's "base", or NULL for any constant.

Returning the expression itself is conservative. Returning a deeper subexpression is more precise and valid as long as it isn't less complex than another subexpression. For expressions involving multiple unscaled values, we need to return the pointer-type SCEVUnknown. This avoids forming chains across objects, such as: PrevOper==a[i], IVOper==b[i], IVInc==b-a.

Since SCEVUnknown is the rightmost type, and pointers are the rightmost SCEVUnknown, we simply return the rightmost SCEV operand.

Definition at line 2677 of file LoopStrengthReduce.cpp.

References llvm::MCID::Add, E, llvm::ScalarEvolution::getMinusSCEV(), llvm::ScalarEvolution::getSCEV(), llvm::SCEV::getSCEVType(), getWideOperand(), I, isHighCostExpansion(), llvm::SCEVNAryExpr::op_begin(), llvm::SCEVNAryExpr::op_end(), llvm::scAddExpr, llvm::scAddRecExpr, llvm::scConstant, llvm::scMulExpr, llvm::scSignExtend, llvm::scTruncate, and llvm::scZeroExtend.

Referenced by isProfitableChain().

◆ getScalingFactorCost()

static unsigned getScalingFactorCost ( const TargetTransformInfo TTI,
const LSRUse &  LU,
const Formula &  F,
const Loop L 
)
static

◆ getWideOperand()

static Value* getWideOperand ( Value Oper)
static

IVChain logic must consistenctly peek base TruncInst operands, so wrap it in a convenient helper.

Definition at line 2649 of file LoopStrengthReduce.cpp.

Referenced by canFoldIVIncExpr(), getExprBase(), and isProfitableChain().

◆ INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN ( LoopStrengthReduce  ,
"loop-reduce ,
"Loop Strength Reduction ,
false  ,
false   
)

◆ isAddRecSExtable()

static bool isAddRecSExtable ( const SCEVAddRecExpr AR,
ScalarEvolution SE 
)
static

Return true if the given addrec can be sign-extended without changing its value.

Definition at line 613 of file LoopStrengthReduce.cpp.

References llvm::IntegerType::get(), llvm::ScalarEvolution::getContext(), llvm::ScalarEvolution::getSignExtendExpr(), llvm::SCEVNAryExpr::getType(), and llvm::ScalarEvolution::getTypeSizeInBits().

Referenced by getExactSDiv().

◆ isAddressUse()

static bool isAddressUse ( Instruction Inst,
Value OperandVal 
)
static

Returns true if the specified instruction is using the specified value as an address.

Definition at line 780 of file LoopStrengthReduce.cpp.

References prefetch, and SI.

Referenced by canFoldIVIncExpr().

◆ isAddSExtable()

static bool isAddSExtable ( const SCEVAddExpr A,
ScalarEvolution SE 
)
static

Return true if the given add can be sign-extended without changing its value.

Definition at line 621 of file LoopStrengthReduce.cpp.

References llvm::IntegerType::get(), llvm::ScalarEvolution::getContext(), llvm::ScalarEvolution::getSignExtendExpr(), llvm::SCEVAddExpr::getType(), and llvm::ScalarEvolution::getTypeSizeInBits().

Referenced by getExactSDiv().

◆ isAlwaysFoldable() [1/2]

static bool isAlwaysFoldable ( const TargetTransformInfo TTI,
LSRUse::KindType  Kind,
MemAccessTy  AccessTy,
GlobalValue BaseGV,
int64_t  BaseOffset,
bool  HasBaseReg 
)
static

Definition at line 1714 of file LoopStrengthReduce.cpp.

References isAMCompletelyFolded().

Referenced by canFoldIVIncExpr(), CollectSubexprs(), and isAlwaysFoldable().

◆ isAlwaysFoldable() [2/2]

static bool isAlwaysFoldable ( const TargetTransformInfo TTI,
ScalarEvolution SE,
int64_t  MinOffset,
int64_t  MaxOffset,
LSRUse::KindType  Kind,
MemAccessTy  AccessTy,
const SCEV S,
bool  HasBaseReg 
)
static

Definition at line 1736 of file LoopStrengthReduce.cpp.

References llvm::ARM_AM::add, llvm::MCID::Add, llvm::dwarf::syntax::Address, llvm::all_of(), llvm::SmallVectorImpl< T >::append(), assert(), B, llvm::SmallVectorTemplateCommon< T >::back(), llvm::CmpInst::BAD_ICMP_PREDICATE, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::begin(), llvm::sys::path::begin(), llvm::SmallVectorTemplateCommon< T >::begin(), C, llvm::Instruction::clone(), llvm::BinaryOperator::Create(), llvm::PHINode::Create(), D, llvm::dbgs(), DEBUG, llvm::Depth, llvm::DominatorTree::dominates(), llvm::dump(), llvm::dyn_cast(), E, llvm::SmallVectorBase::empty(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::end(), llvm::sys::path::end(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::Instruction::eraseFromParent(), ExtractImmediate(), ExtractSymbol(), llvm::DominatorTreeBase< NodeT, IsPostDom >::findNearestCommonDominator(), llvm::ConstantFP::get(), getAccessType(), llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getBackedgeTakenCount(), llvm::BranchInst::getCondition(), llvm::ScalarEvolution::getConstant(), llvm::ScalarEvolution::getEffectiveSCEVType(), getExactSDiv(), llvm::LoopBase< BlockT, LoopT >::getExitingBlocks(), llvm::Type::getFPMantissaWidth(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::PHINode::getIncomingBlock(), llvm::PHINode::getIncomingValue(), llvm::CmpInst::getInversePredicate(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::SCEVAddRecExpr::getLoop(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::LoopBase< BlockT, LoopT >::getLoopPreheader(), llvm::APInt::getMinSignedBits(), llvm::Value::getName(), llvm::PHINode::getNumIncomingValues(), llvm::SCEVNAryExpr::getNumOperands(), llvm::SCEVNAryExpr::getOperand(), llvm::User::getOperand(), llvm::IVStrideUse::getOperandValToReplace(), llvm::BasicBlock::getParent(), llvm::CmpInst::getPredicate(), llvm::ScalarEvolution::getSCEV(), llvm::ConstantInt::getSExtValue(), llvm::ScalarEvolution::getSignExtendExpr(), llvm::SCEVAddRecExpr::getStart(), llvm::SCEVAddRecExpr::getStepRecurrence(), llvm::BasicBlock::getTerminator(), llvm::SCEV::getType(), llvm::Value::getType(), llvm::ScalarEvolution::getTypeSizeInBits(), llvm::ConstantInt::getValue(), llvm::ConstantInt::getZExtValue(), llvm::SCEVNAryExpr::hasNoSignedWrap(), llvm::SCEVNAryExpr::hasNoUnsignedWrap(), llvm::Value::hasOneUse(), I, llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_ULT, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::SCEVAddRecExpr::isAffine(), isAlwaysFoldable(), isAMCompletelyFolded(), llvm::TargetTransformInfo::isLegalAddressingMode(), llvm::APInt::isMinSignedValue(), llvm::ConstantInt::isMinusOne(), llvm::ConstantInt::isOne(), llvm::APInt::isStrictlyPositive(), llvm::CmpInst::isTrueWhenEqual(), llvm::TargetTransformInfo::isTypeLegal(), llvm::BranchInst::isUnconditional(), llvm::SCEV::isZero(), llvm::Instruction::moveBefore(), llvm::RISCVFenceField::O, P, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::DominatorTreeBase< NodeT, IsPostDom >::properlyDominates(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::Value::replaceAllUsesWith(), llvm::User::replaceUsesOfWith(), Rewriter, llvm::Value::setName(), llvm::IVStrideUse::setUser(), llvm::SmallVectorTemplateCommon< T >::size(), std::swap(), llvm::IVStrideUse::transformToPostInc(), llvm::Value::use_empty(), and X.

◆ isAMCompletelyFolded() [1/4]

static bool isAMCompletelyFolded ( const TargetTransformInfo TTI,
const LSRUse &  LU,
const Formula &  F 
)
static

Check if the addressing mode defined by F is completely folded in LU at isel time.

This includes address-mode folding and special icmp tricks. This function returns true if LU can accommodate what F defines and up to 1 base + 1 scaled + offset. In other words, if F has several base registers, this function may still return true. Therefore, users still need to account for additional base registers and/or unfolded offsets to derive an accurate cost model.

Definition at line 1660 of file LoopStrengthReduce.cpp.

References llvm::dwarf::syntax::Address, Fixup, and llvm::TargetTransformInfo::LSRWithInstrQueries().

Referenced by DeleteTriviallyDeadInstructions(), getScalingFactorCost(), isAlwaysFoldable(), isAMCompletelyFolded(), and isLegalUse().

◆ isAMCompletelyFolded() [2/4]

static bool isAMCompletelyFolded ( const TargetTransformInfo TTI,
LSRUse::KindType  Kind,
MemAccessTy  AccessTy,
GlobalValue BaseGV,
int64_t  BaseOffset,
bool  HasBaseReg,
int64_t  Scale,
Instruction Fixup = nullptr 
)
static

◆ isAMCompletelyFolded() [3/4]

static bool isAMCompletelyFolded ( const TargetTransformInfo TTI,
int64_t  MinOffset,
int64_t  MaxOffset,
LSRUse::KindType  Kind,
MemAccessTy  AccessTy,
GlobalValue BaseGV,
int64_t  BaseOffset,
bool  HasBaseReg,
int64_t  Scale 
)
static

Definition at line 1601 of file LoopStrengthReduce.cpp.

References isAMCompletelyFolded().

◆ isAMCompletelyFolded() [4/4]

static bool isAMCompletelyFolded ( const TargetTransformInfo TTI,
int64_t  MinOffset,
int64_t  MaxOffset,
LSRUse::KindType  Kind,
MemAccessTy  AccessTy,
const Formula &  F,
const Loop L 
)
static

Definition at line 1622 of file LoopStrengthReduce.cpp.

References assert(), and isAMCompletelyFolded().

◆ isCompatibleIVType()

static bool isCompatibleIVType ( Value LVal,
Value RVal 
)
static

Return true if we allow an IV chain to include both types.

Definition at line 2656 of file LoopStrengthReduce.cpp.

References llvm::Type::getPointerAddressSpace(), llvm::Value::getType(), and llvm::Type::isPointerTy().

Referenced by canFoldIVIncExpr(), and isProfitableChain().

◆ isExistingPhi()

static bool isExistingPhi ( const SCEVAddRecExpr AR,
ScalarEvolution SE 
)
static

◆ isHighCostExpansion()

static bool isHighCostExpansion ( const SCEV S,
SmallPtrSetImpl< const SCEV *> &  Processed,
ScalarEvolution SE 
)
static

Check if expanding this expression is likely to incur significant cost.

This is tricky because SCEV doesn't track which expressions are actually computed by the current IR.

We currently allow expansion of IV increments that involve adds, multiplication by constants, and AddRecs from existing phis.

TODO: Allow UDivExpr if we can find an existing IV increment that is an obvious multiple of the UDivExpr.

Definition at line 857 of file LoopStrengthReduce.cpp.

References llvm::MCID::Add, llvm::dyn_cast(), llvm::Instruction::getOpcode(), llvm::ScalarEvolution::getSCEV(), llvm::SCEV::getSCEVType(), llvm::Value::getType(), llvm::SmallPtrSetImpl< PtrType >::insert(), isExistingPhi(), llvm::ScalarEvolution::isSCEVable(), llvm::scConstant, llvm::scSignExtend, llvm::scTruncate, llvm::scUnknown, llvm::scZeroExtend, and llvm::Value::users().

Referenced by getExprBase().

◆ isLegalUse() [1/2]

static bool isLegalUse ( const TargetTransformInfo TTI,
int64_t  MinOffset,
int64_t  MaxOffset,
LSRUse::KindType  Kind,
MemAccessTy  AccessTy,
GlobalValue BaseGV,
int64_t  BaseOffset,
bool  HasBaseReg,
int64_t  Scale 
)
static

Test whether we know how to expand the current formula.

Definition at line 1639 of file LoopStrengthReduce.cpp.

References isAMCompletelyFolded().

Referenced by canFoldIVIncExpr(), CollectSubexprs(), and isLegalUse().

◆ isLegalUse() [2/2]

static bool isLegalUse ( const TargetTransformInfo TTI,
int64_t  MinOffset,
int64_t  MaxOffset,
LSRUse::KindType  Kind,
MemAccessTy  AccessTy,
const Formula &  F 
)
static

Definition at line 1653 of file LoopStrengthReduce.cpp.

References isLegalUse().

◆ isMulSExtable()

static bool isMulSExtable ( const SCEVMulExpr M,
ScalarEvolution SE 
)
static

◆ isProfitableChain()

static bool isProfitableChain ( IVChain &  Chain,
SmallPtrSetImpl< Instruction *> &  Users,
ScalarEvolution SE,
const TargetTransformInfo TTI 
)
static

Return true if the number of registers needed for the chain is estimated to be less than the number required for the individual IV users.

First prohibit any IV users that keep the IV live across increments (the Users set should be empty). Next count the number and type of increments in the chain.

Chaining IVs can lead to considerable code bloat if ISEL doesn't effectively use postinc addressing modes. Only consider it profitable it the increments can be computed in fewer registers when chained.

TODO: Consider IVInc free if it's already used in another chains.

Definition at line 2745 of file LoopStrengthReduce.cpp.

References assert(), llvm::BasicBlock::begin(), llvm::SmallPtrSetImpl< PtrType >::begin(), llvm::SmallPtrSetImplBase::clear(), llvm::dbgs(), DEBUG, llvm::dyn_cast(), llvm::SmallPtrSetImplBase::empty(), llvm::SmallPtrSetImpl< PtrType >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::find(), findIVOperand(), for(), llvm::DomTreeNodeBase< NodeT >::getBlock(), getExprBase(), llvm::LoopBase< BlockT, LoopT >::getHeader(), llvm::PHINode::getIncomingValueForBlock(), llvm::LoopBase< BlockT, LoopT >::getLoopLatch(), llvm::ScalarEvolution::getMinusSCEV(), llvm::DominatorTreeBase< NodeT, IsPostDom >::getNode(), llvm::ScalarEvolution::getSCEV(), llvm::Value::getType(), getWideOperand(), I, llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::SmallVectorImpl< T >::insert(), isCompatibleIVType(), llvm::ScalarEvolution::isLoopInvariant(), llvm::ScalarEvolution::isSCEVable(), llvm::SCEV::isZero(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::reverse(), and llvm::Value::users().

◆ ReduceLoopStrength()

static bool ReduceLoopStrength ( Loop L,
IVUsers IU,
ScalarEvolution SE,
DominatorTree DT,
LoopInfo LI,
const TargetTransformInfo TTI 
)
static

Variable Documentation

◆ ComplexityLimit

const size_t ComplexityLimit = std::numeric_limits<uint16_t>::max()
static

Definition at line 4188 of file LoopStrengthReduce.cpp.

◆ EnablePhiElim

cl::opt<bool> EnablePhiElim("enable-lsr-phielim", cl::Hidden, cl::init(true), cl::desc("Enable LSR phi elimination"))
static

Referenced by ReduceLoopStrength().

◆ false

loop Loop Strength false

Definition at line 5495 of file LoopStrengthReduce.cpp.

◆ FilterSameScaledReg

cl::opt<bool> FilterSameScaledReg("lsr-filter-same-scaled-reg", cl::Hidden, cl::init(true), cl::desc("Narrow LSR search space by filtering non-optimal formulae" " with the same ScaledReg and Scale"))
static

◆ InsnsCost

cl::opt<bool> InsnsCost("lsr-insns-cost", cl::Hidden, cl::init(true), cl::desc("Add instruction count to a LSR cost model"))
static

◆ LSRExpNarrow

cl::opt<bool> LSRExpNarrow("lsr-exp-narrow", cl::Hidden, cl::init(false), cl::desc("Narrow LSR complex solution using" " expectation of registers number"))
static

◆ MaxIVUsers

const unsigned MaxIVUsers = 200
static

MaxIVUsers is an arbitrary threshold that provides an early opportunitiy for bail out.

This threshold is far beyond the number of users that LSR can conceivably solve, so it should not affect generated code, but catches the worst cases before LSR burns too much compile time and stack space.

Definition at line 128 of file LoopStrengthReduce.cpp.

◆ reduce

loop reduce

◆ Reduction

loop Loop Strength Reduction

Definition at line 5495 of file LoopStrengthReduce.cpp.

Referenced by llvm::createLoopVectorizePass().

◆ StressIVChain

cl::opt<bool> StressIVChain("stress-ivchain", cl::Hidden, cl::init(false), cl::desc("Stress test LSR IV chains"))
static