|
LLVM
4.0.0
|
#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"#include "llvm/ADT/APInt.h"#include "llvm/ADT/DenseMap.h"#include "llvm/ADT/DenseSet.h"#include "llvm/ADT/Hashing.h"#include "llvm/ADT/PointerIntPair.h"#include "llvm/ADT/STLExtras.h"#include "llvm/ADT/SetVector.h"#include "llvm/ADT/SmallBitVector.h"#include "llvm/ADT/SmallPtrSet.h"#include "llvm/ADT/SmallSet.h"#include "llvm/ADT/SmallVector.h"#include "llvm/Analysis/IVUsers.h"#include "llvm/Analysis/LoopInfo.h"#include "llvm/Analysis/LoopPass.h"#include "llvm/Analysis/ScalarEvolution.h"#include "llvm/Analysis/ScalarEvolutionExpander.h"#include "llvm/Analysis/ScalarEvolutionExpressions.h"#include "llvm/Analysis/ScalarEvolutionNormalization.h"#include "llvm/Analysis/TargetTransformInfo.h"#include "llvm/IR/BasicBlock.h"#include "llvm/IR/Constant.h"#include "llvm/IR/Constants.h"#include "llvm/IR/DerivedTypes.h"#include "llvm/IR/Dominators.h"#include "llvm/IR/GlobalValue.h"#include "llvm/IR/IRBuilder.h"#include "llvm/IR/Instruction.h"#include "llvm/IR/Instructions.h"#include "llvm/IR/IntrinsicInst.h"#include "llvm/IR/Module.h"#include "llvm/IR/OperandTraits.h"#include "llvm/IR/Operator.h"#include "llvm/IR/Type.h"#include "llvm/IR/Value.h"#include "llvm/IR/ValueHandle.h"#include "llvm/Pass.h"#include "llvm/Support/Casting.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Compiler.h"#include "llvm/Support/Debug.h"#include "llvm/Support/ErrorHandling.h"#include "llvm/Support/MathExtras.h"#include "llvm/Support/raw_ostream.h"#include "llvm/Transforms/Scalar.h"#include "llvm/Transforms/Scalar/LoopPassManager.h"#include "llvm/Transforms/Utils/BasicBlockUtils.h"#include "llvm/Transforms/Utils/Local.h"#include <algorithm>#include <cassert>#include <cstddef>#include <cstdint>#include <cstdlib>#include <iterator>#include <map>#include <tuple>#include <utility>Go to the source code of this file.
Macros | |
| #define | DEBUG_TYPE "loop-reduce" |
Functions | |
| static void | DoInitialMatch (const SCEV *S, Loop *L, SmallVectorImpl< const SCEV * > &Good, SmallVectorImpl< const SCEV * > &Bad, ScalarEvolution &SE) |
| Recursion helper for initialMatch. More... | |
| static bool | isAddRecSExtable (const SCEVAddRecExpr *AR, ScalarEvolution &SE) |
| Return true if the given addrec can be sign-extended without changing its value. More... | |
| static bool | isAddSExtable (const SCEVAddExpr *A, ScalarEvolution &SE) |
| Return true if the given add can be sign-extended without changing its value. More... | |
| static bool | isMulSExtable (const SCEVMulExpr *M, ScalarEvolution &SE) |
| Return true if the given mul can be sign-extended without changing its value. More... | |
| static const SCEV * | getExactSDiv (const SCEV *LHS, const SCEV *RHS, ScalarEvolution &SE, bool IgnoreSignificantBits=false) |
| Return an expression for LHS /s RHS, if it can be determined and if the remainder is known to be zero, or null otherwise. More... | |
| static int64_t | ExtractImmediate (const SCEV *&S, ScalarEvolution &SE) |
| If S involves the addition of a constant integer value, return that integer value, and mutate S to point to a new SCEV with that value excluded. More... | |
| static GlobalValue * | ExtractSymbol (const SCEV *&S, ScalarEvolution &SE) |
| If S involves the addition of a GlobalValue address, return that symbol, and mutate S to point to a new SCEV with that value excluded. More... | |
| static bool | isAddressUse (Instruction *Inst, Value *OperandVal) |
| Returns true if the specified instruction is using the specified value as an address. More... | |
| static MemAccessTy | getAccessType (const Instruction *Inst) |
| Return the type of the memory being accessed. More... | |
| static bool | isExistingPhi (const SCEVAddRecExpr *AR, ScalarEvolution &SE) |
| Return true if this AddRec is already a phi in its loop. More... | |
| static bool | isHighCostExpansion (const SCEV *S, SmallPtrSetImpl< const SCEV * > &Processed, ScalarEvolution &SE) |
| Check if expanding this expression is likely to incur significant cost. More... | |
| static bool | DeleteTriviallyDeadInstructions (SmallVectorImpl< WeakVH > &DeadInsts) |
| If any of the instructions is the specified set are trivially dead, delete them and see if this makes any of their operands subsequently dead. More... | |
| static bool | isAMCompletelyFolded (const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F) |
Check if the addressing mode defined by F is completely folded in LU at isel time. More... | |
| static unsigned | getScalingFactorCost (const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F) |
| static bool | isAMCompletelyFolded (const TargetTransformInfo &TTI, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale) |
| static bool | isAMCompletelyFolded (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale) |
| static bool | isAMCompletelyFolded (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, const Formula &F) |
| static bool | isLegalUse (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale) |
| Test whether we know how to expand the current formula. More... | |
| static bool | isLegalUse (const TargetTransformInfo &TTI, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, const Formula &F) |
| static bool | isAlwaysFoldable (const TargetTransformInfo &TTI, LSRUse::KindType Kind, MemAccessTy AccessTy, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg) |
| static bool | isAlwaysFoldable (const TargetTransformInfo &TTI, ScalarEvolution &SE, int64_t MinOffset, int64_t MaxOffset, LSRUse::KindType Kind, MemAccessTy AccessTy, const SCEV *S, bool HasBaseReg) |
| static User::op_iterator | findIVOperand (User::op_iterator OI, User::op_iterator OE, Loop *L, ScalarEvolution &SE) |
| Helper for CollectChains that finds an IV operand (computed by an AddRec in this loop) within [OI,OE) or returns OE. More... | |
| static Value * | getWideOperand (Value *Oper) |
| IVChain logic must consistenctly peek base TruncInst operands, so wrap it in a convenient helper. More... | |
| static bool | isCompatibleIVType (Value *LVal, Value *RVal) |
| Return true if we allow an IV chain to include both types. More... | |
| static const SCEV * | getExprBase (const SCEV *S) |
| Return an approximation of this SCEV expression's "base", or NULL for any constant. More... | |
| static bool | isProfitableChain (IVChain &Chain, SmallPtrSetImpl< Instruction * > &Users, ScalarEvolution &SE, const TargetTransformInfo &TTI) |
| Return true if the number of registers needed for the chain is estimated to be less than the number required for the individual IV users. More... | |
| static bool | canFoldIVIncExpr (const SCEV *IncExpr, Instruction *UserInst, Value *Operand, const TargetTransformInfo &TTI) |
| Return true if the IVInc can be folded into an addressing mode. More... | |
| static const SCEV * | CollectSubexprs (const SCEV *S, const SCEVConstant *C, SmallVectorImpl< const SCEV * > &Ops, const Loop *L, ScalarEvolution &SE, unsigned Depth=0) |
| Split S into subexpressions which can be pulled out into separate registers. More... | |
| static bool | ReduceLoopStrength (Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI, const TargetTransformInfo &TTI) |
| INITIALIZE_PASS_BEGIN (LoopStrengthReduce,"loop-reduce","Loop Strength Reduction", false, false) INITIALIZE_PASS_END(LoopStrengthReduce | |
Variables | |
| static const unsigned | MaxIVUsers = 200 |
| MaxIVUsers is an arbitrary threshold that provides an early opportunitiy for bail out. More... | |
| static cl::opt< bool > | EnablePhiElim ("enable-lsr-phielim", cl::Hidden, cl::init(true), cl::desc("Enable LSR phi elimination")) |
| static cl::opt< bool > | StressIVChain ("stress-ivchain", cl::Hidden, cl::init(false), cl::desc("Stress test LSR IV chains")) |
| static const size_t | ComplexityLimit = UINT16_MAX |
| loop | reduce |
| loop Loop Strength | Reduction |
| loop Loop Strength | false |
| #define DEBUG_TYPE "loop-reduce" |
Definition at line 116 of file LoopStrengthReduce.cpp.
Referenced by ReduceLoopStrength().
|
static |
Return true if the IVInc can be folded into an addressing mode.
Definition at line 2863 of file LoopStrengthReduce.cpp.
References llvm::dwarf::syntax::Address, llvm::dyn_cast(), getAccessType(), llvm::SCEVConstant::getAPInt(), llvm::APInt::getMinSignedBits(), llvm::ConstantInt::getSExtValue(), llvm::SCEVConstant::getValue(), isAddressUse(), and isAlwaysFoldable().
|
static |
Split S into subexpressions which can be pulled out into separate registers.
If C is non-null, multiply each subexpression by C.
Return remainder expression after factoring the subexpressions captured by Ops. If Ops is complete, return NULL.
Definition at line 3220 of file LoopStrengthReduce.cpp.
References llvm::MCID::Add, C, llvm::Depth, llvm::SCEVAddRecExpr::getLoop(), llvm::ScalarEvolution::getMulExpr(), llvm::SCEVAddRecExpr::getStart(), llvm::SCEVAddRecExpr::getStepRecurrence(), llvm::SCEVNAryExpr::getType(), llvm::SCEVAddRecExpr::isAffine(), llvm::SCEV::isZero(), L, and llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back().
|
static |
If any of the instructions is the specified set are trivially dead, delete them and see if this makes any of their operands subsequently dead.
Definition at line 835 of file LoopStrengthReduce.cpp.
References llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorBase::empty(), llvm::Instruction::eraseFromParent(), llvm::isInstructionTriviallyDead(), llvm::User::operands(), and llvm::SmallVectorImpl< T >::pop_back_val().
Referenced by ReduceLoopStrength().
|
static |
Recursion helper for initialMatch.
Definition at line 343 of file LoopStrengthReduce.cpp.
References llvm::MCID::Add, llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddRecExpr(), llvm::Constant::getAllOnesValue(), llvm::ScalarEvolution::getConstant(), llvm::ScalarEvolution::getEffectiveSCEVType(), llvm::LoopBase< N, M >::getHeader(), llvm::ScalarEvolution::getMulExpr(), llvm::ScalarEvolution::getSCEV(), L, llvm::ScalarEvolution::properlyDominates(), and llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back().
|
static |
If S involves the addition of a constant integer value, return that integer value, and mutate S to point to a new SCEV with that value excluded.
Definition at line 659 of file LoopStrengthReduce.cpp.
References llvm::MCID::Add, C, llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getAddRecExpr(), and llvm::ScalarEvolution::getConstant().
Referenced by isAlwaysFoldable().
|
static |
If S involves the addition of a GlobalValue address, return that symbol, and mutate S to point to a new SCEV with that value excluded.
Definition at line 685 of file LoopStrengthReduce.cpp.
References llvm::MCID::Add, llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddExpr(), llvm::ScalarEvolution::getAddRecExpr(), and llvm::ScalarEvolution::getConstant().
Referenced by isAlwaysFoldable().
|
static |
Helper for CollectChains that finds an IV operand (computed by an AddRec in this loop) within [OI,OE) or returns OE.
If IVUsers mapped Instructions to IVStrideUses, we could partially skip this.
Definition at line 2466 of file LoopStrengthReduce.cpp.
References llvm::SCEVAddRecExpr::getLoop(), llvm::ScalarEvolution::getSCEV(), llvm::ScalarEvolution::isSCEVable(), and L.
|
static |
Return the type of the memory being accessed.
Definition at line 731 of file LoopStrengthReduce.cpp.
References llvm::IntegerType::get(), llvm::PointerType::get(), llvm::Value::getType(), and SI.
Referenced by canFoldIVIncExpr().
|
static |
Return an expression for LHS /s RHS, if it can be determined and if the remainder is known to be zero, or null otherwise.
If IgnoreSignificantBits is true, expressions like (X * Y) /s Y are simplified to Y, ignoring that the multiplication may overflow, which is useful when the result will be used in a context where the most significant bits are ignored.
Definition at line 572 of file LoopStrengthReduce.cpp.
References llvm::MCID::Add, C, llvm::dyn_cast(), llvm::SCEV::FlagAnyWrap, llvm::ScalarEvolution::getAddExpr(), llvm::SCEVConstant::getAPInt(), llvm::ScalarEvolution::getConstant(), llvm::ScalarEvolution::getMulExpr(), llvm::SCEV::getType(), isAddRecSExtable(), isAddSExtable(), llvm::APInt::isAllOnesValue(), isMulSExtable(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::APInt::sdiv(), and llvm::APInt::srem().
Return an approximation of this SCEV expression's "base", or NULL for any constant.
Returning the expression itself is conservative. Returning a deeper subexpression is more precise and valid as long as it isn't less complex than another subexpression. For expressions involving multiple unscaled values, we need to return the pointer-type SCEVUnknown. This avoids forming chains across objects, such as: PrevOper==a[i], IVOper==b[i], IVInc==b-a.
Since SCEVUnknown is the rightmost type, and pointers are the rightmost SCEVUnknown, we simply return the rightmost SCEV operand.
Definition at line 2508 of file LoopStrengthReduce.cpp.
References llvm::MCID::Add, E, llvm::SCEV::getSCEVType(), I, llvm::SCEVNAryExpr::op_begin(), llvm::SCEVNAryExpr::op_end(), llvm::scAddExpr, llvm::scAddRecExpr, llvm::scConstant, llvm::scMulExpr, llvm::scSignExtend, llvm::scTruncate, and llvm::scZeroExtend.
|
static |
Definition at line 1517 of file LoopStrengthReduce.cpp.
References llvm::dwarf::syntax::Address, assert(), llvm::codeview::Basic, llvm::TargetTransformInfo::getScalingFactorCost(), isAMCompletelyFolded(), and llvm_unreachable.
IVChain logic must consistenctly peek base TruncInst operands, so wrap it in a convenient helper.
Definition at line 2485 of file LoopStrengthReduce.cpp.
| INITIALIZE_PASS_BEGIN | ( | LoopStrengthReduce | , |
| "loop-reduce" | , | ||
| "Loop Strength Reduction" | , | ||
| false | , | ||
| false | |||
| ) |
|
static |
Return true if the given addrec can be sign-extended without changing its value.
Definition at line 544 of file LoopStrengthReduce.cpp.
References llvm::IntegerType::get(), llvm::ScalarEvolution::getContext(), llvm::ScalarEvolution::getSignExtendExpr(), llvm::SCEVNAryExpr::getType(), and llvm::ScalarEvolution::getTypeSizeInBits().
Referenced by getExactSDiv().
|
static |
Returns true if the specified instruction is using the specified value as an address.
Definition at line 711 of file LoopStrengthReduce.cpp.
Referenced by canFoldIVIncExpr().
|
static |
Return true if the given add can be sign-extended without changing its value.
Definition at line 552 of file LoopStrengthReduce.cpp.
References llvm::IntegerType::get(), llvm::ScalarEvolution::getContext(), llvm::ScalarEvolution::getSignExtendExpr(), llvm::SCEVAddExpr::getType(), and llvm::ScalarEvolution::getTypeSizeInBits().
Referenced by getExactSDiv().
|
static |
Definition at line 1553 of file LoopStrengthReduce.cpp.
References isAMCompletelyFolded().
Referenced by canFoldIVIncExpr().
|
static |
Definition at line 1575 of file LoopStrengthReduce.cpp.
References ExtractImmediate(), ExtractSymbol(), isAMCompletelyFolded(), and llvm::SCEV::isZero().
|
static |
Check if the addressing mode defined by F is completely folded in LU at isel time.
This includes address-mode folding and special icmp tricks. This function returns true if LU can accommodate what F defines and up to 1 base + 1 scaled + offset. In other words, if F has several base registers, this function may still return true. Therefore, users still need to account for additional base registers and/or unfolded offsets to derive an accurate cost model.
Definition at line 1510 of file LoopStrengthReduce.cpp.
Referenced by getScalingFactorCost(), isAlwaysFoldable(), isAMCompletelyFolded(), and isLegalUse().
|
static |
Definition at line 1399 of file LoopStrengthReduce.cpp.
References llvm::dwarf::syntax::Address, llvm::codeview::Basic, llvm::TargetTransformInfo::isLegalAddressingMode(), llvm::TargetTransformInfo::isLegalICmpImmediate(), and llvm_unreachable.
|
static |
Definition at line 1451 of file LoopStrengthReduce.cpp.
References isAMCompletelyFolded().
|
static |
Definition at line 1472 of file LoopStrengthReduce.cpp.
References assert(), and isAMCompletelyFolded().
Return true if we allow an IV chain to include both types.
Definition at line 2492 of file LoopStrengthReduce.cpp.
References llvm::Value::getType(), and llvm::Type::isPointerTy().
|
static |
Return true if this AddRec is already a phi in its loop.
Definition at line 750 of file LoopStrengthReduce.cpp.
References llvm::dyn_cast(), llvm::ScalarEvolution::getEffectiveSCEVType(), llvm::LoopBase< N, M >::getHeader(), llvm::SCEVAddRecExpr::getLoop(), llvm::ScalarEvolution::getSCEV(), llvm::SCEVNAryExpr::getType(), llvm::Value::getType(), I, and llvm::ScalarEvolution::isSCEVable().
Referenced by isHighCostExpansion().
|
static |
Check if expanding this expression is likely to incur significant cost.
This is tricky because SCEV doesn't track which expressions are actually computed by the current IR.
We currently allow expansion of IV increments that involve adds, multiplication by constants, and AddRecs from existing phis.
TODO: Allow UDivExpr if we can find an existing IV increment that is an obvious multiple of the UDivExpr.
Definition at line 771 of file LoopStrengthReduce.cpp.
References llvm::MCID::Add, llvm::dyn_cast(), llvm::Instruction::getOpcode(), llvm::ScalarEvolution::getSCEV(), llvm::SCEV::getSCEVType(), llvm::Value::getType(), llvm::SmallPtrSetImpl< PtrType >::insert(), isExistingPhi(), llvm::ScalarEvolution::isSCEVable(), llvm::scConstant, llvm::scSignExtend, llvm::scTruncate, llvm::scUnknown, llvm::scZeroExtend, and llvm::Value::users().
|
static |
Test whether we know how to expand the current formula.
Definition at line 1489 of file LoopStrengthReduce.cpp.
References isAMCompletelyFolded().
Referenced by isLegalUse().
|
static |
Definition at line 1503 of file LoopStrengthReduce.cpp.
References isLegalUse().
|
static |
Return true if the given mul can be sign-extended without changing its value.
Definition at line 560 of file LoopStrengthReduce.cpp.
References llvm::IntegerType::get(), llvm::ScalarEvolution::getContext(), llvm::SCEVNAryExpr::getNumOperands(), llvm::ScalarEvolution::getSignExtendExpr(), llvm::SCEVNAryExpr::getType(), and llvm::ScalarEvolution::getTypeSizeInBits().
Referenced by getExactSDiv().
|
static |
Return true if the number of registers needed for the chain is estimated to be less than the number required for the individual IV users.
First prohibit any IV users that keep the IV live across increments (the Users set should be empty). Next count the number and type of increments in the chain.
Chaining IVs can lead to considerable code bloat if ISEL doesn't effectively use postinc addressing modes. Only consider it profitable it the increments can be computed in fewer registers when chained.
TODO: Consider IVInc free if it's already used in another chains.
Definition at line 2576 of file LoopStrengthReduce.cpp.
References assert(), llvm::dbgs(), DEBUG, llvm::SmallPtrSetImplBase::empty(), for(), llvm::ScalarEvolution::getSCEV(), and StressIVChain.
|
static |
Definition at line 5022 of file LoopStrengthReduce.cpp.
References DEBUG_TYPE, llvm::DeleteDeadPHIs(), DeleteTriviallyDeadInstructions(), EnablePhiElim, llvm::LoopBase< N, M >::getHeader(), llvm::Loop::isLoopSimplifyForm(), llvm::SCEVExpander::replaceCongruentIVs(), Rewriter, and llvm::SCEVExpander::setDebugType().
Referenced by llvm::LoopStrengthReducePass::run().
|
static |
Definition at line 4002 of file LoopStrengthReduce.cpp.
|
static |
Referenced by ReduceLoopStrength().
| loop Loop Strength false |
Definition at line 5081 of file LoopStrengthReduce.cpp.
MaxIVUsers is an arbitrary threshold that provides an early opportunitiy for bail out.
This threshold is far beyond the number of users that LSR can conceivably solve, so it should not affect generated code, but catches the worst cases before LSR burns too much compile time and stack space.
Definition at line 122 of file LoopStrengthReduce.cpp.
| loop reduce |
Definition at line 5081 of file LoopStrengthReduce.cpp.
| loop Loop Strength Reduction |
Definition at line 5081 of file LoopStrengthReduce.cpp.
|
static |
Referenced by isProfitableChain().
1.8.6