LLVM  4.0.0
Macros | Functions | Variables
LoopUnrollPass.cpp File Reference
#include "llvm/Transforms/Scalar/LoopUnrollPass.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/LoopUnrollAnalyzer.h"
#include "llvm/Analysis/OptimizationDiagnosticInfo.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Metadata.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Scalar/LoopPassManager.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/UnrollLoop.h"
#include <climits>
#include <utility>
Include dependency graph for LoopUnrollPass.cpp:

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "loop-unroll"
 

Functions

static
TargetTransformInfo::UnrollingPreferences 
gatherUnrollingPreferences (Loop *L, const TargetTransformInfo &TTI, Optional< unsigned > UserThreshold, Optional< unsigned > UserCount, Optional< bool > UserAllowPartial, Optional< bool > UserRuntime, Optional< bool > UserUpperBound)
 Gather the various unrolling parameters based on the defaults, compiler flags, TTI overrides and user specified parameters. More...
 
static Optional
< EstimatedUnrollCost > 
analyzeLoopUnrollCost (const Loop *L, unsigned TripCount, DominatorTree &DT, ScalarEvolution &SE, const TargetTransformInfo &TTI, unsigned MaxUnrolledLoopSize)
 Figure out if the loop is worth full unrolling. More...
 
static unsigned ApproximateLoopSize (const Loop *L, unsigned &NumCalls, bool &NotDuplicatable, bool &Convergent, const TargetTransformInfo &TTI, AssumptionCache *AC, unsigned BEInsns)
 ApproximateLoopSize - Approximate the size of the loop. More...
 
static MDNodeGetUnrollMetadataForLoop (const Loop *L, StringRef Name)
 
static bool HasUnrollFullPragma (const Loop *L)
 
static bool HasUnrollEnablePragma (const Loop *L)
 
static bool HasUnrollDisablePragma (const Loop *L)
 
static bool HasRuntimeUnrollDisablePragma (const Loop *L)
 
static unsigned UnrollCountPragmaValue (const Loop *L)
 
static void SetLoopAlreadyUnrolled (Loop *L)
 
static unsigned getFullUnrollBoostingFactor (const EstimatedUnrollCost &Cost, unsigned MaxPercentThresholdBoost)
 
static uint64_t getUnrolledLoopSize (unsigned LoopSize, TargetTransformInfo::UnrollingPreferences &UP)
 
static bool computeUnrollCount (Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE, OptimizationRemarkEmitter *ORE, unsigned &TripCount, unsigned MaxTripCount, unsigned &TripMultiple, unsigned LoopSize, TargetTransformInfo::UnrollingPreferences &UP, bool &UseUpperBound)
 
static bool tryToUnrollLoop (Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE, const TargetTransformInfo &TTI, AssumptionCache &AC, OptimizationRemarkEmitter &ORE, bool PreserveLCSSA, Optional< unsigned > ProvidedCount, Optional< unsigned > ProvidedThreshold, Optional< bool > ProvidedAllowPartial, Optional< bool > ProvidedRuntime, Optional< bool > ProvidedUpperBound)
 

Variables

static cl::opt< unsignedUnrollThreshold ("unroll-threshold", cl::Hidden, cl::desc("The baseline cost threshold for loop unrolling"))
 
static cl::opt< unsignedUnrollMaxPercentThresholdBoost ("unroll-max-percent-threshold-boost", cl::init(400), cl::Hidden, cl::desc("The maximum 'boost' (represented as a percentage >= 100) applied ""to the threshold when aggressively unrolling a loop due to the ""dynamic cost savings. If completely unrolling a loop will reduce ""the total runtime from X to Y, we boost the loop unroll ""threshold to DefaultThreshold*std::min(MaxPercentThresholdBoost, ""X/Y). This limit avoids excessive code bloat."))
 
static cl::opt< unsignedUnrollMaxIterationsCountToAnalyze ("unroll-max-iteration-count-to-analyze", cl::init(10), cl::Hidden, cl::desc("Don't allow loop unrolling to simulate more than this number of""iterations when checking full unroll profitability"))
 
static cl::opt< unsignedUnrollCount ("unroll-count", cl::Hidden, cl::desc("Use this unroll count for all loops including those with ""unroll_count pragma values, for testing purposes"))
 
static cl::opt< unsignedUnrollMaxCount ("unroll-max-count", cl::Hidden, cl::desc("Set the max unroll count for partial and runtime unrolling, for""testing purposes"))
 
static cl::opt< unsignedUnrollFullMaxCount ("unroll-full-max-count", cl::Hidden, cl::desc("Set the max unroll count for full unrolling, for testing purposes"))
 
static cl::opt< boolUnrollAllowPartial ("unroll-allow-partial", cl::Hidden, cl::desc("Allows loops to be partially unrolled until ""-unroll-threshold loop size is reached."))
 
static cl::opt< boolUnrollAllowRemainder ("unroll-allow-remainder", cl::Hidden, cl::desc("Allow generation of a loop remainder (extra iterations) ""when unrolling a loop."))
 
static cl::opt< boolUnrollRuntime ("unroll-runtime", cl::ZeroOrMore, cl::Hidden, cl::desc("Unroll loops with run-time trip counts"))
 
static cl::opt< unsignedUnrollMaxUpperBound ("unroll-max-upperbound", cl::init(8), cl::Hidden, cl::desc("The max of trip count upper bound that is considered in unrolling"))
 
static cl::opt< unsignedPragmaUnrollThreshold ("pragma-unroll-threshold", cl::init(16 *1024), cl::Hidden, cl::desc("Unrolled size limit for loops with an unroll(full) or ""unroll_count pragma."))
 
static cl::opt< unsignedFlatLoopTripCountThreshold ("flat-loop-tripcount-threshold", cl::init(5), cl::Hidden, cl::desc("If the runtime tripcount for the loop is lower than the ""threshold, the loop is considered as flat and will be less ""aggressively unrolled."))
 
static cl::opt< boolUnrollAllowPeeling ("unroll-allow-peeling", cl::Hidden, cl::desc("Allows loops to be peeled when the dynamic ""trip count is known to be low."))
 
static const unsigned NoThreshold = UINT_MAX
 A magic value for use with the Threshold parameter to indicate that the loop unroll should be performed regardless of how much code expansion would result. More...
 

Macro Definition Documentation

#define DEBUG_TYPE   "loop-unroll"

Definition at line 43 of file LoopUnrollPass.cpp.

Referenced by computeUnrollCount().

Function Documentation

static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost ( const Loop L,
unsigned  TripCount,
DominatorTree DT,
ScalarEvolution SE,
const TargetTransformInfo TTI,
unsigned  MaxUnrolledLoopSize 
)
static

Figure out if the loop is worth full unrolling.

Complete loop unrolling can make some loads constant, and we need to know if that would expose any further optimization opportunities. This routine estimates this optimization. It computes cost of unrolled loop (UnrolledCost) and dynamic cost of the original loop (RolledDynamicCost). By dynamic cost we mean that we won't count costs of blocks that are known not to be executed (i.e. if we have a branch in the loop and we know that at the given iteration its condition would be resolved to true, we won't add up the cost of the 'false'-block).

Returns
Optional value, holding the RolledDynamicCost and UnrolledCost. If the analysis failed (no benefits expected from the unrolling, or the loop is too big to analyze), the returned value is None.

Definition at line 253 of file LoopUnrollPass.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorTemplateCommon< T >::begin(), C, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::clear(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::clear(), llvm::SmallVectorImpl< T >::clear(), llvm::LoopBase< N, M >::contains(), llvm::dbgs(), DEBUG, llvm::Value::dump(), llvm::dyn_cast(), llvm::SmallVectorBase::empty(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::empty(), llvm::LoopBase< N, M >::empty(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::end(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::find(), llvm::LoopBase< N, M >::getHeader(), llvm::LoopBase< N, M >::getLoopLatch(), llvm::LoopBase< N, M >::getLoopPreheader(), llvm::TargetTransformInfo::getUserCost(), I, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert(), llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, ValueInfoT, detail::DenseSetPair< ValueT > >, ValueInfoT >::insert(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::Loop::isLCSSAForm(), llvm::Loop::isLoopSimplifyForm(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::lookup(), llvm::None, llvm::User::operands(), llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::pop_back_val(), llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), SI, llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size(), llvm::successors(), UnrollMaxIterationsCountToAnalyze, and llvm::InstVisitor< SubClass, RetTy >::visit().

Referenced by computeUnrollCount().

static unsigned ApproximateLoopSize ( const Loop L,
unsigned NumCalls,
bool NotDuplicatable,
bool Convergent,
const TargetTransformInfo TTI,
AssumptionCache AC,
unsigned  BEInsns 
)
static
static bool computeUnrollCount ( Loop L,
const TargetTransformInfo TTI,
DominatorTree DT,
LoopInfo LI,
ScalarEvolution SE,
OptimizationRemarkEmitter ORE,
unsigned TripCount,
unsigned  MaxTripCount,
unsigned TripMultiple,
unsigned  LoopSize,
TargetTransformInfo::UnrollingPreferences UP,
bool UseUpperBound 
)
static

Definition at line 686 of file LoopUnrollPass.cpp.

References llvm::TargetTransformInfo::UnrollingPreferences::AllowExpensiveTripCount, llvm::TargetTransformInfo::UnrollingPreferences::AllowRemainder, analyzeLoopUnrollCost(), assert(), llvm::TargetTransformInfo::UnrollingPreferences::BEInsns, llvm::computePeelCount(), llvm::TargetTransformInfo::UnrollingPreferences::Count, llvm::dbgs(), DEBUG, DEBUG_TYPE, llvm::TargetTransformInfo::UnrollingPreferences::DefaultUnrollRuntimeCount, llvm::OptimizationRemarkEmitter::emit(), FlatLoopTripCountThreshold, llvm::TargetTransformInfo::UnrollingPreferences::Force, llvm::TargetTransformInfo::UnrollingPreferences::FullUnrollMaxCount, getFullUnrollBoostingFactor(), llvm::LoopBase< N, M >::getHeader(), llvm::getLoopEstimatedTripCount(), llvm::Loop::getStartLoc(), getUnrolledLoopSize(), HasRuntimeUnrollDisablePragma(), HasUnrollEnablePragma(), HasUnrollFullPragma(), llvm::TargetTransformInfo::UnrollingPreferences::MaxCount, llvm::TargetTransformInfo::UnrollingPreferences::MaxPercentThresholdBoost, NoThreshold, llvm::TargetTransformInfo::UnrollingPreferences::Partial, llvm::TargetTransformInfo::UnrollingPreferences::PartialThreshold, llvm::TargetTransformInfo::UnrollingPreferences::PeelCount, PragmaUnrollThreshold, llvm::TargetTransformInfo::UnrollingPreferences::Runtime, llvm::TargetTransformInfo::UnrollingPreferences::Threshold, UnrollCount, UnrollCountPragmaValue(), and llvm::TargetTransformInfo::UnrollingPreferences::UpperBound.

Referenced by tryToUnrollLoop().

static TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences ( Loop L,
const TargetTransformInfo TTI,
Optional< unsigned UserThreshold,
Optional< unsigned UserCount,
Optional< bool UserAllowPartial,
Optional< bool UserRuntime,
Optional< bool UserUpperBound 
)
static

Gather the various unrolling parameters based on the defaults, compiler flags, TTI overrides and user specified parameters.

Definition at line 120 of file LoopUnrollPass.cpp.

References llvm::TargetTransformInfo::UnrollingPreferences::AllowExpensiveTripCount, llvm::TargetTransformInfo::UnrollingPreferences::AllowPeeling, llvm::TargetTransformInfo::UnrollingPreferences::AllowRemainder, llvm::TargetTransformInfo::UnrollingPreferences::BEInsns, llvm::TargetTransformInfo::UnrollingPreferences::Count, llvm::TargetTransformInfo::UnrollingPreferences::DefaultUnrollRuntimeCount, llvm::TargetTransformInfo::UnrollingPreferences::Force, llvm::TargetTransformInfo::UnrollingPreferences::FullUnrollMaxCount, llvm::LoopBase< N, M >::getHeader(), llvm::TargetTransformInfo::getUnrollingPreferences(), llvm::Optional< T >::hasValue(), if(), llvm::TargetTransformInfo::UnrollingPreferences::MaxCount, llvm::TargetTransformInfo::UnrollingPreferences::MaxPercentThresholdBoost, llvm::TargetTransformInfo::UnrollingPreferences::OptSizeThreshold, llvm::TargetTransformInfo::UnrollingPreferences::Partial, llvm::TargetTransformInfo::UnrollingPreferences::PartialOptSizeThreshold, llvm::TargetTransformInfo::UnrollingPreferences::PartialThreshold, llvm::TargetTransformInfo::UnrollingPreferences::PeelCount, llvm::TargetTransformInfo::UnrollingPreferences::Runtime, llvm::TargetTransformInfo::UnrollingPreferences::Threshold, UnrollAllowPartial, UnrollAllowPeeling, UnrollAllowRemainder, UnrollFullMaxCount, UnrollMaxCount, UnrollMaxPercentThresholdBoost, UnrollMaxUpperBound, UnrollRuntime, UnrollThreshold, and llvm::TargetTransformInfo::UnrollingPreferences::UpperBound.

Referenced by tryToUnrollLoop().

static unsigned getFullUnrollBoostingFactor ( const EstimatedUnrollCost &  Cost,
unsigned  MaxPercentThresholdBoost 
)
static

Definition at line 664 of file LoopUnrollPass.cpp.

References fuzzer::min().

Referenced by computeUnrollCount().

static uint64_t getUnrolledLoopSize ( unsigned  LoopSize,
TargetTransformInfo::UnrollingPreferences UP 
)
static
static MDNode* GetUnrollMetadataForLoop ( const Loop L,
StringRef  Name 
)
static
static bool HasRuntimeUnrollDisablePragma ( const Loop L)
static

Definition at line 603 of file LoopUnrollPass.cpp.

References GetUnrollMetadataForLoop().

Referenced by computeUnrollCount().

static bool HasUnrollDisablePragma ( const Loop L)
static

Definition at line 598 of file LoopUnrollPass.cpp.

References GetUnrollMetadataForLoop().

Referenced by tryToUnrollLoop().

static bool HasUnrollEnablePragma ( const Loop L)
static

Definition at line 593 of file LoopUnrollPass.cpp.

References GetUnrollMetadataForLoop().

Referenced by computeUnrollCount().

static bool HasUnrollFullPragma ( const Loop L)
static

Definition at line 587 of file LoopUnrollPass.cpp.

References GetUnrollMetadataForLoop().

Referenced by computeUnrollCount().

static void SetLoopAlreadyUnrolled ( Loop L)
static
static bool tryToUnrollLoop ( Loop L,
DominatorTree DT,
LoopInfo LI,
ScalarEvolution SE,
const TargetTransformInfo TTI,
AssumptionCache AC,
OptimizationRemarkEmitter ORE,
bool  PreserveLCSSA,
Optional< unsigned ProvidedCount,
Optional< unsigned ProvidedThreshold,
Optional< bool ProvidedAllowPartial,
Optional< bool ProvidedRuntime,
Optional< bool ProvidedUpperBound 
)
static
static unsigned UnrollCountPragmaValue ( const Loop L)
static

Variable Documentation

cl::opt<unsigned> FlatLoopTripCountThreshold("flat-loop-tripcount-threshold", cl::init(5), cl::Hidden, cl::desc("If the runtime tripcount for the loop is lower than the ""threshold, the loop is considered as flat and will be less ""aggressively unrolled."))
static

Referenced by computeUnrollCount().

const unsigned NoThreshold = UINT_MAX
static

A magic value for use with the Threshold parameter to indicate that the loop unroll should be performed regardless of how much code expansion would result.

Definition at line 116 of file LoopUnrollPass.cpp.

Referenced by computeUnrollCount().

cl::opt<unsigned> PragmaUnrollThreshold("pragma-unroll-threshold", cl::init(16 *1024), cl::Hidden, cl::desc("Unrolled size limit for loops with an unroll(full) or ""unroll_count pragma."))
static

Referenced by computeUnrollCount().

cl::opt<bool> UnrollAllowPartial("unroll-allow-partial", cl::Hidden, cl::desc("Allows loops to be partially unrolled until ""-unroll-threshold loop size is reached."))
static
cl::opt<bool> UnrollAllowPeeling("unroll-allow-peeling", cl::Hidden, cl::desc("Allows loops to be peeled when the dynamic ""trip count is known to be low."))
static
cl::opt<bool> UnrollAllowRemainder("unroll-allow-remainder", cl::Hidden, cl::desc("Allow generation of a loop remainder (extra iterations) ""when unrolling a loop."))
static
cl::opt<unsigned> UnrollCount("unroll-count", cl::Hidden, cl::desc("Use this unroll count for all loops including those with ""unroll_count pragma values, for testing purposes"))
static

Referenced by computeUnrollCount().

cl::opt<unsigned> UnrollFullMaxCount("unroll-full-max-count", cl::Hidden, cl::desc("Set the max unroll count for full unrolling, for testing purposes"))
static
cl::opt<unsigned> UnrollMaxCount("unroll-max-count", cl::Hidden, cl::desc("Set the max unroll count for partial and runtime unrolling, for""testing purposes"))
static
cl::opt<unsigned> UnrollMaxIterationsCountToAnalyze("unroll-max-iteration-count-to-analyze", cl::init(10), cl::Hidden, cl::desc("Don't allow loop unrolling to simulate more than this number of""iterations when checking full unroll profitability"))
static

Referenced by analyzeLoopUnrollCost().

cl::opt<unsigned> UnrollMaxPercentThresholdBoost("unroll-max-percent-threshold-boost", cl::init(400), cl::Hidden, cl::desc("The maximum 'boost' (represented as a percentage >= 100) applied ""to the threshold when aggressively unrolling a loop due to the ""dynamic cost savings. If completely unrolling a loop will reduce ""the total runtime from X to Y, we boost the loop unroll ""threshold to DefaultThreshold*std::min(MaxPercentThresholdBoost, ""X/Y). This limit avoids excessive code bloat."))
static
cl::opt<unsigned> UnrollMaxUpperBound("unroll-max-upperbound", cl::init(8), cl::Hidden, cl::desc("The max of trip count upper bound that is considered in unrolling"))
static
cl::opt<bool> UnrollRuntime("unroll-runtime", cl::ZeroOrMore, cl::Hidden, cl::desc("Unroll loops with run-time trip counts"))
static
cl::opt<unsigned> UnrollThreshold("unroll-threshold", cl::Hidden, cl::desc("The baseline cost threshold for loop unrolling"))
static