SampleProfile.cpp File Reference

#include "llvm/Transforms/IPO/SampleProfile.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PriorityQueue.h"
#include "llvm/ADT/SCCIterator.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BlockFrequencyInfoImpl.h"
#include "llvm/Analysis/InlineAdvisor.h"
#include "llvm/Analysis/InlineCost.h"
#include "llvm/Analysis/LazyCallGraph.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/ReplayInlineAdvisor.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/PseudoProbe.h"
#include "llvm/IR/ValueSymbolTable.h"
#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/SampleProf.h"
#include "llvm/ProfileData/SampleProfReader.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/VirtualFileSystem.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/IPO.h"
#include "llvm/Transforms/IPO/ProfiledCallGraph.h"
#include "llvm/Transforms/IPO/SampleContextTracker.h"
#include "llvm/Transforms/IPO/SampleProfileMatcher.h"
#include "llvm/Transforms/IPO/SampleProfileProbe.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/CallPromotionUtils.h"
#include "llvm/Transforms/Utils/Cloning.h"
#include "llvm/Transforms/Utils/MisExpect.h"
#include "llvm/Transforms/Utils/SampleProfileLoaderBaseImpl.h"
#include "llvm/Transforms/Utils/SampleProfileLoaderBaseUtil.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <queue>
#include <string>
#include <system_error>
#include <utility>
#include <vector>

Go to the source code of this file.

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

Macros
#define	DEBUG_TYPE   "sample-profile"
#define	CSINLINE_DEBUG   DEBUG_TYPE "-inline"

Typedefs
using	ProfileCount = Function::ProfileCount

Functions
	STATISTIC (NumCSInlined, "Number of functions inlined with context sensitive profile")
	STATISTIC (NumCSNotInlined, "Number of functions not inlined with context sensitive profile")
	STATISTIC (NumMismatchedProfile, "Number of functions with CFG mismatched profile")
	STATISTIC (NumMatchedProfile, "Number of functions with CFG matched profile")
	STATISTIC (NumDuplicatedInlinesite, "Number of inlined callsites with a partial distribution factor")
	STATISTIC (NumCSInlinedHitMinLimit, "Number of functions with FDO inline stopped due to min size limit")
	STATISTIC (NumCSInlinedHitMaxLimit, "Number of functions with FDO inline stopped due to max size limit")
	STATISTIC (NumCSInlinedHitGrowthLimit, "Number of functions with FDO inline stopped due to growth size limit")
cl::opt< bool >	llvm::SortProfiledSCC ("sort-profiled-scc-member", cl::init(true), cl::Hidden, cl::desc("Sort profiled recursion by edge weights."))
cl::opt< int >	llvm::ProfileInlineGrowthLimit ("sample-profile-inline-growth-limit", cl::Hidden, cl::init(12), cl::desc("The size growth ratio limit for proirity-based sample profile " "loader inlining."))
cl::opt< int >	llvm::ProfileInlineLimitMin ("sample-profile-inline-limit-min", cl::Hidden, cl::init(100), cl::desc("The lower bound of size growth limit for " "proirity-based sample profile loader inlining."))
cl::opt< int >	llvm::ProfileInlineLimitMax ("sample-profile-inline-limit-max", cl::Hidden, cl::init(10000), cl::desc("The upper bound of size growth limit for " "proirity-based sample profile loader inlining."))
cl::opt< int >	llvm::SampleHotCallSiteThreshold ("sample-profile-hot-inline-threshold", cl::Hidden, cl::init(3000), cl::desc("Hot callsite threshold for proirity-based sample profile loader " "inlining."))
cl::opt< int >	llvm::SampleColdCallSiteThreshold ("sample-profile-cold-inline-threshold", cl::Hidden, cl::init(45), cl::desc("Threshold for inlining cold callsites"))
static bool	doesHistoryAllowICP (const Instruction &Inst, StringRef Candidate)
	Check whether the indirect call promotion history of Inst allows the promotion for Candidate.
static void	updateIDTMetaData (Instruction &Inst, const SmallVectorImpl< InstrProfValueData > &CallTargets, uint64_t Sum)
	Update indirect call target profile metadata for Inst.
static SmallVector< InstrProfValueData, 2 >	GetSortedValueDataFromCallTargets (const SampleRecord::CallTargetMap &M)
	Returns the sorted CallTargetMap M by count in descending order.

Variables
static cl::opt< std::string >	SampleProfileFile ("sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden)
static cl::opt< std::string >	SampleProfileRemappingFile ("sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden)
cl::opt< bool >	SalvageStaleProfile ("salvage-stale-profile", cl::Hidden, cl::init(false), cl::desc("Salvage stale profile by fuzzy matching and use the remapped " "location for sample profile query."))
cl::opt< bool >	SalvageUnusedProfile ("salvage-unused-profile", cl::Hidden, cl::init(false), cl::desc("Salvage unused profile by matching with new " "functions on call graph."))
cl::opt< bool >	ReportProfileStaleness ("report-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute and report stale profile statistical metrics."))
cl::opt< bool >	PersistProfileStaleness ("persist-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute stale profile statistical metrics and write it into the " "native object file(.llvm_stats section)."))
static cl::opt< bool >	ProfileSampleAccurate ("profile-sample-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "callsite and function as having 0 samples. Otherwise, treat " "un-sampled callsites and functions conservatively as unknown. "))
static cl::opt< bool >	ProfileSampleBlockAccurate ("profile-sample-block-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "branches and calls as having 0 samples. Otherwise, treat " "them conservatively as unknown. "))
static cl::opt< bool >	ProfileAccurateForSymsInList ("profile-accurate-for-symsinlist", cl::Hidden, cl::init(true), cl::desc("For symbols in profile symbol list, regard their profiles to " "be accurate. It may be overriden by profile-sample-accurate. "))
static cl::opt< bool >	ProfileMergeInlinee ("sample-profile-merge-inlinee", cl::Hidden, cl::init(true), cl::desc("Merge past inlinee's profile to outline version if sample " "profile loader decided not to inline a call site. It will " "only be enabled when top-down order of profile loading is " "enabled. "))
static cl::opt< bool >	ProfileTopDownLoad ("sample-profile-top-down-load", cl::Hidden, cl::init(true), cl::desc("Do profile annotation and inlining for functions in top-down " "order of call graph during sample profile loading. It only " "works for new pass manager. "))
static cl::opt< bool >	UseProfiledCallGraph ("use-profiled-call-graph", cl::init(true), cl::Hidden, cl::desc("Process functions in a top-down order " "defined by the profiled call graph when " "-sample-profile-top-down-load is on."))
static cl::opt< bool >	ProfileSizeInline ("sample-profile-inline-size", cl::Hidden, cl::init(false), cl::desc("Inline cold call sites in profile loader if it's beneficial " "for code size."))
static cl::opt< bool >	DisableSampleLoaderInlining ("disable-sample-loader-inlining", cl::Hidden, cl::init(false), cl::desc("If true, artifically skip inline transformation in sample-loader " "pass, and merge (or scale) profiles (as configured by " "--sample-profile-merge-inlinee)."))
static cl::opt< unsigned >	ProfileICPRelativeHotness ("sample-profile-icp-relative-hotness", cl::Hidden, cl::init(25), cl::desc("Relative hotness percentage threshold for indirect " "call promotion in proirity-based sample profile loader inlining."))
static cl::opt< unsigned >	ProfileICPRelativeHotnessSkip ("sample-profile-icp-relative-hotness-skip", cl::Hidden, cl::init(1), cl::desc("Skip relative hotness check for ICP up to given number of targets."))
static cl::opt< unsigned >	HotFuncCutoffForStalenessError ("hot-func-cutoff-for-staleness-error", cl::Hidden, cl::init(800000), cl::desc("A function is considered hot for staleness error check if its " "total sample count is above the specified percentile"))
static cl::opt< unsigned >	MinfuncsForStalenessError ("min-functions-for-staleness-error", cl::Hidden, cl::init(50), cl::desc("Skip the check if the number of hot functions is smaller than " "the specified number."))
static cl::opt< unsigned >	PrecentMismatchForStalenessError ("precent-mismatch-for-staleness-error", cl::Hidden, cl::init(80), cl::desc("Reject the profile if the mismatch percent is higher than the " "given number."))
static cl::opt< bool >	CallsitePrioritizedInline ("sample-profile-prioritized-inline", cl::Hidden, cl::desc("Use call site prioritized inlining for sample profile loader." "Currently only CSSPGO is supported."))
static cl::opt< bool >	UsePreInlinerDecision ("sample-profile-use-preinliner", cl::Hidden, cl::desc("Use the preinliner decisions stored in profile context."))
static cl::opt< bool >	AllowRecursiveInline ("sample-profile-recursive-inline", cl::Hidden, cl::desc("Allow sample loader inliner to inline recursive calls."))
static cl::opt< bool >	RemoveProbeAfterProfileAnnotation ("sample-profile-remove-probe", cl::Hidden, cl::init(false), cl::desc("Remove pseudo-probe after sample profile annotation."))
static cl::opt< std::string >	ProfileInlineReplayFile ("sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc("Optimization remarks file containing inline remarks to be replayed " "by inlining from sample profile loader."), cl::Hidden)
static cl::opt< ReplayInlinerSettings::Scope >	ProfileInlineReplayScope ("sample-profile-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during sample profile inlining."), cl::Hidden)
static cl::opt< ReplayInlinerSettings::Fallback >	ProfileInlineReplayFallback ("sample-profile-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values(clEnumValN(ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How sample profile inline replay treats sites that don't come " "from the replay. Original: defers to original advisor, " "AlwaysInline: inline all sites not in replay, NeverInline: " "inline no sites not in replay"), cl::Hidden)
static cl::opt< CallSiteFormat::Format >	ProfileInlineReplayFormat ("sample-profile-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values(clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How sample profile inline replay file is formatted"), cl::Hidden)
static cl::opt< unsigned >	MaxNumPromotions ("sample-profile-icp-max-prom", cl::init(3), cl::Hidden, cl::desc("Max number of promotions for a single indirect " "call callsite in sample profile loader"))
static cl::opt< bool >	OverwriteExistingWeights ("overwrite-existing-weights", cl::Hidden, cl::init(false), cl::desc("Ignore existing branch weights on IR and always overwrite."))
static cl::opt< bool >	AnnotateSampleProfileInlinePhase ("annotate-sample-profile-inline-phase", cl::Hidden, cl::init(false), cl::desc("Annotate LTO phase (prelink / postlink), or main (no LTO) for " "sample-profile inline pass name."))

Macro Definition Documentation

CSINLINE_DEBUG

#define CSINLINE_DEBUG   DEBUG_TYPE "-inline"

Definition at line 100 of file SampleProfile.cpp.

DEBUG_TYPE

#define DEBUG_TYPE   "sample-profile"

Definition at line 99 of file SampleProfile.cpp.

Typedef Documentation

ProfileCount

using ProfileCount = Function::ProfileCount

Definition at line 98 of file SampleProfile.cpp.

Function Documentation

doesHistoryAllowICP()

static bool doesHistoryAllowICP ( const Instruction &  Inst, StringRef  Candidate  )

static

Check whether the indirect call promotion history of Inst allows the promotion for Candidate.

If the profile count for the promotion candidate Candidate is NOMORE_ICP_MAGICNUM, it means Candidate has already been promoted for Inst. If we already have at least MaxNumPromotions NOMORE_ICP_MAGICNUM count values in the value profile of Inst, we cannot promote for Inst anymore.

Definition at line 804 of file SampleProfile.cpp.

References llvm::getValueProfDataFromInst(), MaxNumPromotions, and llvm::NOMORE_ICP_MAGICNUM.

GetSortedValueDataFromCallTargets()

static SmallVector< InstrProfValueData, 2 > GetSortedValueDataFromCallTargets ( const SampleRecord::CallTargetMap &  M )

static

Returns the sorted CallTargetMap M by count in descending order.

Definition at line 1601 of file SampleProfile.cpp.

References I, and llvm::sampleprof::SampleRecord::sortCallTargets().

STATISTIC() [1/8]

STATISTIC	(	NumCSInlined	,
		"Number of functions inlined with context sensitive profile"
	)

STATISTIC() [2/8]

STATISTIC	(	NumCSInlinedHitGrowthLimit	,
		"Number of functions with FDO inline stopped due to growth size limit"
	)

STATISTIC() [3/8]

STATISTIC	(	NumCSInlinedHitMaxLimit	,
		"Number of functions with FDO inline stopped due to max size limit"
	)

STATISTIC() [4/8]

STATISTIC	(	NumCSInlinedHitMinLimit	,
		"Number of functions with FDO inline stopped due to min size limit"
	)

STATISTIC() [5/8]

STATISTIC	(	NumCSNotInlined	,
		"Number of functions not inlined with context sensitive profile"
	)

STATISTIC() [6/8]

STATISTIC	(	NumDuplicatedInlinesite	,
		"Number of inlined callsites with a partial distribution factor"
	)

STATISTIC() [7/8]

STATISTIC	(	NumMatchedProfile	,
		"Number of functions with CFG matched profile"
	)

STATISTIC() [8/8]

STATISTIC	(	NumMismatchedProfile	,
		"Number of functions with CFG mismatched profile"
	)

updateIDTMetaData()

static void updateIDTMetaData ( Instruction &  Inst, const SmallVectorImpl< InstrProfValueData > &  CallTargets, uint64_t  Sum  )

static

Update indirect call target profile metadata for Inst.

Usually Sum is the sum of counts of all the targets for Inst. If it is 0, it means updateIDTMetaData is used to mark a certain target to be promoted already. If it is not zero, we expect to use it to update the total count in the value profile.

Definition at line 837 of file SampleProfile.cpp.

References llvm::annotateValueSite(), assert(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), llvm::getValueProfDataFromInst(), MaxNumPromotions, llvm::NOMORE_ICP_MAGICNUM, llvm::SmallVectorBase< Size_T >::size(), llvm::sort(), and llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace().

Variable Documentation

AllowRecursiveInline

cl::opt< bool > AllowRecursiveInline("sample-profile-recursive-inline", cl::Hidden, cl::desc("Allow sample loader inliner to inline recursive calls.")) ( "sample-profile-recursive-inline"  , cl::Hidden  , cl::desc("Allow sample loader inliner to inline recursive calls.")    )

static

AnnotateSampleProfileInlinePhase

cl::opt< bool > AnnotateSampleProfileInlinePhase("annotate-sample-profile-inline-phase", cl::Hidden, cl::init(false), cl::desc("Annotate LTO phase (prelink / postlink), or main (no LTO) for " "sample-profile inline pass name.")) ( "annotate-sample-profile-inline-phase"  , cl::Hidden  , cl::init(false)  , cl::desc("Annotate LTO phase (prelink / postlink), or main (no LTO) for " "sample-profile inline pass name.")    )

static

CallsitePrioritizedInline

cl::opt< bool > CallsitePrioritizedInline("sample-profile-prioritized-inline", cl::Hidden, cl::desc("Use call site prioritized inlining for sample profile loader." "Currently only CSSPGO is supported.")) ( "sample-profile-prioritized-inline"  , cl::Hidden  , cl::desc("Use call site prioritized inlining for sample profile loader." "Currently only CSSPGO is supported.")    )

static

DisableSampleLoaderInlining

cl::opt< bool > DisableSampleLoaderInlining("disable-sample-loader-inlining", cl::Hidden, cl::init(false), cl::desc("If true, artifically skip inline transformation in sample-loader " "pass, and merge (or scale) profiles (as configured by " "--sample-profile-merge-inlinee).")) ( "disable-sample-loader-inlining"  , cl::Hidden  , cl::init(false)  , cl::desc("If true, artifically skip inline transformation in sample-loader " "pass, and merge (or scale) profiles (as configured by " "--sample-profile-merge-inlinee).")    )

static

HotFuncCutoffForStalenessError

cl::opt< unsigned > HotFuncCutoffForStalenessError("hot-func-cutoff-for-staleness-error", cl::Hidden, cl::init(800000), cl::desc("A function is considered hot for staleness error check if its " "total sample count is above the specified percentile")) ( "hot-func-cutoff-for-staleness-error"  , cl::Hidden  , cl::init(800000)  , cl::desc("A function is considered hot for staleness error check if its " "total sample count is above the specified percentile")    )

static

MaxNumPromotions

cl::opt< unsigned > MaxNumPromotions("sample-profile-icp-max-prom", cl::init(3), cl::Hidden, cl::desc("Max number of promotions for a single indirect " "call callsite in sample profile loader")) ( "sample-profile-icp-max-prom"  , cl::init(3)  , cl::Hidden  , cl::desc("Max number of promotions for a single indirect " "call callsite in sample profile loader")    )

static

Referenced by doesHistoryAllowICP(), and updateIDTMetaData().

MinfuncsForStalenessError

cl::opt< unsigned > MinfuncsForStalenessError("min-functions-for-staleness-error", cl::Hidden, cl::init(50), cl::desc("Skip the check if the number of hot functions is smaller than " "the specified number.")) ( "min-functions-for-staleness-error"  , cl::Hidden  , cl::init(50)  , cl::desc("Skip the check if the number of hot functions is smaller than " "the specified number.")    )

static

OverwriteExistingWeights

cl::opt< bool > OverwriteExistingWeights("overwrite-existing-weights", cl::Hidden, cl::init(false), cl::desc("Ignore existing branch weights on IR and always overwrite.")) ( "overwrite-existing-weights"  , cl::Hidden  , cl::init(false)  , cl::desc("Ignore existing branch weights on IR and always overwrite.")    )

static

PersistProfileStaleness

cl::opt< bool > PersistProfileStaleness("persist-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute stale profile statistical metrics and write it into the " "native object file(.llvm_stats section)."))	(	"persist-profile-staleness"	,
		cl::Hidden	,
		cl::init(false)	,
		cl::desc("Compute stale profile statistical metrics and write it into the " "native object file(.llvm_stats section).")
	)

PrecentMismatchForStalenessError

cl::opt< unsigned > PrecentMismatchForStalenessError("precent-mismatch-for-staleness-error", cl::Hidden, cl::init(80), cl::desc("Reject the profile if the mismatch percent is higher than the " "given number.")) ( "precent-mismatch-for-staleness-error"  , cl::Hidden  , cl::init(80)  , cl::desc("Reject the profile if the mismatch percent is higher than the " "given number.")    )

static

ProfileAccurateForSymsInList

cl::opt< bool > ProfileAccurateForSymsInList("profile-accurate-for-symsinlist", cl::Hidden, cl::init(true), cl::desc("For symbols in profile symbol list, regard their profiles to " "be accurate. It may be overriden by profile-sample-accurate. ")) ( "profile-accurate-for-symsinlist"  , cl::Hidden  , cl::init(true)  , cl::desc("For symbols in profile symbol list, regard their profiles to " "be accurate. It may be overriden by profile-sample-accurate. ")    )

static

ProfileICPRelativeHotness

cl::opt< unsigned > ProfileICPRelativeHotness("sample-profile-icp-relative-hotness", cl::Hidden, cl::init(25), cl::desc( "Relative hotness percentage threshold for indirect " "call promotion in proirity-based sample profile loader inlining.")) ( "sample-profile-icp-relative-hotness"  , cl::Hidden  , cl::init(25)  , cl::desc( "Relative hotness percentage threshold for indirect " "call promotion in proirity-based sample profile loader inlining.")    )

static

ProfileICPRelativeHotnessSkip

cl::opt< unsigned > ProfileICPRelativeHotnessSkip("sample-profile-icp-relative-hotness-skip", cl::Hidden, cl::init(1), cl::desc( "Skip relative hotness check for ICP up to given number of targets.")) ( "sample-profile-icp-relative-hotness-skip"  , cl::Hidden  , cl::init(1)  , cl::desc( "Skip relative hotness check for ICP up to given number of targets.")    )

static

ProfileInlineReplayFallback

cl::opt< ReplayInlinerSettings::Fallback > ProfileInlineReplayFallback("sample-profile-inline-replay-fallback", cl::init(ReplayInlinerSettings::Fallback::Original), cl::values( clEnumValN( ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined")), cl::desc("How sample profile inline replay treats sites that don't come " "from the replay. Original: defers to original advisor, " "AlwaysInline: inline all sites not in replay, NeverInline: " "inline no sites not in replay"), cl::Hidden) ( "sample-profile-inline-replay-fallback"  , cl::init(ReplayInlinerSettings::Fallback::Original)  , cl::values( clEnumValN( ReplayInlinerSettings::Fallback::Original, "Original", "All decisions not in replay send to original advisor (default)"), clEnumValN(ReplayInlinerSettings::Fallback::AlwaysInline, "AlwaysInline", "All decisions not in replay are inlined"), clEnumValN(ReplayInlinerSettings::Fallback::NeverInline, "NeverInline", "All decisions not in replay are not inlined"))  , cl::desc("How sample profile inline replay treats sites that don't come " "from the replay. Original: defers to original advisor, " "AlwaysInline: inline all sites not in replay, NeverInline: " "inline no sites not in replay")  , cl::Hidden    )

static

ProfileInlineReplayFile

cl::opt< std::string > ProfileInlineReplayFile("sample-profile-inline-replay", cl::init(""), cl::value_desc("filename"), cl::desc( "Optimization remarks file containing inline remarks to be replayed " "by inlining from sample profile loader."), cl::Hidden) ( "sample-profile-inline-replay"  , cl::init("")  , cl::value_desc("filename")  , cl::desc( "Optimization remarks file containing inline remarks to be replayed " "by inlining from sample profile loader.")  , cl::Hidden    )

static

ProfileInlineReplayFormat

cl::opt< CallSiteFormat::Format > ProfileInlineReplayFormat("sample-profile-inline-replay-format", cl::init(CallSiteFormat::Format::LineColumnDiscriminator), cl::values( clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)")), cl::desc("How sample profile inline replay file is formatted"), cl::Hidden) ( "sample-profile-inline-replay-format"  , cl::init(CallSiteFormat::Format::LineColumnDiscriminator)  , cl::values( clEnumValN(CallSiteFormat::Format::Line, "Line", "<Line Number>"), clEnumValN(CallSiteFormat::Format::LineColumn, "LineColumn", "<Line Number>:<Column Number>"), clEnumValN(CallSiteFormat::Format::LineDiscriminator, "LineDiscriminator", "<Line Number>.<Discriminator>"), clEnumValN(CallSiteFormat::Format::LineColumnDiscriminator, "LineColumnDiscriminator", "<Line Number>:<Column Number>.<Discriminator> (default)"))  , cl::desc("How sample profile inline replay file is formatted")  , cl::Hidden    )

static

ProfileInlineReplayScope

cl::opt< ReplayInlinerSettings::Scope > ProfileInlineReplayScope("sample-profile-inline-replay-scope", cl::init(ReplayInlinerSettings::Scope::Function), cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module")), cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during sample profile inlining."), cl::Hidden) ( "sample-profile-inline-replay-scope"  , cl::init(ReplayInlinerSettings::Scope::Function)  , cl::values(clEnumValN(ReplayInlinerSettings::Scope::Function, "Function", "Replay on functions that have remarks associated " "with them (default)"), clEnumValN(ReplayInlinerSettings::Scope::Module, "Module", "Replay on the entire module"))  , cl::desc("Whether inline replay should be applied to the entire " "Module or just the Functions (default) that are present as " "callers in remarks during sample profile inlining.")  , cl::Hidden    )

static

ProfileMergeInlinee

cl::opt< bool > ProfileMergeInlinee("sample-profile-merge-inlinee", cl::Hidden, cl::init(true), cl::desc("Merge past inlinee's profile to outline version if sample " "profile loader decided not to inline a call site. It will " "only be enabled when top-down order of profile loading is " "enabled. ")) ( "sample-profile-merge-inlinee"  , cl::Hidden  , cl::init(true)  , cl::desc("Merge past inlinee's profile to outline version if sample " "profile loader decided not to inline a call site. It will " "only be enabled when top-down order of profile loading is " "enabled. ")    )

static

ProfileSampleAccurate

cl::opt< bool > ProfileSampleAccurate("profile-sample-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "callsite and function as having 0 samples. Otherwise, treat " "un-sampled callsites and functions conservatively as unknown. ")) ( "profile-sample-accurate"  , cl::Hidden  , cl::init(false)  , cl::desc("If the sample profile is accurate, we will mark all un-sampled " "callsite and function as having 0 samples. Otherwise, treat " "un-sampled callsites and functions conservatively as unknown. ")    )

static

ProfileSampleBlockAccurate

cl::opt< bool > ProfileSampleBlockAccurate("profile-sample-block-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " "branches and calls as having 0 samples. Otherwise, treat " "them conservatively as unknown. ")) ( "profile-sample-block-accurate"  , cl::Hidden  , cl::init(false)  , cl::desc("If the sample profile is accurate, we will mark all un-sampled " "branches and calls as having 0 samples. Otherwise, treat " "them conservatively as unknown. ")    )

static

ProfileSizeInline

cl::opt< bool > ProfileSizeInline("sample-profile-inline-size", cl::Hidden, cl::init(false), cl::desc("Inline cold call sites in profile loader if it's beneficial " "for code size.")) ( "sample-profile-inline-size"  , cl::Hidden  , cl::init(false)  , cl::desc("Inline cold call sites in profile loader if it's beneficial " "for code size.")    )

static

ProfileTopDownLoad

cl::opt< bool > ProfileTopDownLoad("sample-profile-top-down-load", cl::Hidden, cl::init(true), cl::desc("Do profile annotation and inlining for functions in top-down " "order of call graph during sample profile loading. It only " "works for new pass manager. ")) ( "sample-profile-top-down-load"  , cl::Hidden  , cl::init(true)  , cl::desc("Do profile annotation and inlining for functions in top-down " "order of call graph during sample profile loading. It only " "works for new pass manager. ")    )

static

RemoveProbeAfterProfileAnnotation

cl::opt< bool > RemoveProbeAfterProfileAnnotation("sample-profile-remove-probe", cl::Hidden, cl::init(false), cl::desc("Remove pseudo-probe after sample profile annotation.")) ( "sample-profile-remove-probe"  , cl::Hidden  , cl::init(false)  , cl::desc("Remove pseudo-probe after sample profile annotation.")    )

static

ReportProfileStaleness

cl::opt< bool > ReportProfileStaleness("report-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute and report stale profile statistical metrics."))	(	"report-profile-staleness"	,
		cl::Hidden	,
		cl::init(false)	,
		cl::desc("Compute and report stale profile statistical metrics.")
	)

SalvageStaleProfile

cl::opt< bool > SalvageStaleProfile("salvage-stale-profile", cl::Hidden, cl::init(false), cl::desc("Salvage stale profile by fuzzy matching and use the remapped " "location for sample profile query."))	(	"salvage-stale-profile"	,
		cl::Hidden	,
		cl::init(false)	,
		cl::desc("Salvage stale profile by fuzzy matching and use the remapped " "location for sample profile query.")
	)

Referenced by llvm::SampleProfileMatcher::runOnModule().

SalvageUnusedProfile

cl::opt< bool > SalvageUnusedProfile("salvage-unused-profile", cl::Hidden, cl::init(false), cl::desc("Salvage unused profile by matching with new " "functions on call graph."))	(	"salvage-unused-profile"	,
		cl::Hidden	,
		cl::init(false)	,
		cl::desc("Salvage unused profile by matching with new " "functions on call graph.")
	)

Referenced by llvm::SampleProfileMatcher::runOnModule().

SampleProfileFile

cl::opt< std::string > SampleProfileFile("sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden) ( "sample-profile-file"  , cl::init("")  , cl::value_desc("filename")  , cl::desc("Profile file loaded by -sample-profile")  , cl::Hidden    )

static

Referenced by llvm::SampleProfileLoaderPass::run().

SampleProfileRemappingFile

cl::opt< std::string > SampleProfileRemappingFile("sample-profile-remapping-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile remapping file loaded by -sample-profile"), cl::Hidden) ( "sample-profile-remapping-file"  , cl::init("")  , cl::value_desc("filename")  , cl::desc("Profile remapping file loaded by -sample-profile")  , cl::Hidden    )

static

Referenced by llvm::SampleProfileLoaderPass::run().

UsePreInlinerDecision

cl::opt< bool > UsePreInlinerDecision("sample-profile-use-preinliner", cl::Hidden, cl::desc("Use the preinliner decisions stored in profile context.")) ( "sample-profile-use-preinliner"  , cl::Hidden  , cl::desc("Use the preinliner decisions stored in profile context.")    )

static

UseProfiledCallGraph

cl::opt< bool > UseProfiledCallGraph("use-profiled-call-graph", cl::init(true), cl::Hidden, cl::desc("Process functions in a top-down order " "defined by the profiled call graph when " "-sample-profile-top-down-load is on.")) ( "use-profiled-call-graph"  , cl::init(true)  , cl::Hidden  , cl::desc("Process functions in a top-down order " "defined by the profiled call graph when " "-sample-profile-top-down-load is on.")    )

static