23#define DEBUG_TYPE "inline-order"
29 cl::desc(
"Choose the priority mode to use in module inline"),
31 "Use callee size priority."),
33 "Use inline cost priority."),
35 "Use cost-benefit ratio."),
40 cl::desc(
"The cost threshold for call sites that get inlined without the "
41 "cost-benefit analysis"));
51 .getCachedResult<ProfileSummaryAnalysis>(
52 *CB.
getParent()->getParent()->getParent());
68 Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
70 return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
71 GetBFI, PSI, RemarksEnabled ? &ORE :
nullptr);
76 SizePriority() =
default;
83 static bool isMoreDesirable(
const SizePriority &P1,
const SizePriority &P2) {
84 return P1.Size < P2.Size;
88 unsigned Size = UINT_MAX;
93 CostPriority() =
default;
96 auto IC = getInlineCostWrapper(
const_cast<CallBase &
>(*CB),
FAM, Params);
100 Cost = IC.isNever() ? INT_MAX : INT_MIN;
103 static bool isMoreDesirable(
const CostPriority &P1,
const CostPriority &P2) {
104 return P1.Cost < P2.Cost;
111class CostBenefitPriority {
113 CostBenefitPriority() =
default;
116 auto IC = getInlineCostWrapper(
const_cast<CallBase &
>(*CB),
FAM, Params);
120 Cost = IC.isNever() ? INT_MAX : INT_MIN;
121 StaticBonusApplied = IC.getStaticBonusApplied();
125 static bool isMoreDesirable(
const CostBenefitPriority &P1,
126 const CostBenefitPriority &P2) {
142 bool P1ReducesCallerSize =
144 bool P2ReducesCallerSize =
146 if (P1ReducesCallerSize || P2ReducesCallerSize) {
149 if (P1ReducesCallerSize != P2ReducesCallerSize)
150 return P1ReducesCallerSize;
154 return P1.Cost < P2.Cost;
157 bool P1HasCB = P1.CostBenefit.has_value();
158 bool P2HasCB = P2.CostBenefit.has_value();
159 if (P1HasCB || P2HasCB) {
162 if (P1HasCB != P2HasCB)
167 APInt LHS = P1.CostBenefit->getBenefit() * P2.CostBenefit->getCost();
168 APInt RHS = P2.CostBenefit->getBenefit() * P1.CostBenefit->getCost();
173 return P1.Cost < P2.Cost;
178 int StaticBonusApplied = 0;
184 MLPriority() =
default;
187 auto IC = getInlineCostWrapper(
const_cast<CallBase &
>(*CB),
FAM, Params);
191 Cost = IC.isNever() ? INT_MAX : INT_MIN;
194 static bool isMoreDesirable(
const MLPriority &P1,
const MLPriority &P2) {
195 return P1.Cost < P2.Cost;
202template <
typename PriorityT>
203class PriorityInlineOrder :
public InlineOrder<std::pair<CallBase *, int>> {
204 using T = std::pair<CallBase *, int>;
207 const auto I1 = Priorities.find(L);
208 const auto I2 = Priorities.find(R);
209 assert(I1 != Priorities.end() && I2 != Priorities.end());
210 return PriorityT::isMoreDesirable(I2->second,
I1->second);
213 bool updateAndCheckDecreased(
const CallBase *CB) {
214 auto It = Priorities.find(CB);
215 const auto OldPriority = It->second;
216 It->second = PriorityT(CB, FAM, Params);
217 const auto NewPriority = It->second;
218 return PriorityT::isMoreDesirable(OldPriority, NewPriority);
228 void pop_heap_adjust() {
229 std::pop_heap(Heap.begin(), Heap.end(), isLess);
230 while (updateAndCheckDecreased(Heap.back())) {
231 std::push_heap(Heap.begin(), Heap.end(), isLess);
232 std::pop_heap(Heap.begin(), Heap.end(), isLess);
238 :
FAM(
FAM), Params(Params) {
240 return hasLowerPriority(L, R);
244 size_t size()
override {
return Heap.size(); }
246 void push(
const T &Elt)
override {
248 const int InlineHistoryID = Elt.second;
251 Priorities[CB] = PriorityT(CB, FAM, Params);
252 std::push_heap(Heap.begin(), Heap.end(), isLess);
253 InlineHistoryMap[CB] = InlineHistoryID;
261 T Result = std::make_pair(CB, InlineHistoryMap[CB]);
262 InlineHistoryMap.erase(CB);
267 auto PredWrapper = [=](
CallBase *CB) ->
bool {
268 return Pred(std::make_pair(CB, InlineHistoryMap[CB]));
271 std::make_heap(Heap.begin(), Heap.end(), isLess);
287std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
293 LLVM_DEBUG(
dbgs() <<
" Current used priority: Size priority ---- \n");
294 return std::make_unique<PriorityInlineOrder<SizePriority>>(
FAM, Params);
297 LLVM_DEBUG(
dbgs() <<
" Current used priority: Cost priority ---- \n");
298 return std::make_unique<PriorityInlineOrder<CostPriority>>(
FAM, Params);
302 dbgs() <<
" Current used priority: cost-benefit priority ---- \n");
303 return std::make_unique<PriorityInlineOrder<CostBenefitPriority>>(
FAM,
306 LLVM_DEBUG(
dbgs() <<
" Current used priority: ML priority ---- \n");
307 return std::make_unique<PriorityInlineOrder<MLPriority>>(
FAM, Params);
312std::unique_ptr<InlineOrder<std::pair<CallBase *, int>>>
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This is the interface for a simple mod/ref and alias analysis over globals.
static cl::opt< int > ModuleInlinerTopPriorityThreshold("module-inliner-top-priority-threshold", cl::Hidden, cl::init(0), cl::desc("The cost threshold for call sites that get inlined without the " "cost-benefit analysis"))
static cl::opt< InlinePriorityMode > UseInlinePriority("inline-priority-mode", cl::init(InlinePriorityMode::Size), cl::Hidden, cl::desc("Choose the priority mode to use in module inline"), cl::values(clEnumValN(InlinePriorityMode::Size, "size", "Use callee size priority."), clEnumValN(InlinePriorityMode::Cost, "cost", "Use inline cost priority."), clEnumValN(InlinePriorityMode::CostBenefit, "cost-benefit", "Use cost-benefit ratio."), clEnumValN(InlinePriorityMode::ML, "ml", "Use ML.")))
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Class for arbitrary precision integers.
A container for analyses that lazily runs them and caches their results.
bool isPassRegistered() const
Returns true if the specified analysis pass is registered.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Function * getCaller()
Helper to get the caller (the parent function).
Represents the cost of inlining a function.
virtual void erase_if(function_ref< bool(T)> Pred)=0
virtual void push(const T &Elt)=0
A Module instance is used to store all the information related to an LLVM module.
An analysis over an "inner" IR unit that provides access to an analysis manager over a "outer" IR uni...
Used for dynamically loading instances of InlineOrder as plugins.
Analysis providing profile information.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Analysis pass providing the TargetTransformInfo.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
InlineCost getInlineCost(CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, function_ref< AssumptionCache &(Function &)> GetAssumptionCache, function_ref< const TargetLibraryInfo &(Function &)> GetTLI, function_ref< BlockFrequencyInfo &(Function &)> GetBFI=nullptr, ProfileSummaryInfo *PSI=nullptr, OptimizationRemarkEmitter *ORE=nullptr)
Get an InlineCost object representing the cost of inlining this callsite.
std::unique_ptr< InlineOrder< std::pair< CallBase *, int > > > getDefaultInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params, ModuleAnalysisManager &MAM, Module &M)
std::unique_ptr< InlineOrder< std::pair< CallBase *, int > > > getInlineOrder(FunctionAnalysisManager &FAM, const InlineParams &Params, ModuleAnalysisManager &MAM, Module &M)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
A special type used by analysis passes to provide an address that identifies that particular analysis...
Thresholds to tune inline cost analysis.