LCOV - code coverage report
Current view: top level - lib/Target/AMDGPU - AMDGPUInline.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 55 60 91.7 %
Date: 2018-10-16 05:50:02 Functions: 9 9 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- AMDGPUInline.cpp - Code to perform simple function inlining --------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : /// \file
      11             : /// This is AMDGPU specific replacement of the standard inliner.
      12             : /// The main purpose is to account for the fact that calls not only expensive
      13             : /// on the AMDGPU, but much more expensive if a private memory pointer is
      14             : /// passed to a function as an argument. In this situation, we are unable to
      15             : /// eliminate private memory in the caller unless inlined and end up with slow
      16             : /// and expensive scratch access. Thus, we boost the inline threshold for such
      17             : /// functions here.
      18             : ///
      19             : //===----------------------------------------------------------------------===//
      20             : 
      21             : 
      22             : #include "AMDGPU.h"
      23             : #include "llvm/Transforms/IPO.h"
      24             : #include "llvm/Analysis/AssumptionCache.h"
      25             : #include "llvm/Analysis/CallGraph.h"
      26             : #include "llvm/Analysis/InlineCost.h"
      27             : #include "llvm/Analysis/ValueTracking.h"
      28             : #include "llvm/Analysis/TargetTransformInfo.h"
      29             : #include "llvm/IR/CallSite.h"
      30             : #include "llvm/IR/DataLayout.h"
      31             : #include "llvm/IR/Instructions.h"
      32             : #include "llvm/IR/Module.h"
      33             : #include "llvm/IR/Type.h"
      34             : #include "llvm/Support/CommandLine.h"
      35             : #include "llvm/Support/Debug.h"
      36             : #include "llvm/Transforms/IPO/Inliner.h"
      37             : 
      38             : using namespace llvm;
      39             : 
      40             : #define DEBUG_TYPE "inline"
      41             : 
      42             : static cl::opt<int>
      43             : ArgAllocaCost("amdgpu-inline-arg-alloca-cost", cl::Hidden, cl::init(2200),
      44             :               cl::desc("Cost of alloca argument"));
      45             : 
      46             : // If the amount of scratch memory to eliminate exceeds our ability to allocate
      47             : // it into registers we gain nothing by agressively inlining functions for that
      48             : // heuristic.
      49             : static cl::opt<unsigned>
      50             : ArgAllocaCutoff("amdgpu-inline-arg-alloca-cutoff", cl::Hidden, cl::init(256),
      51             :                 cl::desc("Maximum alloca size to use for inline cost"));
      52             : 
      53             : namespace {
      54             : 
      55             : class AMDGPUInliner : public LegacyInlinerBase {
      56             : 
      57             : public:
      58           2 :   AMDGPUInliner() : LegacyInlinerBase(ID) {
      59           2 :     initializeAMDGPUInlinerPass(*PassRegistry::getPassRegistry());
      60           2 :     Params = getInlineParams();
      61           2 :   }
      62             : 
      63             :   static char ID; // Pass identification, replacement for typeid
      64             : 
      65             :   unsigned getInlineThreshold(CallSite CS) const;
      66             : 
      67             :   InlineCost getInlineCost(CallSite CS) override;
      68             : 
      69             :   bool runOnSCC(CallGraphSCC &SCC) override;
      70             : 
      71             :   void getAnalysisUsage(AnalysisUsage &AU) const override;
      72             : 
      73             : private:
      74             :   TargetTransformInfoWrapperPass *TTIWP;
      75             : 
      76             :   InlineParams Params;
      77             : };
      78             : 
      79             : } // end anonymous namespace
      80             : 
      81             : char AMDGPUInliner::ID = 0;
      82       84407 : INITIALIZE_PASS_BEGIN(AMDGPUInliner, "amdgpu-inline",
      83             :                 "AMDGPU Function Integration/Inlining", false, false)
      84       84407 : INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
      85       84407 : INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
      86       84407 : INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
      87       84407 : INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
      88       84407 : INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
      89      197403 : INITIALIZE_PASS_END(AMDGPUInliner, "amdgpu-inline",
      90             :                 "AMDGPU Function Integration/Inlining", false, false)
      91             : 
      92           2 : Pass *llvm::createAMDGPUFunctionInliningPass() { return new AMDGPUInliner(); }
      93             : 
      94          24 : bool AMDGPUInliner::runOnSCC(CallGraphSCC &SCC) {
      95          24 :   TTIWP = &getAnalysis<TargetTransformInfoWrapperPass>();
      96          24 :   return LegacyInlinerBase::runOnSCC(SCC);
      97             : }
      98             : 
      99           2 : void AMDGPUInliner::getAnalysisUsage(AnalysisUsage &AU) const {
     100             :   AU.addRequired<TargetTransformInfoWrapperPass>();
     101           2 :   LegacyInlinerBase::getAnalysisUsage(AU);
     102           2 : }
     103             : 
     104          11 : unsigned AMDGPUInliner::getInlineThreshold(CallSite CS) const {
     105          11 :   int Thres = Params.DefaultThreshold;
     106             : 
     107             :   Function *Caller = CS.getCaller();
     108             :   // Listen to the inlinehint attribute when it would increase the threshold
     109             :   // and the caller does not need to minimize its size.
     110             :   Function *Callee = CS.getCalledFunction();
     111          22 :   bool InlineHint = Callee && !Callee->isDeclaration() &&
     112             :     Callee->hasFnAttribute(Attribute::InlineHint);
     113           0 :   if (InlineHint && Params.HintThreshold && Params.HintThreshold > Thres
     114           0 :       && !Caller->hasFnAttribute(Attribute::MinSize))
     115           0 :     Thres = Params.HintThreshold.getValue();
     116             : 
     117          11 :   const DataLayout &DL = Caller->getParent()->getDataLayout();
     118          11 :   if (!Callee)
     119           0 :     return (unsigned)Thres;
     120             : 
     121             :   // If we have a pointer to private array passed into a function
     122             :   // it will not be optimized out, leaving scratch usage.
     123             :   // Increase the inline threshold to allow inliniting in this case.
     124             :   uint64_t AllocaSize = 0;
     125             :   SmallPtrSet<const AllocaInst *, 8> AIVisited;
     126          29 :   for (Value *PtrArg : CS.args()) {
     127          20 :     Type *Ty = PtrArg->getType();
     128          20 :     if (!Ty->isPointerTy() ||
     129             :         Ty->getPointerAddressSpace() != AMDGPUAS::PRIVATE_ADDRESS)
     130             :       continue;
     131          14 :     PtrArg = GetUnderlyingObject(PtrArg, DL);
     132             :     if (const AllocaInst *AI = dyn_cast<AllocaInst>(PtrArg)) {
     133          14 :       if (!AI->isStaticAlloca() || !AIVisited.insert(AI).second)
     134           2 :         continue;
     135          12 :       AllocaSize += DL.getTypeAllocSize(AI->getAllocatedType());
     136             :       // If the amount of stack memory is excessive we will not be able
     137             :       // to get rid of the scratch anyway, bail out.
     138          12 :       if (AllocaSize > ArgAllocaCutoff) {
     139             :         AllocaSize = 0;
     140             :         break;
     141             :       }
     142             :     }
     143             :   }
     144          11 :   if (AllocaSize)
     145           6 :     Thres += ArgAllocaCost;
     146             : 
     147          11 :   return (unsigned)Thres;
     148             : }
     149             : 
     150             : // Check if call is just a wrapper around another call.
     151             : // In this case we only have call and ret instructions.
     152          13 : static bool isWrapperOnlyCall(CallSite CS) {
     153             :   Function *Callee = CS.getCalledFunction();
     154          13 :   if (!Callee || Callee->size() != 1)
     155             :     return false;
     156             :   const BasicBlock &BB = Callee->getEntryBlock();
     157           5 :   if (const Instruction *I = BB.getFirstNonPHI()) {
     158           5 :     if (!isa<CallInst>(I)) {
     159             :       return false;
     160             :     }
     161           4 :     if (isa<ReturnInst>(*std::next(I->getIterator()))) {
     162             :       LLVM_DEBUG(dbgs() << "    Wrapper only call detected: "
     163             :                         << Callee->getName() << '\n');
     164           2 :       return true;
     165             :     }
     166             :   }
     167             :   return false;
     168             : }
     169             : 
     170          17 : InlineCost AMDGPUInliner::getInlineCost(CallSite CS) {
     171             :   Function *Callee = CS.getCalledFunction();
     172             :   Function *Caller = CS.getCaller();
     173          17 :   TargetTransformInfo &TTI = TTIWP->getTTI(*Callee);
     174             : 
     175          17 :   if (!Callee || Callee->isDeclaration())
     176             :     return llvm::InlineCost::getNever("undefined callee");
     177             : 
     178          17 :   if (CS.isNoInline())
     179             :     return llvm::InlineCost::getNever("noinline");
     180             : 
     181          13 :   if (!TTI.areInlineCompatible(Caller, Callee))
     182             :     return llvm::InlineCost::getNever("incompatible");
     183             : 
     184          13 :   if (CS.hasFnAttr(Attribute::AlwaysInline)) {
     185           0 :     if (isInlineViable(*Callee))
     186             :       return llvm::InlineCost::getAlways("alwaysinline viable");
     187             :     return llvm::InlineCost::getNever("alwaysinline unviable");
     188             :   }
     189             : 
     190          13 :   if (isWrapperOnlyCall(CS))
     191             :     return llvm::InlineCost::getAlways("wrapper-only call");
     192             : 
     193          11 :   InlineParams LocalParams = Params;
     194          11 :   LocalParams.DefaultThreshold = (int)getInlineThreshold(CS);
     195             :   bool RemarksEnabled = false;
     196             :   const auto &BBs = Caller->getBasicBlockList();
     197          11 :   if (!BBs.empty()) {
     198          11 :     auto DI = OptimizationRemark(DEBUG_TYPE, "", DebugLoc(), &BBs.front());
     199          11 :     if (DI.isEnabled())
     200             :       RemarksEnabled = true;
     201             :   }
     202             : 
     203          11 :   OptimizationRemarkEmitter ORE(Caller);
     204             :   std::function<AssumptionCache &(Function &)> GetAssumptionCache =
     205             :       [this](Function &F) -> AssumptionCache & {
     206           8 :     return ACT->getAssumptionCache(F);
     207             :   };
     208             : 
     209             :   return llvm::getInlineCost(CS, Callee, LocalParams, TTI, GetAssumptionCache,
     210          33 :                              None, PSI, RemarksEnabled ? &ORE : nullptr);
     211             : }

Generated by: LCOV version 1.13