LCOV - code coverage report
Current view: top level - lib/Analysis - CGSCCPassManager.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 172 174 98.9 %
Date: 2018-05-20 00:06:23 Functions: 11 11 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===- CGSCCPassManager.cpp - Managing & running CGSCC passes -------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : 
      10             : #include "llvm/Analysis/CGSCCPassManager.h"
      11             : #include "llvm/ADT/ArrayRef.h"
      12             : #include "llvm/ADT/Optional.h"
      13             : #include "llvm/ADT/STLExtras.h"
      14             : #include "llvm/ADT/SetVector.h"
      15             : #include "llvm/ADT/SmallPtrSet.h"
      16             : #include "llvm/ADT/SmallVector.h"
      17             : #include "llvm/ADT/iterator_range.h"
      18             : #include "llvm/Analysis/LazyCallGraph.h"
      19             : #include "llvm/IR/CallSite.h"
      20             : #include "llvm/IR/Constant.h"
      21             : #include "llvm/IR/InstIterator.h"
      22             : #include "llvm/IR/Instruction.h"
      23             : #include "llvm/IR/PassManager.h"
      24             : #include "llvm/Support/Casting.h"
      25             : #include "llvm/Support/Debug.h"
      26             : #include "llvm/Support/raw_ostream.h"
      27             : #include <algorithm>
      28             : #include <cassert>
      29             : #include <iterator>
      30             : 
      31             : #define DEBUG_TYPE "cgscc"
      32             : 
      33             : using namespace llvm;
      34             : 
      35             : // Explicit template instantiations and specialization definitions for core
      36             : // template typedefs.
      37             : namespace llvm {
      38             : 
      39             : // Explicit instantiations for the core proxy templates.
      40             : template class AllAnalysesOn<LazyCallGraph::SCC>;
      41             : template class AnalysisManager<LazyCallGraph::SCC, LazyCallGraph &>;
      42             : template class PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager,
      43             :                            LazyCallGraph &, CGSCCUpdateResult &>;
      44             : template class InnerAnalysisManagerProxy<CGSCCAnalysisManager, Module>;
      45             : template class OuterAnalysisManagerProxy<ModuleAnalysisManager,
      46             :                                          LazyCallGraph::SCC, LazyCallGraph &>;
      47             : template class OuterAnalysisManagerProxy<CGSCCAnalysisManager, Function>;
      48             : 
      49             : /// Explicitly specialize the pass manager run method to handle call graph
      50             : /// updates.
      51             : template <>
      52             : PreservedAnalyses
      53        1051 : PassManager<LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,
      54             :             CGSCCUpdateResult &>::run(LazyCallGraph::SCC &InitialC,
      55             :                                       CGSCCAnalysisManager &AM,
      56             :                                       LazyCallGraph &G, CGSCCUpdateResult &UR) {
      57             :   PreservedAnalyses PA = PreservedAnalyses::all();
      58             : 
      59        1051 :   if (DebugLogging)
      60         186 :     dbgs() << "Starting CGSCC pass manager run.\n";
      61             : 
      62             :   // The SCC may be refined while we are running passes over it, so set up
      63             :   // a pointer that we can update.
      64             :   LazyCallGraph::SCC *C = &InitialC;
      65             : 
      66        2704 :   for (auto &Pass : Passes) {
      67        1655 :     if (DebugLogging)
      68         830 :       dbgs() << "Running pass: " << Pass->name() << " on " << *C << "\n";
      69             : 
      70        3308 :     PreservedAnalyses PassPA = Pass->run(*C, AM, G, UR);
      71             : 
      72             :     // Update the SCC if necessary.
      73        1655 :     C = UR.UpdatedC ? UR.UpdatedC : C;
      74             : 
      75             :     // If the CGSCC pass wasn't able to provide a valid updated SCC, the
      76             :     // current SCC may simply need to be skipped if invalid.
      77        1655 :     if (UR.InvalidatedSCCs.count(C)) {
      78             :       LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n");
      79           2 :       break;
      80             :     }
      81             :     // Check that we didn't miss any update scenario.
      82             :     assert(C->begin() != C->end() && "Cannot have an empty SCC!");
      83             : 
      84             :     // Update the analysis manager as each pass runs and potentially
      85             :     // invalidates analyses.
      86        1653 :     AM.invalidate(*C, PassPA);
      87             : 
      88             :     // Finally, we intersect the final preserved analyses to compute the
      89             :     // aggregate preserved set for this pass manager.
      90        1653 :     PA.intersect(std::move(PassPA));
      91             : 
      92             :     // FIXME: Historically, the pass managers all called the LLVM context's
      93             :     // yield function here. We don't have a generic way to acquire the
      94             :     // context and it isn't yet clear what the right pattern is for yielding
      95             :     // in the new pass manager so it is currently omitted.
      96             :     // ...getContext().yield();
      97             :   }
      98             : 
      99             :   // Invalidation was handled after each pass in the above loop for the current
     100             :   // SCC. Therefore, the remaining analysis results in the AnalysisManager are
     101             :   // preserved. We mark this with a set so that we don't need to inspect each
     102             :   // one individually.
     103             :   PA.preserveSet<AllAnalysesOn<LazyCallGraph::SCC>>();
     104             : 
     105        1051 :   if (DebugLogging)
     106         186 :     dbgs() << "Finished CGSCC pass manager run.\n";
     107             : 
     108        1051 :   return PA;
     109             : }
     110             : 
     111         212 : bool CGSCCAnalysisManagerModuleProxy::Result::invalidate(
     112             :     Module &M, const PreservedAnalyses &PA,
     113             :     ModuleAnalysisManager::Invalidator &Inv) {
     114             :   // If literally everything is preserved, we're done.
     115             :   if (PA.areAllPreserved())
     116             :     return false; // This is still a valid proxy.
     117             : 
     118             :   // If this proxy or the call graph is going to be invalidated, we also need
     119             :   // to clear all the keys coming from that analysis.
     120             :   //
     121             :   // We also directly invalidate the FAM's module proxy if necessary, and if
     122             :   // that proxy isn't preserved we can't preserve this proxy either. We rely on
     123             :   // it to handle module -> function analysis invalidation in the face of
     124             :   // structural changes and so if it's unavailable we conservatively clear the
     125             :   // entire SCC layer as well rather than trying to do invalidation ourselves.
     126             :   auto PAC = PA.getChecker<CGSCCAnalysisManagerModuleProxy>();
     127         424 :   if (!(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Module>>()) ||
     128         401 :       Inv.invalidate<LazyCallGraphAnalysis>(M, PA) ||
     129             :       Inv.invalidate<FunctionAnalysisManagerModuleProxy>(M, PA)) {
     130          23 :     InnerAM->clear();
     131             : 
     132             :     // And the proxy itself should be marked as invalid so that we can observe
     133             :     // the new call graph. This isn't strictly necessary because we cheat
     134             :     // above, but is still useful.
     135          23 :     return true;
     136             :   }
     137             : 
     138             :   // Directly check if the relevant set is preserved so we can short circuit
     139             :   // invalidating SCCs below.
     140             :   bool AreSCCAnalysesPreserved =
     141             :       PA.allAnalysesInSetPreserved<AllAnalysesOn<LazyCallGraph::SCC>>();
     142             : 
     143             :   // Ok, we have a graph, so we can propagate the invalidation down into it.
     144         189 :   G->buildRefSCCs();
     145        1675 :   for (auto &RC : G->postorder_ref_sccs())
     146        1511 :     for (auto &C : RC) {
     147             :       Optional<PreservedAnalyses> InnerPA;
     148             : 
     149             :       // Check to see whether the preserved set needs to be adjusted based on
     150             :       // module-level analysis invalidation triggering deferred invalidation
     151             :       // for this SCC.
     152             :       if (auto *OuterProxy =
     153         768 :               InnerAM->getCachedResult<ModuleAnalysisManagerCGSCCProxy>(C))
     154             :         for (const auto &OuterInvalidationPair :
     155        1530 :              OuterProxy->getOuterInvalidations()) {
     156          14 :           AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first;
     157             :           const auto &InnerAnalysisIDs = OuterInvalidationPair.second;
     158          14 :           if (Inv.invalidate(OuterAnalysisID, M, PA)) {
     159          14 :             if (!InnerPA)
     160             :               InnerPA = PA;
     161          42 :             for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs)
     162          14 :               InnerPA->abandon(InnerAnalysisID);
     163             :           }
     164             :         }
     165             : 
     166             :       // Check if we needed a custom PA set. If so we'll need to run the inner
     167             :       // invalidation.
     168         782 :       if (InnerPA) {
     169          14 :         InnerAM->invalidate(C, *InnerPA);
     170             :         continue;
     171             :       }
     172             : 
     173             :       // Otherwise we only need to do invalidation if the original PA set didn't
     174             :       // preserve all SCC analyses.
     175         754 :       if (!AreSCCAnalysesPreserved)
     176          18 :         InnerAM->invalidate(C, PA);
     177             :     }
     178             : 
     179             :   // Return false to indicate that this result is still a valid proxy.
     180         189 :   return false;
     181             : }
     182             : 
     183             : template <>
     184             : CGSCCAnalysisManagerModuleProxy::Result
     185         268 : CGSCCAnalysisManagerModuleProxy::run(Module &M, ModuleAnalysisManager &AM) {
     186             :   // Force the Function analysis manager to also be available so that it can
     187             :   // be accessed in an SCC analysis and proxied onward to function passes.
     188             :   // FIXME: It is pretty awkward to just drop the result here and assert that
     189             :   // we can find it again later.
     190             :   (void)AM.getResult<FunctionAnalysisManagerModuleProxy>(M);
     191             : 
     192         268 :   return Result(*InnerAM, AM.getResult<LazyCallGraphAnalysis>(M));
     193             : }
     194             : 
     195             : AnalysisKey FunctionAnalysisManagerCGSCCProxy::Key;
     196             : 
     197             : FunctionAnalysisManagerCGSCCProxy::Result
     198        1073 : FunctionAnalysisManagerCGSCCProxy::run(LazyCallGraph::SCC &C,
     199             :                                        CGSCCAnalysisManager &AM,
     200             :                                        LazyCallGraph &CG) {
     201             :   // Collect the FunctionAnalysisManager from the Module layer and use that to
     202             :   // build the proxy result.
     203             :   //
     204             :   // This allows us to rely on the FunctionAnalysisMangaerModuleProxy to
     205             :   // invalidate the function analyses.
     206             :   auto &MAM = AM.getResult<ModuleAnalysisManagerCGSCCProxy>(C, CG).getManager();
     207        1073 :   Module &M = *C.begin()->getFunction().getParent();
     208             :   auto *FAMProxy = MAM.getCachedResult<FunctionAnalysisManagerModuleProxy>(M);
     209             :   assert(FAMProxy && "The CGSCC pass manager requires that the FAM module "
     210             :                      "proxy is run on the module prior to entering the CGSCC "
     211             :                      "walk.");
     212             : 
     213             :   // Note that we special-case invalidation handling of this proxy in the CGSCC
     214             :   // analysis manager's Module proxy. This avoids the need to do anything
     215             :   // special here to recompute all of this if ever the FAM's module proxy goes
     216             :   // away.
     217        1073 :   return Result(FAMProxy->getManager());
     218             : }
     219             : 
     220         750 : bool FunctionAnalysisManagerCGSCCProxy::Result::invalidate(
     221             :     LazyCallGraph::SCC &C, const PreservedAnalyses &PA,
     222             :     CGSCCAnalysisManager::Invalidator &Inv) {
     223             :   // If literally everything is preserved, we're done.
     224             :   if (PA.areAllPreserved())
     225             :     return false; // This is still a valid proxy.
     226             : 
     227             :   // If this proxy isn't marked as preserved, then even if the result remains
     228             :   // valid, the key itself may no longer be valid, so we clear everything.
     229             :   //
     230             :   // Note that in order to preserve this proxy, a module pass must ensure that
     231             :   // the FAM has been completely updated to handle the deletion of functions.
     232             :   // Specifically, any FAM-cached results for those functions need to have been
     233             :   // forcibly cleared. When preserved, this proxy will only invalidate results
     234             :   // cached on functions *still in the module* at the end of the module pass.
     235             :   auto PAC = PA.getChecker<FunctionAnalysisManagerCGSCCProxy>();
     236         750 :   if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<LazyCallGraph::SCC>>()) {
     237         511 :     for (LazyCallGraph::Node &N : C)
     238         265 :       FAM->clear(N.getFunction(), N.getFunction().getName());
     239             : 
     240             :     return true;
     241             :   }
     242             : 
     243             :   // Directly check if the relevant set is preserved.
     244             :   bool AreFunctionAnalysesPreserved =
     245             :       PA.allAnalysesInSetPreserved<AllAnalysesOn<Function>>();
     246             : 
     247             :   // Now walk all the functions to see if any inner analysis invalidation is
     248             :   // necessary.
     249        1227 :   for (LazyCallGraph::Node &N : C) {
     250         723 :     Function &F = N.getFunction();
     251             :     Optional<PreservedAnalyses> FunctionPA;
     252             : 
     253             :     // Check to see whether the preserved set needs to be pruned based on
     254             :     // SCC-level analysis invalidation that triggers deferred invalidation
     255             :     // registered with the outer analysis manager proxy for this function.
     256             :     if (auto *OuterProxy =
     257         723 :             FAM->getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F))
     258             :       for (const auto &OuterInvalidationPair :
     259          80 :            OuterProxy->getOuterInvalidations()) {
     260          10 :         AnalysisKey *OuterAnalysisID = OuterInvalidationPair.first;
     261             :         const auto &InnerAnalysisIDs = OuterInvalidationPair.second;
     262          10 :         if (Inv.invalidate(OuterAnalysisID, C, PA)) {
     263           9 :           if (!FunctionPA)
     264             :             FunctionPA = PA;
     265          27 :           for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs)
     266           9 :             FunctionPA->abandon(InnerAnalysisID);
     267             :         }
     268             :       }
     269             : 
     270             :     // Check if we needed a custom PA set, and if so we'll need to run the
     271             :     // inner invalidation.
     272         732 :     if (FunctionPA) {
     273           9 :       FAM->invalidate(F, *FunctionPA);
     274             :       continue;
     275             :     }
     276             : 
     277             :     // Otherwise we only need to do invalidation if the original PA set didn't
     278             :     // preserve all function analyses.
     279         714 :     if (!AreFunctionAnalysesPreserved)
     280         421 :       FAM->invalidate(F, PA);
     281             :   }
     282             : 
     283             :   // Return false to indicate that this result is still a valid proxy.
     284             :   return false;
     285             : }
     286             : 
     287             : } // end namespace llvm
     288             : 
     289             : /// When a new SCC is created for the graph and there might be function
     290             : /// analysis results cached for the functions now in that SCC two forms of
     291             : /// updates are required.
     292             : ///
     293             : /// First, a proxy from the SCC to the FunctionAnalysisManager needs to be
     294             : /// created so that any subsequent invalidation events to the SCC are
     295             : /// propagated to the function analysis results cached for functions within it.
     296             : ///
     297             : /// Second, if any of the functions within the SCC have analysis results with
     298             : /// outer analysis dependencies, then those dependencies would point to the
     299             : /// *wrong* SCC's analysis result. We forcibly invalidate the necessary
     300             : /// function analyses so that they don't retain stale handles.
     301          47 : static void updateNewSCCFunctionAnalyses(LazyCallGraph::SCC &C,
     302             :                                          LazyCallGraph &G,
     303             :                                          CGSCCAnalysisManager &AM) {
     304             :   // Get the relevant function analysis manager.
     305             :   auto &FAM =
     306             :       AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, G).getManager();
     307             : 
     308             :   // Now walk the functions in this SCC and invalidate any function analysis
     309             :   // results that might have outer dependencies on an SCC analysis.
     310         110 :   for (LazyCallGraph::Node &N : C) {
     311          63 :     Function &F = N.getFunction();
     312             : 
     313             :     auto *OuterProxy =
     314             :         FAM.getCachedResult<CGSCCAnalysisManagerFunctionProxy>(F);
     315          61 :     if (!OuterProxy)
     316             :       // No outer analyses were queried, nothing to do.
     317          61 :       continue;
     318             : 
     319             :     // Forcibly abandon all the inner analyses with dependencies, but
     320             :     // invalidate nothing else.
     321           2 :     auto PA = PreservedAnalyses::all();
     322             :     for (const auto &OuterInvalidationPair :
     323           6 :          OuterProxy->getOuterInvalidations()) {
     324             :       const auto &InnerAnalysisIDs = OuterInvalidationPair.second;
     325           6 :       for (AnalysisKey *InnerAnalysisID : InnerAnalysisIDs)
     326           2 :         PA.abandon(InnerAnalysisID);
     327             :     }
     328             : 
     329             :     // Now invalidate anything we found.
     330           2 :     FAM.invalidate(F, PA);
     331             :   }
     332          47 : }
     333             : 
     334             : /// Helper function to update both the \c CGSCCAnalysisManager \p AM and the \c
     335             : /// CGSCCPassManager's \c CGSCCUpdateResult \p UR based on a range of newly
     336             : /// added SCCs.
     337             : ///
     338             : /// The range of new SCCs must be in postorder already. The SCC they were split
     339             : /// out of must be provided as \p C. The current node being mutated and
     340             : /// triggering updates must be passed as \p N.
     341             : ///
     342             : /// This function returns the SCC containing \p N. This will be either \p C if
     343             : /// no new SCCs have been split out, or it will be the new SCC containing \p N.
     344             : template <typename SCCRangeT>
     345             : static LazyCallGraph::SCC *
     346          49 : incorporateNewSCCRange(const SCCRangeT &NewSCCRange, LazyCallGraph &G,
     347             :                        LazyCallGraph::Node &N, LazyCallGraph::SCC *C,
     348             :                        CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR) {
     349             :   using SCC = LazyCallGraph::SCC;
     350             : 
     351          49 :   if (NewSCCRange.begin() == NewSCCRange.end())
     352          16 :     return C;
     353             : 
     354             :   // Add the current SCC to the worklist as its shape has changed.
     355          33 :   UR.CWorklist.insert(C);
     356             :   LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist:" << *C
     357             :                     << "\n");
     358             : 
     359          33 :   SCC *OldC = C;
     360             : 
     361             :   // Update the current SCC. Note that if we have new SCCs, this must actually
     362             :   // change the SCC.
     363             :   assert(C != &*NewSCCRange.begin() &&
     364             :          "Cannot insert new SCCs without changing current SCC!");
     365          33 :   C = &*NewSCCRange.begin();
     366             :   assert(G.lookupSCC(N) == C && "Failed to update current SCC!");
     367             : 
     368             :   // If we had a cached FAM proxy originally, we will want to create more of
     369             :   // them for each SCC that was split off.
     370             :   bool NeedFAMProxy =
     371             :       AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(*OldC) != nullptr;
     372             : 
     373             :   // We need to propagate an invalidation call to all but the newly current SCC
     374             :   // because the outer pass manager won't do that for us after splitting them.
     375             :   // FIXME: We should accept a PreservedAnalysis from the CG updater so that if
     376             :   // there are preserved analysis we can avoid invalidating them here for
     377             :   // split-off SCCs.
     378             :   // We know however that this will preserve any FAM proxy so go ahead and mark
     379             :   // that.
     380          33 :   PreservedAnalyses PA;
     381             :   PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
     382          33 :   AM.invalidate(*OldC, PA);
     383             : 
     384             :   // Ensure the now-current SCC's function analyses are updated.
     385          33 :   if (NeedFAMProxy)
     386          33 :     updateNewSCCFunctionAnalyses(*C, G, AM);
     387             : 
     388          47 :   for (SCC &NewC : llvm::reverse(make_range(std::next(NewSCCRange.begin()),
     389             :                                             NewSCCRange.end()))) {
     390             :     assert(C != &NewC && "No need to re-visit the current SCC!");
     391             :     assert(OldC != &NewC && "Already handled the original SCC!");
     392          14 :     UR.CWorklist.insert(&NewC);
     393             :     LLVM_DEBUG(dbgs() << "Enqueuing a newly formed SCC:" << NewC << "\n");
     394             : 
     395             :     // Ensure new SCCs' function analyses are updated.
     396          14 :     if (NeedFAMProxy)
     397          14 :       updateNewSCCFunctionAnalyses(NewC, G, AM);
     398             : 
     399             :     // Also propagate a normal invalidation to the new SCC as only the current
     400             :     // will get one from the pass manager infrastructure.
     401          14 :     AM.invalidate(NewC, PA);
     402             :   }
     403          33 :   return C;
     404             : }
     405             : 
     406         422 : LazyCallGraph::SCC &llvm::updateCGAndAnalysisManagerForFunctionPass(
     407             :     LazyCallGraph &G, LazyCallGraph::SCC &InitialC, LazyCallGraph::Node &N,
     408             :     CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR) {
     409             :   using Node = LazyCallGraph::Node;
     410             :   using Edge = LazyCallGraph::Edge;
     411             :   using SCC = LazyCallGraph::SCC;
     412             :   using RefSCC = LazyCallGraph::RefSCC;
     413             : 
     414         422 :   RefSCC &InitialRC = InitialC.getOuterRefSCC();
     415         422 :   SCC *C = &InitialC;
     416         422 :   RefSCC *RC = &InitialRC;
     417         422 :   Function &F = N.getFunction();
     418             : 
     419             :   // Walk the function body and build up the set of retained, promoted, and
     420             :   // demoted edges.
     421             :   SmallVector<Constant *, 16> Worklist;
     422             :   SmallPtrSet<Constant *, 16> Visited;
     423             :   SmallPtrSet<Node *, 16> RetainedEdges;
     424             :   SmallSetVector<Node *, 4> PromotedRefTargets;
     425             :   SmallSetVector<Node *, 4> DemotedCallTargets;
     426             : 
     427             :   // First walk the function and handle all called functions. We do this first
     428             :   // because if there is a single call edge, whether there are ref edges is
     429             :   // irrelevant.
     430        4169 :   for (Instruction &I : instructions(F))
     431        4169 :     if (auto CS = CallSite(&I))
     432             :       if (Function *Callee = CS.getCalledFunction())
     433        1820 :         if (Visited.insert(Callee).second && !Callee->isDeclaration()) {
     434             :           Node &CalleeN = *G.lookup(*Callee);
     435         169 :           Edge *E = N->lookup(CalleeN);
     436             :           // FIXME: We should really handle adding new calls. While it will
     437             :           // make downstream usage more complex, there is no fundamental
     438             :           // limitation and it will allow passes within the CGSCC to be a bit
     439             :           // more flexible in what transforms they can do. Until then, we
     440             :           // verify that new calls haven't been introduced.
     441             :           assert(E && "No function transformations should introduce *new* "
     442             :                       "call edges! Any new calls should be modeled as "
     443             :                       "promoted existing ref edges!");
     444         169 :           bool Inserted = RetainedEdges.insert(&CalleeN).second;
     445             :           (void)Inserted;
     446             :           assert(Inserted && "We should never visit a function twice.");
     447         169 :           if (!E->isCall())
     448          58 :             PromotedRefTargets.insert(&CalleeN);
     449             :         }
     450             : 
     451             :   // Now walk all references.
     452        4169 :   for (Instruction &I : instructions(F))
     453       14807 :     for (Value *Op : I.operand_values())
     454        6469 :       if (auto *C = dyn_cast<Constant>(Op))
     455        3132 :         if (Visited.insert(C).second)
     456         741 :           Worklist.push_back(C);
     457             : 
     458          70 :   auto VisitRef = [&](Function &Referee) {
     459          70 :     Node &RefereeN = *G.lookup(Referee);
     460         140 :     Edge *E = N->lookup(RefereeN);
     461             :     // FIXME: Similarly to new calls, we also currently preclude
     462             :     // introducing new references. See above for details.
     463             :     assert(E && "No function transformations should introduce *new* ref "
     464             :                 "edges! Any new ref edges would require IPO which "
     465             :                 "function passes aren't allowed to do!");
     466          70 :     bool Inserted = RetainedEdges.insert(&RefereeN).second;
     467             :     (void)Inserted;
     468             :     assert(Inserted && "We should never visit a function twice.");
     469          70 :     if (E->isCall())
     470          15 :       DemotedCallTargets.insert(&RefereeN);
     471         492 :   };
     472         422 :   LazyCallGraph::visitReferences(Worklist, Visited, VisitRef);
     473             : 
     474             :   // Include synthetic reference edges to known, defined lib functions.
     475         464 :   for (auto *F : G.getLibFunctions())
     476             :     // While the list of lib functions doesn't have repeats, don't re-visit
     477             :     // anything handled above.
     478          21 :     if (!Visited.count(F))
     479          19 :       VisitRef(*F);
     480             : 
     481             :   // First remove all of the edges that are no longer present in this function.
     482             :   // The first step makes these edges uniformly ref edges and accumulates them
     483             :   // into a separate data structure so removal doesn't invalidate anything.
     484             :   SmallVector<Node *, 4> DeadTargets;
     485         963 :   for (Edge &E : *N) {
     486         541 :     if (RetainedEdges.count(&E.getNode()))
     487         239 :       continue;
     488             : 
     489             :     SCC &TargetC = *G.lookupSCC(E.getNode());
     490         302 :     RefSCC &TargetRC = TargetC.getOuterRefSCC();
     491         349 :     if (&TargetRC == RC && E.isCall()) {
     492          41 :       if (C != &TargetC) {
     493             :         // For separate SCCs this is trivial.
     494           2 :         RC->switchTrivialInternalEdgeToRef(N, E.getNode());
     495             :       } else {
     496             :         // Now update the call graph.
     497          39 :         C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, E.getNode()),
     498             :                                    G, N, C, AM, UR);
     499             :       }
     500             :     }
     501             : 
     502             :     // Now that this is ready for actual removal, put it into our list.
     503         302 :     DeadTargets.push_back(&E.getNode());
     504             :   }
     505             :   // Remove the easy cases quickly and actually pull them out of our list.
     506             :   DeadTargets.erase(
     507             :       llvm::remove_if(DeadTargets,
     508         302 :                       [&](Node *TargetN) {
     509         302 :                         SCC &TargetC = *G.lookupSCC(*TargetN);
     510         302 :                         RefSCC &TargetRC = TargetC.getOuterRefSCC();
     511             : 
     512             :                         // We can't trivially remove internal targets, so skip
     513             :                         // those.
     514         302 :                         if (&TargetRC == RC)
     515             :                           return false;
     516             : 
     517         255 :                         RC->removeOutgoingEdge(N, *TargetN);
     518             :                         LLVM_DEBUG(dbgs() << "Deleting outgoing edge from '"
     519             :                                           << N << "' to '" << TargetN << "'\n");
     520         255 :                         return true;
     521             :                       }),
     522             :       DeadTargets.end());
     523             : 
     524             :   // Now do a batch removal of the internal ref edges left.
     525         844 :   auto NewRefSCCs = RC->removeInternalRefEdge(N, DeadTargets);
     526         422 :   if (!NewRefSCCs.empty()) {
     527             :     // The old RefSCC is dead, mark it as such.
     528          19 :     UR.InvalidatedRefSCCs.insert(RC);
     529             : 
     530             :     // Note that we don't bother to invalidate analyses as ref-edge
     531             :     // connectivity is not really observable in any way and is intended
     532             :     // exclusively to be used for ordering of transforms rather than for
     533             :     // analysis conclusions.
     534             : 
     535             :     // Update RC to the "bottom".
     536             :     assert(G.lookupSCC(N) == C && "Changed the SCC when splitting RefSCCs!");
     537          19 :     RC = &C->getOuterRefSCC();
     538             :     assert(G.lookupRefSCC(N) == RC && "Failed to update current RefSCC!");
     539             : 
     540             :     // The RC worklist is in reverse postorder, so we enqueue the new ones in
     541             :     // RPO except for the one which contains the source node as that is the
     542             :     // "bottom" we will continue processing in the bottom-up walk.
     543             :     assert(NewRefSCCs.front() == RC &&
     544             :            "New current RefSCC not first in the returned list!");
     545             :     for (RefSCC *NewRC : llvm::reverse(make_range(std::next(NewRefSCCs.begin()),
     546          44 :                                                   NewRefSCCs.end()))) {
     547             :       assert(NewRC != RC && "Should not encounter the current RefSCC further "
     548             :                             "in the postorder list of new RefSCCs.");
     549          25 :       UR.RCWorklist.insert(NewRC);
     550             :       LLVM_DEBUG(dbgs() << "Enqueuing a new RefSCC in the update worklist: "
     551             :                         << *NewRC << "\n");
     552             :     }
     553             :   }
     554             : 
     555             :   // Next demote all the call edges that are now ref edges. This helps make
     556             :   // the SCCs small which should minimize the work below as we don't want to
     557             :   // form cycles that this would break.
     558         452 :   for (Node *RefTarget : DemotedCallTargets) {
     559             :     SCC &TargetC = *G.lookupSCC(*RefTarget);
     560          15 :     RefSCC &TargetRC = TargetC.getOuterRefSCC();
     561             : 
     562             :     // The easy case is when the target RefSCC is not this RefSCC. This is
     563             :     // only supported when the target RefSCC is a child of this RefSCC.
     564          20 :     if (&TargetRC != RC) {
     565             :       assert(RC->isAncestorOf(TargetRC) &&
     566             :              "Cannot potentially form RefSCC cycles here!");
     567           5 :       RC->switchOutgoingEdgeToRef(N, *RefTarget);
     568             :       LLVM_DEBUG(dbgs() << "Switch outgoing call edge to a ref edge from '" << N
     569             :                         << "' to '" << *RefTarget << "'\n");
     570           5 :       continue;
     571             :     }
     572             : 
     573             :     // We are switching an internal call edge to a ref edge. This may split up
     574             :     // some SCCs.
     575          10 :     if (C != &TargetC) {
     576             :       // For separate SCCs this is trivial.
     577           0 :       RC->switchTrivialInternalEdgeToRef(N, *RefTarget);
     578           0 :       continue;
     579             :     }
     580             : 
     581             :     // Now update the call graph.
     582          10 :     C = incorporateNewSCCRange(RC->switchInternalEdgeToRef(N, *RefTarget), G, N,
     583             :                                C, AM, UR);
     584             :   }
     585             : 
     586             :   // Now promote ref edges into call edges.
     587         538 :   for (Node *CallTarget : PromotedRefTargets) {
     588             :     SCC &TargetC = *G.lookupSCC(*CallTarget);
     589          58 :     RefSCC &TargetRC = TargetC.getOuterRefSCC();
     590             : 
     591             :     // The easy case is when the target RefSCC is not this RefSCC. This is
     592             :     // only supported when the target RefSCC is a child of this RefSCC.
     593          61 :     if (&TargetRC != RC) {
     594             :       assert(RC->isAncestorOf(TargetRC) &&
     595             :              "Cannot potentially form RefSCC cycles here!");
     596           3 :       RC->switchOutgoingEdgeToCall(N, *CallTarget);
     597             :       LLVM_DEBUG(dbgs() << "Switch outgoing ref edge to a call edge from '" << N
     598             :                         << "' to '" << *CallTarget << "'\n");
     599           3 :       continue;
     600             :     }
     601             :     LLVM_DEBUG(dbgs() << "Switch an internal ref edge to a call edge from '"
     602             :                       << N << "' to '" << *CallTarget << "'\n");
     603             : 
     604             :     // Otherwise we are switching an internal ref edge to a call edge. This
     605             :     // may merge away some SCCs, and we add those to the UpdateResult. We also
     606             :     // need to make sure to update the worklist in the event SCCs have moved
     607             :     // before the current one in the post-order sequence
     608          55 :     bool HasFunctionAnalysisProxy = false;
     609          55 :     auto InitialSCCIndex = RC->find(*C) - RC->begin();
     610         110 :     bool FormedCycle = RC->switchInternalEdgeToCall(
     611          22 :         N, *CallTarget, [&](ArrayRef<SCC *> MergedSCCs) {
     612          66 :           for (SCC *MergedC : MergedSCCs) {
     613             :             assert(MergedC != &TargetC && "Cannot merge away the target SCC!");
     614             : 
     615          22 :             HasFunctionAnalysisProxy |=
     616          44 :                 AM.getCachedResult<FunctionAnalysisManagerCGSCCProxy>(
     617          22 :                     *MergedC) != nullptr;
     618             : 
     619             :             // Mark that this SCC will no longer be valid.
     620          22 :             UR.InvalidatedSCCs.insert(MergedC);
     621             : 
     622             :             // FIXME: We should really do a 'clear' here to forcibly release
     623             :             // memory, but we don't have a good way of doing that and
     624             :             // preserving the function analyses.
     625          22 :             auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>();
     626             :             PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
     627          22 :             AM.invalidate(*MergedC, PA);
     628             :           }
     629          77 :         });
     630             : 
     631             :     // If we formed a cycle by creating this call, we need to update more data
     632             :     // structures.
     633          55 :     if (FormedCycle) {
     634          20 :       C = &TargetC;
     635             :       assert(G.lookupSCC(N) == C && "Failed to update current SCC!");
     636             : 
     637             :       // If one of the invalidated SCCs had a cached proxy to a function
     638             :       // analysis manager, we need to create a proxy in the new current SCC as
     639             :       // the invalidated SCCs had their functions moved.
     640          20 :       if (HasFunctionAnalysisProxy)
     641             :         AM.getResult<FunctionAnalysisManagerCGSCCProxy>(*C, G);
     642             : 
     643             :       // Any analyses cached for this SCC are no longer precise as the shape
     644             :       // has changed by introducing this cycle. However, we have taken care to
     645             :       // update the proxies so it remains valide.
     646          20 :       auto PA = PreservedAnalyses::allInSet<AllAnalysesOn<Function>>();
     647             :       PA.preserve<FunctionAnalysisManagerCGSCCProxy>();
     648          20 :       AM.invalidate(*C, PA);
     649             :     }
     650         110 :     auto NewSCCIndex = RC->find(*C) - RC->begin();
     651             :     // If we have actually moved an SCC to be topologically "below" the current
     652             :     // one due to merging, we will need to revisit the current SCC after
     653             :     // visiting those moved SCCs.
     654             :     //
     655             :     // It is critical that we *do not* revisit the current SCC unless we
     656             :     // actually move SCCs in the process of merging because otherwise we may
     657             :     // form a cycle where an SCC is split apart, merged, split, merged and so
     658             :     // on infinitely.
     659          55 :     if (InitialSCCIndex < NewSCCIndex) {
     660             :       // Put our current SCC back onto the worklist as we'll visit other SCCs
     661             :       // that are now definitively ordered prior to the current one in the
     662             :       // post-order sequence, and may end up observing more precise context to
     663             :       // optimize the current SCC.
     664           2 :       UR.CWorklist.insert(C);
     665             :       LLVM_DEBUG(dbgs() << "Enqueuing the existing SCC in the worklist: " << *C
     666             :                         << "\n");
     667             :       // Enqueue in reverse order as we pop off the back of the worklist.
     668             :       for (SCC &MovedC : llvm::reverse(make_range(RC->begin() + InitialSCCIndex,
     669           6 :                                                   RC->begin() + NewSCCIndex))) {
     670           2 :         UR.CWorklist.insert(&MovedC);
     671             :         LLVM_DEBUG(dbgs() << "Enqueuing a newly earlier in post-order SCC: "
     672             :                           << MovedC << "\n");
     673             :       }
     674             :     }
     675             :   }
     676             : 
     677             :   assert(!UR.InvalidatedSCCs.count(C) && "Invalidated the current SCC!");
     678             :   assert(!UR.InvalidatedRefSCCs.count(RC) && "Invalidated the current RefSCC!");
     679             :   assert(&C->getOuterRefSCC() == RC && "Current SCC not in current RefSCC!");
     680             : 
     681             :   // Record the current RefSCC and SCC for higher layers of the CGSCC pass
     682             :   // manager now that all the updates have been applied.
     683         422 :   if (RC != &InitialRC)
     684          19 :     UR.UpdatedRC = RC;
     685         422 :   if (C != &InitialC)
     686          37 :     UR.UpdatedC = C;
     687             : 
     688         422 :   return *C;
     689             : }

Generated by: LCOV version 1.13