LLVM  4.0.0
LazyCallGraph.cpp
Go to the documentation of this file.
1 //===- LazyCallGraph.cpp - Analysis of a Module's call graph --------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
11 #include "llvm/ADT/ScopeExit.h"
12 #include "llvm/ADT/Sequence.h"
13 #include "llvm/ADT/STLExtras.h"
14 #include "llvm/ADT/ScopeExit.h"
15 #include "llvm/IR/CallSite.h"
16 #include "llvm/IR/InstVisitor.h"
17 #include "llvm/IR/Instructions.h"
18 #include "llvm/IR/PassManager.h"
19 #include "llvm/Support/Debug.h"
21 
22 using namespace llvm;
23 
24 #define DEBUG_TYPE "lcg"
25 
27  DenseMap<Function *, int> &EdgeIndexMap, Function &F,
29  if (!EdgeIndexMap.insert({&F, Edges.size()}).second)
30  return;
31 
32  DEBUG(dbgs() << " Added callable function: " << F.getName() << "\n");
33  Edges.emplace_back(LazyCallGraph::Edge(F, EK));
34 }
35 
36 LazyCallGraph::Node::Node(LazyCallGraph &G, Function &F)
37  : G(&G), F(F), DFSNumber(0), LowLink(0) {
38  DEBUG(dbgs() << " Adding functions called by '" << F.getName()
39  << "' to the graph.\n");
40 
44 
45  // Find all the potential call graph edges in this function. We track both
46  // actual call edges and indirect references to functions. The direct calls
47  // are trivially added, but to accumulate the latter we walk the instructions
48  // and add every operand which is a constant to the worklist to process
49  // afterward.
50  //
51  // Note that we consider *any* function with a definition to be a viable
52  // edge. Even if the function's definition is subject to replacement by
53  // some other module (say, a weak definition) there may still be
54  // optimizations which essentially speculate based on the definition and
55  // a way to check that the specific definition is in fact the one being
56  // used. For example, this could be done by moving the weak definition to
57  // a strong (internal) definition and making the weak definition be an
58  // alias. Then a test of the address of the weak function against the new
59  // strong definition's address would be an effective way to determine the
60  // safety of optimizing a direct call edge.
61  for (BasicBlock &BB : F)
62  for (Instruction &I : BB) {
63  if (auto CS = CallSite(&I))
64  if (Function *Callee = CS.getCalledFunction())
65  if (!Callee->isDeclaration())
66  if (Callees.insert(Callee).second) {
67  Visited.insert(Callee);
68  addEdge(Edges, EdgeIndexMap, *Callee, LazyCallGraph::Edge::Call);
69  }
70 
71  for (Value *Op : I.operand_values())
72  if (Constant *C = dyn_cast<Constant>(Op))
73  if (Visited.insert(C).second)
74  Worklist.push_back(C);
75  }
76 
77  // We've collected all the constant (and thus potentially function or
78  // function containing) operands to all of the instructions in the function.
79  // Process them (recursively) collecting every function found.
80  visitReferences(Worklist, Visited, [&](Function &F) {
81  addEdge(Edges, EdgeIndexMap, F, LazyCallGraph::Edge::Ref);
82  });
83 }
84 
85 void LazyCallGraph::Node::insertEdgeInternal(Function &Target, Edge::Kind EK) {
86  if (Node *N = G->lookup(Target))
87  return insertEdgeInternal(*N, EK);
88 
89  EdgeIndexMap.insert({&Target, Edges.size()});
90  Edges.emplace_back(Target, EK);
91 }
92 
93 void LazyCallGraph::Node::insertEdgeInternal(Node &TargetN, Edge::Kind EK) {
94  EdgeIndexMap.insert({&TargetN.getFunction(), Edges.size()});
95  Edges.emplace_back(TargetN, EK);
96 }
97 
98 void LazyCallGraph::Node::setEdgeKind(Function &TargetF, Edge::Kind EK) {
99  Edges[EdgeIndexMap.find(&TargetF)->second].setKind(EK);
100 }
101 
102 void LazyCallGraph::Node::removeEdgeInternal(Function &Target) {
103  auto IndexMapI = EdgeIndexMap.find(&Target);
104  assert(IndexMapI != EdgeIndexMap.end() &&
105  "Target not in the edge set for this caller?");
106 
107  Edges[IndexMapI->second] = Edge();
108  EdgeIndexMap.erase(IndexMapI);
109 }
110 
111 void LazyCallGraph::Node::dump() const {
112  dbgs() << *this << '\n';
113 }
114 
115 LazyCallGraph::LazyCallGraph(Module &M) : NextDFSNumber(0) {
116  DEBUG(dbgs() << "Building CG for module: " << M.getModuleIdentifier()
117  << "\n");
118  for (Function &F : M)
119  if (!F.isDeclaration() && !F.hasLocalLinkage())
120  if (EntryIndexMap.insert({&F, EntryEdges.size()}).second) {
121  DEBUG(dbgs() << " Adding '" << F.getName()
122  << "' to entry set of the graph.\n");
123  EntryEdges.emplace_back(F, Edge::Ref);
124  }
125 
126  // Now add entry nodes for functions reachable via initializers to globals.
129  for (GlobalVariable &GV : M.globals())
130  if (GV.hasInitializer())
131  if (Visited.insert(GV.getInitializer()).second)
132  Worklist.push_back(GV.getInitializer());
133 
134  DEBUG(dbgs() << " Adding functions referenced by global initializers to the "
135  "entry set.\n");
136  visitReferences(Worklist, Visited, [&](Function &F) {
137  addEdge(EntryEdges, EntryIndexMap, F, LazyCallGraph::Edge::Ref);
138  });
139 
140  for (const Edge &E : EntryEdges)
141  RefSCCEntryNodes.push_back(&E.getFunction());
142 }
143 
145  : BPA(std::move(G.BPA)), NodeMap(std::move(G.NodeMap)),
146  EntryEdges(std::move(G.EntryEdges)),
147  EntryIndexMap(std::move(G.EntryIndexMap)), SCCBPA(std::move(G.SCCBPA)),
148  SCCMap(std::move(G.SCCMap)), LeafRefSCCs(std::move(G.LeafRefSCCs)),
149  DFSStack(std::move(G.DFSStack)),
150  RefSCCEntryNodes(std::move(G.RefSCCEntryNodes)),
151  NextDFSNumber(G.NextDFSNumber) {
152  updateGraphPtrs();
153 }
154 
156  BPA = std::move(G.BPA);
157  NodeMap = std::move(G.NodeMap);
158  EntryEdges = std::move(G.EntryEdges);
159  EntryIndexMap = std::move(G.EntryIndexMap);
160  SCCBPA = std::move(G.SCCBPA);
161  SCCMap = std::move(G.SCCMap);
162  LeafRefSCCs = std::move(G.LeafRefSCCs);
163  DFSStack = std::move(G.DFSStack);
164  RefSCCEntryNodes = std::move(G.RefSCCEntryNodes);
165  NextDFSNumber = G.NextDFSNumber;
166  updateGraphPtrs();
167  return *this;
168 }
169 
170 void LazyCallGraph::SCC::dump() const {
171  dbgs() << *this << '\n';
172 }
173 
174 #ifndef NDEBUG
175 void LazyCallGraph::SCC::verify() {
176  assert(OuterRefSCC && "Can't have a null RefSCC!");
177  assert(!Nodes.empty() && "Can't have an empty SCC!");
178 
179  for (Node *N : Nodes) {
180  assert(N && "Can't have a null node!");
181  assert(OuterRefSCC->G->lookupSCC(*N) == this &&
182  "Node does not map to this SCC!");
183  assert(N->DFSNumber == -1 &&
184  "Must set DFS numbers to -1 when adding a node to an SCC!");
185  assert(N->LowLink == -1 &&
186  "Must set low link to -1 when adding a node to an SCC!");
187  for (Edge &E : *N)
188  assert(E.getNode() && "Can't have an edge to a raw function!");
189  }
190 }
191 #endif
192 
194  if (this == &C)
195  return false;
196 
197  for (Node &N : *this)
198  for (Edge &E : N.calls())
199  if (Node *CalleeN = E.getNode())
200  if (OuterRefSCC->G->lookupSCC(*CalleeN) == &C)
201  return true;
202 
203  // No edges found.
204  return false;
205 }
206 
207 bool LazyCallGraph::SCC::isAncestorOf(const SCC &TargetC) const {
208  if (this == &TargetC)
209  return false;
210 
211  LazyCallGraph &G = *OuterRefSCC->G;
212 
213  // Start with this SCC.
214  SmallPtrSet<const SCC *, 16> Visited = {this};
215  SmallVector<const SCC *, 16> Worklist = {this};
216 
217  // Walk down the graph until we run out of edges or find a path to TargetC.
218  do {
219  const SCC &C = *Worklist.pop_back_val();
220  for (Node &N : C)
221  for (Edge &E : N.calls()) {
222  Node *CalleeN = E.getNode();
223  if (!CalleeN)
224  continue;
225  SCC *CalleeC = G.lookupSCC(*CalleeN);
226  if (!CalleeC)
227  continue;
228 
229  // If the callee's SCC is the TargetC, we're done.
230  if (CalleeC == &TargetC)
231  return true;
232 
233  // If this is the first time we've reached this SCC, put it on the
234  // worklist to recurse through.
235  if (Visited.insert(CalleeC).second)
236  Worklist.push_back(CalleeC);
237  }
238  } while (!Worklist.empty());
239 
240  // No paths found.
241  return false;
242 }
243 
244 LazyCallGraph::RefSCC::RefSCC(LazyCallGraph &G) : G(&G) {}
245 
246 void LazyCallGraph::RefSCC::dump() const {
247  dbgs() << *this << '\n';
248 }
249 
250 #ifndef NDEBUG
251 void LazyCallGraph::RefSCC::verify() {
252  assert(G && "Can't have a null graph!");
253  assert(!SCCs.empty() && "Can't have an empty SCC!");
254 
255  // Verify basic properties of the SCCs.
256  SmallPtrSet<SCC *, 4> SCCSet;
257  for (SCC *C : SCCs) {
258  assert(C && "Can't have a null SCC!");
259  C->verify();
260  assert(&C->getOuterRefSCC() == this &&
261  "SCC doesn't think it is inside this RefSCC!");
262  bool Inserted = SCCSet.insert(C).second;
263  assert(Inserted && "Found a duplicate SCC!");
264  auto IndexIt = SCCIndices.find(C);
265  assert(IndexIt != SCCIndices.end() &&
266  "Found an SCC that doesn't have an index!");
267  }
268 
269  // Check that our indices map correctly.
270  for (auto &SCCIndexPair : SCCIndices) {
271  SCC *C = SCCIndexPair.first;
272  int i = SCCIndexPair.second;
273  assert(C && "Can't have a null SCC in the indices!");
274  assert(SCCSet.count(C) && "Found an index for an SCC not in the RefSCC!");
275  assert(SCCs[i] == C && "Index doesn't point to SCC!");
276  }
277 
278  // Check that the SCCs are in fact in post-order.
279  for (int i = 0, Size = SCCs.size(); i < Size; ++i) {
280  SCC &SourceSCC = *SCCs[i];
281  for (Node &N : SourceSCC)
282  for (Edge &E : N) {
283  if (!E.isCall())
284  continue;
285  SCC &TargetSCC = *G->lookupSCC(*E.getNode());
286  if (&TargetSCC.getOuterRefSCC() == this) {
287  assert(SCCIndices.find(&TargetSCC)->second <= i &&
288  "Edge between SCCs violates post-order relationship.");
289  continue;
290  }
291  assert(TargetSCC.getOuterRefSCC().Parents.count(this) &&
292  "Edge to a RefSCC missing us in its parent set.");
293  }
294  }
295 
296  // Check that our parents are actually parents.
297  for (RefSCC *ParentRC : Parents) {
298  assert(ParentRC != this && "Cannot be our own parent!");
299  auto HasConnectingEdge = [&] {
300  for (SCC &C : *ParentRC)
301  for (Node &N : C)
302  for (Edge &E : N)
303  if (G->lookupRefSCC(*E.getNode()) == this)
304  return true;
305  return false;
306  };
307  assert(HasConnectingEdge() && "No edge connects the parent to us!");
308  }
309 }
310 #endif
311 
313  // Walk up the parents of this SCC and verify that we eventually find C.
314  SmallVector<const RefSCC *, 4> AncestorWorklist;
315  AncestorWorklist.push_back(this);
316  do {
317  const RefSCC *AncestorC = AncestorWorklist.pop_back_val();
318  if (AncestorC->isChildOf(C))
319  return true;
320  for (const RefSCC *ParentC : AncestorC->Parents)
321  AncestorWorklist.push_back(ParentC);
322  } while (!AncestorWorklist.empty());
323 
324  return false;
325 }
326 
327 /// Generic helper that updates a postorder sequence of SCCs for a potentially
328 /// cycle-introducing edge insertion.
329 ///
330 /// A postorder sequence of SCCs of a directed graph has one fundamental
331 /// property: all deges in the DAG of SCCs point "up" the sequence. That is,
332 /// all edges in the SCC DAG point to prior SCCs in the sequence.
333 ///
334 /// This routine both updates a postorder sequence and uses that sequence to
335 /// compute the set of SCCs connected into a cycle. It should only be called to
336 /// insert a "downward" edge which will require changing the sequence to
337 /// restore it to a postorder.
338 ///
339 /// When inserting an edge from an earlier SCC to a later SCC in some postorder
340 /// sequence, all of the SCCs which may be impacted are in the closed range of
341 /// those two within the postorder sequence. The algorithm used here to restore
342 /// the state is as follows:
343 ///
344 /// 1) Starting from the source SCC, construct a set of SCCs which reach the
345 /// source SCC consisting of just the source SCC. Then scan toward the
346 /// target SCC in postorder and for each SCC, if it has an edge to an SCC
347 /// in the set, add it to the set. Otherwise, the source SCC is not
348 /// a successor, move it in the postorder sequence to immediately before
349 /// the source SCC, shifting the source SCC and all SCCs in the set one
350 /// position toward the target SCC. Stop scanning after processing the
351 /// target SCC.
352 /// 2) If the source SCC is now past the target SCC in the postorder sequence,
353 /// and thus the new edge will flow toward the start, we are done.
354 /// 3) Otherwise, starting from the target SCC, walk all edges which reach an
355 /// SCC between the source and the target, and add them to the set of
356 /// connected SCCs, then recurse through them. Once a complete set of the
357 /// SCCs the target connects to is known, hoist the remaining SCCs between
358 /// the source and the target to be above the target. Note that there is no
359 /// need to process the source SCC, it is already known to connect.
360 /// 4) At this point, all of the SCCs in the closed range between the source
361 /// SCC and the target SCC in the postorder sequence are connected,
362 /// including the target SCC and the source SCC. Inserting the edge from
363 /// the source SCC to the target SCC will form a cycle out of precisely
364 /// these SCCs. Thus we can merge all of the SCCs in this closed range into
365 /// a single SCC.
366 ///
367 /// This process has various important properties:
368 /// - Only mutates the SCCs when adding the edge actually changes the SCC
369 /// structure.
370 /// - Never mutates SCCs which are unaffected by the change.
371 /// - Updates the postorder sequence to correctly satisfy the postorder
372 /// constraint after the edge is inserted.
373 /// - Only reorders SCCs in the closed postorder sequence from the source to
374 /// the target, so easy to bound how much has changed even in the ordering.
375 /// - Big-O is the number of edges in the closed postorder range of SCCs from
376 /// source to target.
377 ///
378 /// This helper routine, in addition to updating the postorder sequence itself
379 /// will also update a map from SCCs to indices within that sequecne.
380 ///
381 /// The sequence and the map must operate on pointers to the SCC type.
382 ///
383 /// Two callbacks must be provided. The first computes the subset of SCCs in
384 /// the postorder closed range from the source to the target which connect to
385 /// the source SCC via some (transitive) set of edges. The second computes the
386 /// subset of the same range which the target SCC connects to via some
387 /// (transitive) set of edges. Both callbacks should populate the set argument
388 /// provided.
389 template <typename SCCT, typename PostorderSequenceT, typename SCCIndexMapT,
390  typename ComputeSourceConnectedSetCallableT,
391  typename ComputeTargetConnectedSetCallableT>
394  SCCT &SourceSCC, SCCT &TargetSCC, PostorderSequenceT &SCCs,
395  SCCIndexMapT &SCCIndices,
396  ComputeSourceConnectedSetCallableT ComputeSourceConnectedSet,
397  ComputeTargetConnectedSetCallableT ComputeTargetConnectedSet) {
398  int SourceIdx = SCCIndices[&SourceSCC];
399  int TargetIdx = SCCIndices[&TargetSCC];
400  assert(SourceIdx < TargetIdx && "Cannot have equal indices here!");
401 
402  SmallPtrSet<SCCT *, 4> ConnectedSet;
403 
404  // Compute the SCCs which (transitively) reach the source.
405  ComputeSourceConnectedSet(ConnectedSet);
406 
407  // Partition the SCCs in this part of the port-order sequence so only SCCs
408  // connecting to the source remain between it and the target. This is
409  // a benign partition as it preserves postorder.
410  auto SourceI = std::stable_partition(
411  SCCs.begin() + SourceIdx, SCCs.begin() + TargetIdx + 1,
412  [&ConnectedSet](SCCT *C) { return !ConnectedSet.count(C); });
413  for (int i = SourceIdx, e = TargetIdx + 1; i < e; ++i)
414  SCCIndices.find(SCCs[i])->second = i;
415 
416  // If the target doesn't connect to the source, then we've corrected the
417  // post-order and there are no cycles formed.
418  if (!ConnectedSet.count(&TargetSCC)) {
419  assert(SourceI > (SCCs.begin() + SourceIdx) &&
420  "Must have moved the source to fix the post-order.");
421  assert(*std::prev(SourceI) == &TargetSCC &&
422  "Last SCC to move should have bene the target.");
423 
424  // Return an empty range at the target SCC indicating there is nothing to
425  // merge.
426  return make_range(std::prev(SourceI), std::prev(SourceI));
427  }
428 
429  assert(SCCs[TargetIdx] == &TargetSCC &&
430  "Should not have moved target if connected!");
431  SourceIdx = SourceI - SCCs.begin();
432  assert(SCCs[SourceIdx] == &SourceSCC &&
433  "Bad updated index computation for the source SCC!");
434 
435 
436  // See whether there are any remaining intervening SCCs between the source
437  // and target. If so we need to make sure they all are reachable form the
438  // target.
439  if (SourceIdx + 1 < TargetIdx) {
440  ConnectedSet.clear();
441  ComputeTargetConnectedSet(ConnectedSet);
442 
443  // Partition SCCs so that only SCCs reached from the target remain between
444  // the source and the target. This preserves postorder.
445  auto TargetI = std::stable_partition(
446  SCCs.begin() + SourceIdx + 1, SCCs.begin() + TargetIdx + 1,
447  [&ConnectedSet](SCCT *C) { return ConnectedSet.count(C); });
448  for (int i = SourceIdx + 1, e = TargetIdx + 1; i < e; ++i)
449  SCCIndices.find(SCCs[i])->second = i;
450  TargetIdx = std::prev(TargetI) - SCCs.begin();
451  assert(SCCs[TargetIdx] == &TargetSCC &&
452  "Should always end with the target!");
453  }
454 
455  // At this point, we know that connecting source to target forms a cycle
456  // because target connects back to source, and we know that all of the SCCs
457  // between the source and target in the postorder sequence participate in that
458  // cycle.
459  return make_range(SCCs.begin() + SourceIdx, SCCs.begin() + TargetIdx);
460 }
461 
464  assert(!SourceN[TargetN].isCall() && "Must start with a ref edge!");
465  SmallVector<SCC *, 1> DeletedSCCs;
466 
467 #ifndef NDEBUG
468  // In a debug build, verify the RefSCC is valid to start with and when this
469  // routine finishes.
470  verify();
471  auto VerifyOnExit = make_scope_exit([&]() { verify(); });
472 #endif
473 
474  SCC &SourceSCC = *G->lookupSCC(SourceN);
475  SCC &TargetSCC = *G->lookupSCC(TargetN);
476 
477  // If the two nodes are already part of the same SCC, we're also done as
478  // we've just added more connectivity.
479  if (&SourceSCC == &TargetSCC) {
480  SourceN.setEdgeKind(TargetN.getFunction(), Edge::Call);
481  return DeletedSCCs;
482  }
483 
484  // At this point we leverage the postorder list of SCCs to detect when the
485  // insertion of an edge changes the SCC structure in any way.
486  //
487  // First and foremost, we can eliminate the need for any changes when the
488  // edge is toward the beginning of the postorder sequence because all edges
489  // flow in that direction already. Thus adding a new one cannot form a cycle.
490  int SourceIdx = SCCIndices[&SourceSCC];
491  int TargetIdx = SCCIndices[&TargetSCC];
492  if (TargetIdx < SourceIdx) {
493  SourceN.setEdgeKind(TargetN.getFunction(), Edge::Call);
494  return DeletedSCCs;
495  }
496 
497  // Compute the SCCs which (transitively) reach the source.
498  auto ComputeSourceConnectedSet = [&](SmallPtrSetImpl<SCC *> &ConnectedSet) {
499 #ifndef NDEBUG
500  // Check that the RefSCC is still valid before computing this as the
501  // results will be nonsensical of we've broken its invariants.
502  verify();
503 #endif
504  ConnectedSet.insert(&SourceSCC);
505  auto IsConnected = [&](SCC &C) {
506  for (Node &N : C)
507  for (Edge &E : N.calls()) {
508  assert(E.getNode() && "Must have formed a node within an SCC!");
509  if (ConnectedSet.count(G->lookupSCC(*E.getNode())))
510  return true;
511  }
512 
513  return false;
514  };
515 
516  for (SCC *C :
517  make_range(SCCs.begin() + SourceIdx + 1, SCCs.begin() + TargetIdx + 1))
518  if (IsConnected(*C))
519  ConnectedSet.insert(C);
520  };
521 
522  // Use a normal worklist to find which SCCs the target connects to. We still
523  // bound the search based on the range in the postorder list we care about,
524  // but because this is forward connectivity we just "recurse" through the
525  // edges.
526  auto ComputeTargetConnectedSet = [&](SmallPtrSetImpl<SCC *> &ConnectedSet) {
527 #ifndef NDEBUG
528  // Check that the RefSCC is still valid before computing this as the
529  // results will be nonsensical of we've broken its invariants.
530  verify();
531 #endif
532  ConnectedSet.insert(&TargetSCC);
533  SmallVector<SCC *, 4> Worklist;
534  Worklist.push_back(&TargetSCC);
535  do {
536  SCC &C = *Worklist.pop_back_val();
537  for (Node &N : C)
538  for (Edge &E : N) {
539  assert(E.getNode() && "Must have formed a node within an SCC!");
540  if (!E.isCall())
541  continue;
542  SCC &EdgeC = *G->lookupSCC(*E.getNode());
543  if (&EdgeC.getOuterRefSCC() != this)
544  // Not in this RefSCC...
545  continue;
546  if (SCCIndices.find(&EdgeC)->second <= SourceIdx)
547  // Not in the postorder sequence between source and target.
548  continue;
549 
550  if (ConnectedSet.insert(&EdgeC).second)
551  Worklist.push_back(&EdgeC);
552  }
553  } while (!Worklist.empty());
554  };
555 
556  // Use a generic helper to update the postorder sequence of SCCs and return
557  // a range of any SCCs connected into a cycle by inserting this edge. This
558  // routine will also take care of updating the indices into the postorder
559  // sequence.
560  auto MergeRange = updatePostorderSequenceForEdgeInsertion(
561  SourceSCC, TargetSCC, SCCs, SCCIndices, ComputeSourceConnectedSet,
562  ComputeTargetConnectedSet);
563 
564  // If the merge range is empty, then adding the edge didn't actually form any
565  // new cycles. We're done.
566  if (MergeRange.begin() == MergeRange.end()) {
567  // Now that the SCC structure is finalized, flip the kind to call.
568  SourceN.setEdgeKind(TargetN.getFunction(), Edge::Call);
569  return DeletedSCCs;
570  }
571 
572 #ifndef NDEBUG
573  // Before merging, check that the RefSCC remains valid after all the
574  // postorder updates.
575  verify();
576 #endif
577 
578  // Otherwise we need to merge all of the SCCs in the cycle into a single
579  // result SCC.
580  //
581  // NB: We merge into the target because all of these functions were already
582  // reachable from the target, meaning any SCC-wide properties deduced about it
583  // other than the set of functions within it will not have changed.
584  for (SCC *C : MergeRange) {
585  assert(C != &TargetSCC &&
586  "We merge *into* the target and shouldn't process it here!");
587  SCCIndices.erase(C);
588  TargetSCC.Nodes.append(C->Nodes.begin(), C->Nodes.end());
589  for (Node *N : C->Nodes)
590  G->SCCMap[N] = &TargetSCC;
591  C->clear();
592  DeletedSCCs.push_back(C);
593  }
594 
595  // Erase the merged SCCs from the list and update the indices of the
596  // remaining SCCs.
597  int IndexOffset = MergeRange.end() - MergeRange.begin();
598  auto EraseEnd = SCCs.erase(MergeRange.begin(), MergeRange.end());
599  for (SCC *C : make_range(EraseEnd, SCCs.end()))
600  SCCIndices[C] -= IndexOffset;
601 
602  // Now that the SCC structure is finalized, flip the kind to call.
603  SourceN.setEdgeKind(TargetN.getFunction(), Edge::Call);
604 
605  // And we're done!
606  return DeletedSCCs;
607 }
608 
610  Node &TargetN) {
611  assert(SourceN[TargetN].isCall() && "Must start with a call edge!");
612 
613 #ifndef NDEBUG
614  // In a debug build, verify the RefSCC is valid to start with and when this
615  // routine finishes.
616  verify();
617  auto VerifyOnExit = make_scope_exit([&]() { verify(); });
618 #endif
619 
620  assert(G->lookupRefSCC(SourceN) == this &&
621  "Source must be in this RefSCC.");
622  assert(G->lookupRefSCC(TargetN) == this &&
623  "Target must be in this RefSCC.");
624  assert(G->lookupSCC(SourceN) != G->lookupSCC(TargetN) &&
625  "Source and Target must be in separate SCCs for this to be trivial!");
626 
627  // Set the edge kind.
628  SourceN.setEdgeKind(TargetN.getFunction(), Edge::Ref);
629 }
630 
633  assert(SourceN[TargetN].isCall() && "Must start with a call edge!");
634 
635 #ifndef NDEBUG
636  // In a debug build, verify the RefSCC is valid to start with and when this
637  // routine finishes.
638  verify();
639  auto VerifyOnExit = make_scope_exit([&]() { verify(); });
640 #endif
641 
642  assert(G->lookupRefSCC(SourceN) == this &&
643  "Source must be in this RefSCC.");
644  assert(G->lookupRefSCC(TargetN) == this &&
645  "Target must be in this RefSCC.");
646 
647  SCC &TargetSCC = *G->lookupSCC(TargetN);
648  assert(G->lookupSCC(SourceN) == &TargetSCC && "Source and Target must be in "
649  "the same SCC to require the "
650  "full CG update.");
651 
652  // Set the edge kind.
653  SourceN.setEdgeKind(TargetN.getFunction(), Edge::Ref);
654 
655  // Otherwise we are removing a call edge from a single SCC. This may break
656  // the cycle. In order to compute the new set of SCCs, we need to do a small
657  // DFS over the nodes within the SCC to form any sub-cycles that remain as
658  // distinct SCCs and compute a postorder over the resulting SCCs.
659  //
660  // However, we specially handle the target node. The target node is known to
661  // reach all other nodes in the original SCC by definition. This means that
662  // we want the old SCC to be replaced with an SCC contaning that node as it
663  // will be the root of whatever SCC DAG results from the DFS. Assumptions
664  // about an SCC such as the set of functions called will continue to hold,
665  // etc.
666 
667  SCC &OldSCC = TargetSCC;
669  SmallVector<Node *, 16> PendingSCCStack;
670  SmallVector<SCC *, 4> NewSCCs;
671 
672  // Prepare the nodes for a fresh DFS.
673  SmallVector<Node *, 16> Worklist;
674  Worklist.swap(OldSCC.Nodes);
675  for (Node *N : Worklist) {
676  N->DFSNumber = N->LowLink = 0;
677  G->SCCMap.erase(N);
678  }
679 
680  // Force the target node to be in the old SCC. This also enables us to take
681  // a very significant short-cut in the standard Tarjan walk to re-form SCCs
682  // below: whenever we build an edge that reaches the target node, we know
683  // that the target node eventually connects back to all other nodes in our
684  // walk. As a consequence, we can detect and handle participants in that
685  // cycle without walking all the edges that form this connection, and instead
686  // by relying on the fundamental guarantee coming into this operation (all
687  // nodes are reachable from the target due to previously forming an SCC).
688  TargetN.DFSNumber = TargetN.LowLink = -1;
689  OldSCC.Nodes.push_back(&TargetN);
690  G->SCCMap[&TargetN] = &OldSCC;
691 
692  // Scan down the stack and DFS across the call edges.
693  for (Node *RootN : Worklist) {
694  assert(DFSStack.empty() &&
695  "Cannot begin a new root with a non-empty DFS stack!");
696  assert(PendingSCCStack.empty() &&
697  "Cannot begin a new root with pending nodes for an SCC!");
698 
699  // Skip any nodes we've already reached in the DFS.
700  if (RootN->DFSNumber != 0) {
701  assert(RootN->DFSNumber == -1 &&
702  "Shouldn't have any mid-DFS root nodes!");
703  continue;
704  }
705 
706  RootN->DFSNumber = RootN->LowLink = 1;
707  int NextDFSNumber = 2;
708 
709  DFSStack.push_back({RootN, RootN->call_begin()});
710  do {
711  Node *N;
713  std::tie(N, I) = DFSStack.pop_back_val();
714  auto E = N->call_end();
715  while (I != E) {
716  Node &ChildN = *I->getNode();
717  if (ChildN.DFSNumber == 0) {
718  // We haven't yet visited this child, so descend, pushing the current
719  // node onto the stack.
720  DFSStack.push_back({N, I});
721 
722  assert(!G->SCCMap.count(&ChildN) &&
723  "Found a node with 0 DFS number but already in an SCC!");
724  ChildN.DFSNumber = ChildN.LowLink = NextDFSNumber++;
725  N = &ChildN;
726  I = N->call_begin();
727  E = N->call_end();
728  continue;
729  }
730 
731  // Check for the child already being part of some component.
732  if (ChildN.DFSNumber == -1) {
733  if (G->lookupSCC(ChildN) == &OldSCC) {
734  // If the child is part of the old SCC, we know that it can reach
735  // every other node, so we have formed a cycle. Pull the entire DFS
736  // and pending stacks into it. See the comment above about setting
737  // up the old SCC for why we do this.
738  int OldSize = OldSCC.size();
739  OldSCC.Nodes.push_back(N);
740  OldSCC.Nodes.append(PendingSCCStack.begin(), PendingSCCStack.end());
741  PendingSCCStack.clear();
742  while (!DFSStack.empty())
743  OldSCC.Nodes.push_back(DFSStack.pop_back_val().first);
744  for (Node &N : make_range(OldSCC.begin() + OldSize, OldSCC.end())) {
745  N.DFSNumber = N.LowLink = -1;
746  G->SCCMap[&N] = &OldSCC;
747  }
748  N = nullptr;
749  break;
750  }
751 
752  // If the child has already been added to some child component, it
753  // couldn't impact the low-link of this parent because it isn't
754  // connected, and thus its low-link isn't relevant so skip it.
755  ++I;
756  continue;
757  }
758 
759  // Track the lowest linked child as the lowest link for this node.
760  assert(ChildN.LowLink > 0 && "Must have a positive low-link number!");
761  if (ChildN.LowLink < N->LowLink)
762  N->LowLink = ChildN.LowLink;
763 
764  // Move to the next edge.
765  ++I;
766  }
767  if (!N)
768  // Cleared the DFS early, start another round.
769  break;
770 
771  // We've finished processing N and its descendents, put it on our pending
772  // SCC stack to eventually get merged into an SCC of nodes.
773  PendingSCCStack.push_back(N);
774 
775  // If this node is linked to some lower entry, continue walking up the
776  // stack.
777  if (N->LowLink != N->DFSNumber)
778  continue;
779 
780  // Otherwise, we've completed an SCC. Append it to our post order list of
781  // SCCs.
782  int RootDFSNumber = N->DFSNumber;
783  // Find the range of the node stack by walking down until we pass the
784  // root DFS number.
785  auto SCCNodes = make_range(
786  PendingSCCStack.rbegin(),
787  find_if(reverse(PendingSCCStack), [RootDFSNumber](const Node *N) {
788  return N->DFSNumber < RootDFSNumber;
789  }));
790 
791  // Form a new SCC out of these nodes and then clear them off our pending
792  // stack.
793  NewSCCs.push_back(G->createSCC(*this, SCCNodes));
794  for (Node &N : *NewSCCs.back()) {
795  N.DFSNumber = N.LowLink = -1;
796  G->SCCMap[&N] = NewSCCs.back();
797  }
798  PendingSCCStack.erase(SCCNodes.end().base(), PendingSCCStack.end());
799  } while (!DFSStack.empty());
800  }
801 
802  // Insert the remaining SCCs before the old one. The old SCC can reach all
803  // other SCCs we form because it contains the target node of the removed edge
804  // of the old SCC. This means that we will have edges into all of the new
805  // SCCs, which means the old one must come last for postorder.
806  int OldIdx = SCCIndices[&OldSCC];
807  SCCs.insert(SCCs.begin() + OldIdx, NewSCCs.begin(), NewSCCs.end());
808 
809  // Update the mapping from SCC* to index to use the new SCC*s, and remove the
810  // old SCC from the mapping.
811  for (int Idx = OldIdx, Size = SCCs.size(); Idx < Size; ++Idx)
812  SCCIndices[SCCs[Idx]] = Idx;
813 
814  return make_range(SCCs.begin() + OldIdx,
815  SCCs.begin() + OldIdx + NewSCCs.size());
816 }
817 
819  Node &TargetN) {
820  assert(!SourceN[TargetN].isCall() && "Must start with a ref edge!");
821 
822  assert(G->lookupRefSCC(SourceN) == this && "Source must be in this RefSCC.");
823  assert(G->lookupRefSCC(TargetN) != this &&
824  "Target must not be in this RefSCC.");
825  assert(G->lookupRefSCC(TargetN)->isDescendantOf(*this) &&
826  "Target must be a descendant of the Source.");
827 
828  // Edges between RefSCCs are the same regardless of call or ref, so we can
829  // just flip the edge here.
830  SourceN.setEdgeKind(TargetN.getFunction(), Edge::Call);
831 
832 #ifndef NDEBUG
833  // Check that the RefSCC is still valid.
834  verify();
835 #endif
836 }
837 
839  Node &TargetN) {
840  assert(SourceN[TargetN].isCall() && "Must start with a call edge!");
841 
842  assert(G->lookupRefSCC(SourceN) == this && "Source must be in this RefSCC.");
843  assert(G->lookupRefSCC(TargetN) != this &&
844  "Target must not be in this RefSCC.");
845  assert(G->lookupRefSCC(TargetN)->isDescendantOf(*this) &&
846  "Target must be a descendant of the Source.");
847 
848  // Edges between RefSCCs are the same regardless of call or ref, so we can
849  // just flip the edge here.
850  SourceN.setEdgeKind(TargetN.getFunction(), Edge::Ref);
851 
852 #ifndef NDEBUG
853  // Check that the RefSCC is still valid.
854  verify();
855 #endif
856 }
857 
859  Node &TargetN) {
860  assert(G->lookupRefSCC(SourceN) == this && "Source must be in this RefSCC.");
861  assert(G->lookupRefSCC(TargetN) == this && "Target must be in this RefSCC.");
862 
863  SourceN.insertEdgeInternal(TargetN, Edge::Ref);
864 
865 #ifndef NDEBUG
866  // Check that the RefSCC is still valid.
867  verify();
868 #endif
869 }
870 
872  Edge::Kind EK) {
873  // First insert it into the caller.
874  SourceN.insertEdgeInternal(TargetN, EK);
875 
876  assert(G->lookupRefSCC(SourceN) == this && "Source must be in this RefSCC.");
877 
878  RefSCC &TargetC = *G->lookupRefSCC(TargetN);
879  assert(&TargetC != this && "Target must not be in this RefSCC.");
880  assert(TargetC.isDescendantOf(*this) &&
881  "Target must be a descendant of the Source.");
882 
883  // The only change required is to add this SCC to the parent set of the
884  // callee.
885  TargetC.Parents.insert(this);
886 
887 #ifndef NDEBUG
888  // Check that the RefSCC is still valid.
889  verify();
890 #endif
891 }
892 
895  assert(G->lookupRefSCC(TargetN) == this && "Target must be in this RefSCC.");
896  RefSCC &SourceC = *G->lookupRefSCC(SourceN);
897  assert(&SourceC != this && "Source must not be in this RefSCC.");
898  assert(SourceC.isDescendantOf(*this) &&
899  "Source must be a descendant of the Target.");
900 
901  SmallVector<RefSCC *, 1> DeletedRefSCCs;
902 
903 #ifndef NDEBUG
904  // In a debug build, verify the RefSCC is valid to start with and when this
905  // routine finishes.
906  verify();
907  auto VerifyOnExit = make_scope_exit([&]() { verify(); });
908 #endif
909 
910  int SourceIdx = G->RefSCCIndices[&SourceC];
911  int TargetIdx = G->RefSCCIndices[this];
912  assert(SourceIdx < TargetIdx &&
913  "Postorder list doesn't see edge as incoming!");
914 
915  // Compute the RefSCCs which (transitively) reach the source. We do this by
916  // working backwards from the source using the parent set in each RefSCC,
917  // skipping any RefSCCs that don't fall in the postorder range. This has the
918  // advantage of walking the sparser parent edge (in high fan-out graphs) but
919  // more importantly this removes examining all forward edges in all RefSCCs
920  // within the postorder range which aren't in fact connected. Only connected
921  // RefSCCs (and their edges) are visited here.
922  auto ComputeSourceConnectedSet = [&](SmallPtrSetImpl<RefSCC *> &Set) {
923  Set.insert(&SourceC);
924  SmallVector<RefSCC *, 4> Worklist;
925  Worklist.push_back(&SourceC);
926  do {
927  RefSCC &RC = *Worklist.pop_back_val();
928  for (RefSCC &ParentRC : RC.parents()) {
929  // Skip any RefSCCs outside the range of source to target in the
930  // postorder sequence.
931  int ParentIdx = G->getRefSCCIndex(ParentRC);
932  assert(ParentIdx > SourceIdx && "Parent cannot precede source in postorder!");
933  if (ParentIdx > TargetIdx)
934  continue;
935  if (Set.insert(&ParentRC).second)
936  // First edge connecting to this parent, add it to our worklist.
937  Worklist.push_back(&ParentRC);
938  }
939  } while (!Worklist.empty());
940  };
941 
942  // Use a normal worklist to find which SCCs the target connects to. We still
943  // bound the search based on the range in the postorder list we care about,
944  // but because this is forward connectivity we just "recurse" through the
945  // edges.
946  auto ComputeTargetConnectedSet = [&](SmallPtrSetImpl<RefSCC *> &Set) {
947  Set.insert(this);
948  SmallVector<RefSCC *, 4> Worklist;
949  Worklist.push_back(this);
950  do {
951  RefSCC &RC = *Worklist.pop_back_val();
952  for (SCC &C : RC)
953  for (Node &N : C)
954  for (Edge &E : N) {
955  assert(E.getNode() && "Must have formed a node!");
956  RefSCC &EdgeRC = *G->lookupRefSCC(*E.getNode());
957  if (G->getRefSCCIndex(EdgeRC) <= SourceIdx)
958  // Not in the postorder sequence between source and target.
959  continue;
960 
961  if (Set.insert(&EdgeRC).second)
962  Worklist.push_back(&EdgeRC);
963  }
964  } while (!Worklist.empty());
965  };
966 
967  // Use a generic helper to update the postorder sequence of RefSCCs and return
968  // a range of any RefSCCs connected into a cycle by inserting this edge. This
969  // routine will also take care of updating the indices into the postorder
970  // sequence.
973  SourceC, *this, G->PostOrderRefSCCs, G->RefSCCIndices,
974  ComputeSourceConnectedSet, ComputeTargetConnectedSet);
975 
976  // Build a set so we can do fast tests for whether a RefSCC will end up as
977  // part of the merged RefSCC.
978  SmallPtrSet<RefSCC *, 16> MergeSet(MergeRange.begin(), MergeRange.end());
979 
980  // This RefSCC will always be part of that set, so just insert it here.
981  MergeSet.insert(this);
982 
983  // Now that we have identified all of the SCCs which need to be merged into
984  // a connected set with the inserted edge, merge all of them into this SCC.
985  SmallVector<SCC *, 16> MergedSCCs;
986  int SCCIndex = 0;
987  for (RefSCC *RC : MergeRange) {
988  assert(RC != this && "We're merging into the target RefSCC, so it "
989  "shouldn't be in the range.");
990 
991  // Merge the parents which aren't part of the merge into the our parents.
992  for (RefSCC *ParentRC : RC->Parents)
993  if (!MergeSet.count(ParentRC))
994  Parents.insert(ParentRC);
995  RC->Parents.clear();
996 
997  // Walk the inner SCCs to update their up-pointer and walk all the edges to
998  // update any parent sets.
999  // FIXME: We should try to find a way to avoid this (rather expensive) edge
1000  // walk by updating the parent sets in some other manner.
1001  for (SCC &InnerC : *RC) {
1002  InnerC.OuterRefSCC = this;
1003  SCCIndices[&InnerC] = SCCIndex++;
1004  for (Node &N : InnerC) {
1005  G->SCCMap[&N] = &InnerC;
1006  for (Edge &E : N) {
1007  assert(E.getNode() &&
1008  "Cannot have a null node within a visited SCC!");
1009  RefSCC &ChildRC = *G->lookupRefSCC(*E.getNode());
1010  if (MergeSet.count(&ChildRC))
1011  continue;
1012  ChildRC.Parents.erase(RC);
1013  ChildRC.Parents.insert(this);
1014  }
1015  }
1016  }
1017 
1018  // Now merge in the SCCs. We can actually move here so try to reuse storage
1019  // the first time through.
1020  if (MergedSCCs.empty())
1021  MergedSCCs = std::move(RC->SCCs);
1022  else
1023  MergedSCCs.append(RC->SCCs.begin(), RC->SCCs.end());
1024  RC->SCCs.clear();
1025  DeletedRefSCCs.push_back(RC);
1026  }
1027 
1028  // Append our original SCCs to the merged list and move it into place.
1029  for (SCC &InnerC : *this)
1030  SCCIndices[&InnerC] = SCCIndex++;
1031  MergedSCCs.append(SCCs.begin(), SCCs.end());
1032  SCCs = std::move(MergedSCCs);
1033 
1034  // Remove the merged away RefSCCs from the post order sequence.
1035  for (RefSCC *RC : MergeRange)
1036  G->RefSCCIndices.erase(RC);
1037  int IndexOffset = MergeRange.end() - MergeRange.begin();
1038  auto EraseEnd =
1039  G->PostOrderRefSCCs.erase(MergeRange.begin(), MergeRange.end());
1040  for (RefSCC *RC : make_range(EraseEnd, G->PostOrderRefSCCs.end()))
1041  G->RefSCCIndices[RC] -= IndexOffset;
1042 
1043  // At this point we have a merged RefSCC with a post-order SCCs list, just
1044  // connect the nodes to form the new edge.
1045  SourceN.insertEdgeInternal(TargetN, Edge::Ref);
1046 
1047  // We return the list of SCCs which were merged so that callers can
1048  // invalidate any data they have associated with those SCCs. Note that these
1049  // SCCs are no longer in an interesting state (they are totally empty) but
1050  // the pointers will remain stable for the life of the graph itself.
1051  return DeletedRefSCCs;
1052 }
1053 
1055  assert(G->lookupRefSCC(SourceN) == this &&
1056  "The source must be a member of this RefSCC.");
1057 
1058  RefSCC &TargetRC = *G->lookupRefSCC(TargetN);
1059  assert(&TargetRC != this && "The target must not be a member of this RefSCC");
1060 
1061  assert(!is_contained(G->LeafRefSCCs, this) &&
1062  "Cannot have a leaf RefSCC source.");
1063 
1064 #ifndef NDEBUG
1065  // In a debug build, verify the RefSCC is valid to start with and when this
1066  // routine finishes.
1067  verify();
1068  auto VerifyOnExit = make_scope_exit([&]() { verify(); });
1069 #endif
1070 
1071  // First remove it from the node.
1072  SourceN.removeEdgeInternal(TargetN.getFunction());
1073 
1074  bool HasOtherEdgeToChildRC = false;
1075  bool HasOtherChildRC = false;
1076  for (SCC *InnerC : SCCs) {
1077  for (Node &N : *InnerC) {
1078  for (Edge &E : N) {
1079  assert(E.getNode() && "Cannot have a missing node in a visited SCC!");
1080  RefSCC &OtherChildRC = *G->lookupRefSCC(*E.getNode());
1081  if (&OtherChildRC == &TargetRC) {
1082  HasOtherEdgeToChildRC = true;
1083  break;
1084  }
1085  if (&OtherChildRC != this)
1086  HasOtherChildRC = true;
1087  }
1088  if (HasOtherEdgeToChildRC)
1089  break;
1090  }
1091  if (HasOtherEdgeToChildRC)
1092  break;
1093  }
1094  // Because the SCCs form a DAG, deleting such an edge cannot change the set
1095  // of SCCs in the graph. However, it may cut an edge of the SCC DAG, making
1096  // the source SCC no longer connected to the target SCC. If so, we need to
1097  // update the target SCC's map of its parents.
1098  if (!HasOtherEdgeToChildRC) {
1099  bool Removed = TargetRC.Parents.erase(this);
1100  (void)Removed;
1101  assert(Removed &&
1102  "Did not find the source SCC in the target SCC's parent list!");
1103 
1104  // It may orphan an SCC if it is the last edge reaching it, but that does
1105  // not violate any invariants of the graph.
1106  if (TargetRC.Parents.empty())
1107  DEBUG(dbgs() << "LCG: Update removing " << SourceN.getFunction().getName()
1108  << " -> " << TargetN.getFunction().getName()
1109  << " edge orphaned the callee's SCC!\n");
1110 
1111  // It may make the Source SCC a leaf SCC.
1112  if (!HasOtherChildRC)
1113  G->LeafRefSCCs.push_back(this);
1114  }
1115 }
1116 
1119  assert(!SourceN[TargetN].isCall() &&
1120  "Cannot remove a call edge, it must first be made a ref edge");
1121 
1122 #ifndef NDEBUG
1123  // In a debug build, verify the RefSCC is valid to start with and when this
1124  // routine finishes.
1125  verify();
1126  auto VerifyOnExit = make_scope_exit([&]() { verify(); });
1127 #endif
1128 
1129  // First remove the actual edge.
1130  SourceN.removeEdgeInternal(TargetN.getFunction());
1131 
1132  // We return a list of the resulting *new* RefSCCs in post-order.
1133  SmallVector<RefSCC *, 1> Result;
1134 
1135  // Direct recursion doesn't impact the SCC graph at all.
1136  if (&SourceN == &TargetN)
1137  return Result;
1138 
1139  // If this ref edge is within an SCC then there are sufficient other edges to
1140  // form a cycle without this edge so removing it is a no-op.
1141  SCC &SourceC = *G->lookupSCC(SourceN);
1142  SCC &TargetC = *G->lookupSCC(TargetN);
1143  if (&SourceC == &TargetC)
1144  return Result;
1145 
1146  // We build somewhat synthetic new RefSCCs by providing a postorder mapping
1147  // for each inner SCC. We also store these associated with *nodes* rather
1148  // than SCCs because this saves a round-trip through the node->SCC map and in
1149  // the common case, SCCs are small. We will verify that we always give the
1150  // same number to every node in the SCC such that these are equivalent.
1151  const int RootPostOrderNumber = 0;
1152  int PostOrderNumber = RootPostOrderNumber + 1;
1153  SmallDenseMap<Node *, int> PostOrderMapping;
1154 
1155  // Every node in the target SCC can already reach every node in this RefSCC
1156  // (by definition). It is the only node we know will stay inside this RefSCC.
1157  // Everything which transitively reaches Target will also remain in the
1158  // RefSCC. We handle this by pre-marking that the nodes in the target SCC map
1159  // back to the root post order number.
1160  //
1161  // This also enables us to take a very significant short-cut in the standard
1162  // Tarjan walk to re-form RefSCCs below: whenever we build an edge that
1163  // references the target node, we know that the target node eventually
1164  // references all other nodes in our walk. As a consequence, we can detect
1165  // and handle participants in that cycle without walking all the edges that
1166  // form the connections, and instead by relying on the fundamental guarantee
1167  // coming into this operation.
1168  for (Node &N : TargetC)
1169  PostOrderMapping[&N] = RootPostOrderNumber;
1170 
1171  // Reset all the other nodes to prepare for a DFS over them, and add them to
1172  // our worklist.
1173  SmallVector<Node *, 8> Worklist;
1174  for (SCC *C : SCCs) {
1175  if (C == &TargetC)
1176  continue;
1177 
1178  for (Node &N : *C)
1179  N.DFSNumber = N.LowLink = 0;
1180 
1181  Worklist.append(C->Nodes.begin(), C->Nodes.end());
1182  }
1183 
1184  auto MarkNodeForSCCNumber = [&PostOrderMapping](Node &N, int Number) {
1185  N.DFSNumber = N.LowLink = -1;
1186  PostOrderMapping[&N] = Number;
1187  };
1188 
1190  SmallVector<Node *, 4> PendingRefSCCStack;
1191  do {
1192  assert(DFSStack.empty() &&
1193  "Cannot begin a new root with a non-empty DFS stack!");
1194  assert(PendingRefSCCStack.empty() &&
1195  "Cannot begin a new root with pending nodes for an SCC!");
1196 
1197  Node *RootN = Worklist.pop_back_val();
1198  // Skip any nodes we've already reached in the DFS.
1199  if (RootN->DFSNumber != 0) {
1200  assert(RootN->DFSNumber == -1 &&
1201  "Shouldn't have any mid-DFS root nodes!");
1202  continue;
1203  }
1204 
1205  RootN->DFSNumber = RootN->LowLink = 1;
1206  int NextDFSNumber = 2;
1207 
1208  DFSStack.push_back({RootN, RootN->begin()});
1209  do {
1210  Node *N;
1211  edge_iterator I;
1212  std::tie(N, I) = DFSStack.pop_back_val();
1213  auto E = N->end();
1214 
1215  assert(N->DFSNumber != 0 && "We should always assign a DFS number "
1216  "before processing a node.");
1217 
1218  while (I != E) {
1219  Node &ChildN = I->getNode(*G);
1220  if (ChildN.DFSNumber == 0) {
1221  // Mark that we should start at this child when next this node is the
1222  // top of the stack. We don't start at the next child to ensure this
1223  // child's lowlink is reflected.
1224  DFSStack.push_back({N, I});
1225 
1226  // Continue, resetting to the child node.
1227  ChildN.LowLink = ChildN.DFSNumber = NextDFSNumber++;
1228  N = &ChildN;
1229  I = ChildN.begin();
1230  E = ChildN.end();
1231  continue;
1232  }
1233  if (ChildN.DFSNumber == -1) {
1234  // Check if this edge's target node connects to the deleted edge's
1235  // target node. If so, we know that every node connected will end up
1236  // in this RefSCC, so collapse the entire current stack into the root
1237  // slot in our SCC numbering. See above for the motivation of
1238  // optimizing the target connected nodes in this way.
1239  auto PostOrderI = PostOrderMapping.find(&ChildN);
1240  if (PostOrderI != PostOrderMapping.end() &&
1241  PostOrderI->second == RootPostOrderNumber) {
1242  MarkNodeForSCCNumber(*N, RootPostOrderNumber);
1243  while (!PendingRefSCCStack.empty())
1244  MarkNodeForSCCNumber(*PendingRefSCCStack.pop_back_val(),
1245  RootPostOrderNumber);
1246  while (!DFSStack.empty())
1247  MarkNodeForSCCNumber(*DFSStack.pop_back_val().first,
1248  RootPostOrderNumber);
1249  // Ensure we break all the way out of the enclosing loop.
1250  N = nullptr;
1251  break;
1252  }
1253 
1254  // If this child isn't currently in this RefSCC, no need to process
1255  // it. However, we do need to remove this RefSCC from its RefSCC's
1256  // parent set.
1257  RefSCC &ChildRC = *G->lookupRefSCC(ChildN);
1258  ChildRC.Parents.erase(this);
1259  ++I;
1260  continue;
1261  }
1262 
1263  // Track the lowest link of the children, if any are still in the stack.
1264  // Any child not on the stack will have a LowLink of -1.
1265  assert(ChildN.LowLink != 0 &&
1266  "Low-link must not be zero with a non-zero DFS number.");
1267  if (ChildN.LowLink >= 0 && ChildN.LowLink < N->LowLink)
1268  N->LowLink = ChildN.LowLink;
1269  ++I;
1270  }
1271  if (!N)
1272  // We short-circuited this node.
1273  break;
1274 
1275  // We've finished processing N and its descendents, put it on our pending
1276  // stack to eventually get merged into a RefSCC.
1277  PendingRefSCCStack.push_back(N);
1278 
1279  // If this node is linked to some lower entry, continue walking up the
1280  // stack.
1281  if (N->LowLink != N->DFSNumber) {
1282  assert(!DFSStack.empty() &&
1283  "We never found a viable root for a RefSCC to pop off!");
1284  continue;
1285  }
1286 
1287  // Otherwise, form a new RefSCC from the top of the pending node stack.
1288  int RootDFSNumber = N->DFSNumber;
1289  // Find the range of the node stack by walking down until we pass the
1290  // root DFS number.
1291  auto RefSCCNodes = make_range(
1292  PendingRefSCCStack.rbegin(),
1293  find_if(reverse(PendingRefSCCStack), [RootDFSNumber](const Node *N) {
1294  return N->DFSNumber < RootDFSNumber;
1295  }));
1296 
1297  // Mark the postorder number for these nodes and clear them off the
1298  // stack. We'll use the postorder number to pull them into RefSCCs at the
1299  // end. FIXME: Fuse with the loop above.
1300  int RefSCCNumber = PostOrderNumber++;
1301  for (Node *N : RefSCCNodes)
1302  MarkNodeForSCCNumber(*N, RefSCCNumber);
1303 
1304  PendingRefSCCStack.erase(RefSCCNodes.end().base(),
1305  PendingRefSCCStack.end());
1306  } while (!DFSStack.empty());
1307 
1308  assert(DFSStack.empty() && "Didn't flush the entire DFS stack!");
1309  assert(PendingRefSCCStack.empty() && "Didn't flush all pending nodes!");
1310  } while (!Worklist.empty());
1311 
1312  // We now have a post-order numbering for RefSCCs and a mapping from each
1313  // node in this RefSCC to its final RefSCC. We create each new RefSCC node
1314  // (re-using this RefSCC node for the root) and build a radix-sort style map
1315  // from postorder number to the RefSCC. We then append SCCs to each of these
1316  // RefSCCs in the order they occured in the original SCCs container.
1317  for (int i = 1; i < PostOrderNumber; ++i)
1318  Result.push_back(G->createRefSCC(*G));
1319 
1320  // Insert the resulting postorder sequence into the global graph postorder
1321  // sequence before the current RefSCC in that sequence. The idea being that
1322  // this RefSCC is the target of the reference edge removed, and thus has
1323  // a direct or indirect edge to every other RefSCC formed and so must be at
1324  // the end of any postorder traversal.
1325  //
1326  // FIXME: It'd be nice to change the APIs so that we returned an iterator
1327  // range over the global postorder sequence and generally use that sequence
1328  // rather than building a separate result vector here.
1329  if (!Result.empty()) {
1330  int Idx = G->getRefSCCIndex(*this);
1331  G->PostOrderRefSCCs.insert(G->PostOrderRefSCCs.begin() + Idx,
1332  Result.begin(), Result.end());
1333  for (int i : seq<int>(Idx, G->PostOrderRefSCCs.size()))
1334  G->RefSCCIndices[G->PostOrderRefSCCs[i]] = i;
1335  assert(G->PostOrderRefSCCs[G->getRefSCCIndex(*this)] == this &&
1336  "Failed to update this RefSCC's index after insertion!");
1337  }
1338 
1339  for (SCC *C : SCCs) {
1340  auto PostOrderI = PostOrderMapping.find(&*C->begin());
1341  assert(PostOrderI != PostOrderMapping.end() &&
1342  "Cannot have missing mappings for nodes!");
1343  int SCCNumber = PostOrderI->second;
1344 #ifndef NDEBUG
1345  for (Node &N : *C)
1346  assert(PostOrderMapping.find(&N)->second == SCCNumber &&
1347  "Cannot have different numbers for nodes in the same SCC!");
1348 #endif
1349  if (SCCNumber == 0)
1350  // The root node is handled separately by removing the SCCs.
1351  continue;
1352 
1353  RefSCC &RC = *Result[SCCNumber - 1];
1354  int SCCIndex = RC.SCCs.size();
1355  RC.SCCs.push_back(C);
1356  RC.SCCIndices[C] = SCCIndex;
1357  C->OuterRefSCC = &RC;
1358  }
1359 
1360  // FIXME: We re-walk the edges in each RefSCC to establish whether it is
1361  // a leaf and connect it to the rest of the graph's parents lists. This is
1362  // really wasteful. We should instead do this during the DFS to avoid yet
1363  // another edge walk.
1364  for (RefSCC *RC : Result)
1365  G->connectRefSCC(*RC);
1366 
1367  // Now erase all but the root's SCCs.
1368  SCCs.erase(remove_if(SCCs,
1369  [&](SCC *C) {
1370  return PostOrderMapping.lookup(&*C->begin()) !=
1371  RootPostOrderNumber;
1372  }),
1373  SCCs.end());
1374  SCCIndices.clear();
1375  for (int i = 0, Size = SCCs.size(); i < Size; ++i)
1376  SCCIndices[SCCs[i]] = i;
1377 
1378 #ifndef NDEBUG
1379  // Now we need to reconnect the current (root) SCC to the graph. We do this
1380  // manually because we can special case our leaf handling and detect errors.
1381  bool IsLeaf = true;
1382 #endif
1383  for (SCC *C : SCCs)
1384  for (Node &N : *C) {
1385  for (Edge &E : N) {
1386  assert(E.getNode() && "Cannot have a missing node in a visited SCC!");
1387  RefSCC &ChildRC = *G->lookupRefSCC(*E.getNode());
1388  if (&ChildRC == this)
1389  continue;
1390  ChildRC.Parents.insert(this);
1391 #ifndef NDEBUG
1392  IsLeaf = false;
1393 #endif
1394  }
1395  }
1396 #ifndef NDEBUG
1397  if (!Result.empty())
1398  assert(!IsLeaf && "This SCC cannot be a leaf as we have split out new "
1399  "SCCs by removing this edge.");
1400  if (none_of(G->LeafRefSCCs, [&](RefSCC *C) { return C == this; }))
1401  assert(!IsLeaf && "This SCC cannot be a leaf as it already had child "
1402  "SCCs before we removed this edge.");
1403 #endif
1404  // And connect both this RefSCC and all the new ones to the correct parents.
1405  // The easiest way to do this is just to re-analyze the old parent set.
1406  SmallVector<RefSCC *, 4> OldParents(Parents.begin(), Parents.end());
1407  Parents.clear();
1408  for (RefSCC *ParentRC : OldParents)
1409  for (SCC &ParentC : *ParentRC)
1410  for (Node &ParentN : ParentC)
1411  for (Edge &E : ParentN) {
1412  assert(E.getNode() && "Cannot have a missing node in a visited SCC!");
1413  RefSCC &RC = *G->lookupRefSCC(*E.getNode());
1414  if (&RC != ParentRC)
1415  RC.Parents.insert(ParentRC);
1416  }
1417 
1418  // If this SCC stopped being a leaf through this edge removal, remove it from
1419  // the leaf SCC list. Note that this DTRT in the case where this was never
1420  // a leaf.
1421  // FIXME: As LeafRefSCCs could be very large, we might want to not walk the
1422  // entire list if this RefSCC wasn't a leaf before the edge removal.
1423  if (!Result.empty())
1424  G->LeafRefSCCs.erase(
1425  std::remove(G->LeafRefSCCs.begin(), G->LeafRefSCCs.end(), this),
1426  G->LeafRefSCCs.end());
1427 
1428 #ifndef NDEBUG
1429  // Verify all of the new RefSCCs.
1430  for (RefSCC *RC : Result)
1431  RC->verify();
1432 #endif
1433 
1434  // Return the new list of SCCs.
1435  return Result;
1436 }
1437 
1438 void LazyCallGraph::RefSCC::handleTrivialEdgeInsertion(Node &SourceN,
1439  Node &TargetN) {
1440  // The only trivial case that requires any graph updates is when we add new
1441  // ref edge and may connect different RefSCCs along that path. This is only
1442  // because of the parents set. Every other part of the graph remains constant
1443  // after this edge insertion.
1444  assert(G->lookupRefSCC(SourceN) == this && "Source must be in this RefSCC.");
1445  RefSCC &TargetRC = *G->lookupRefSCC(TargetN);
1446  if (&TargetRC == this) {
1447 
1448  return;
1449  }
1450 
1451  assert(TargetRC.isDescendantOf(*this) &&
1452  "Target must be a descendant of the Source.");
1453  // The only change required is to add this RefSCC to the parent set of the
1454  // target. This is a set and so idempotent if the edge already existed.
1455  TargetRC.Parents.insert(this);
1456 }
1457 
1459  Node &TargetN) {
1460 #ifndef NDEBUG
1461  // Check that the RefSCC is still valid when we finish.
1462  auto ExitVerifier = make_scope_exit([this] { verify(); });
1463 
1464  // Check that we aren't breaking some invariants of the SCC graph.
1465  SCC &SourceC = *G->lookupSCC(SourceN);
1466  SCC &TargetC = *G->lookupSCC(TargetN);
1467  if (&SourceC != &TargetC)
1468  assert(SourceC.isAncestorOf(TargetC) &&
1469  "Call edge is not trivial in the SCC graph!");
1470 #endif
1471  // First insert it into the source or find the existing edge.
1472  auto InsertResult = SourceN.EdgeIndexMap.insert(
1473  {&TargetN.getFunction(), SourceN.Edges.size()});
1474  if (!InsertResult.second) {
1475  // Already an edge, just update it.
1476  Edge &E = SourceN.Edges[InsertResult.first->second];
1477  if (E.isCall())
1478  return; // Nothing to do!
1479  E.setKind(Edge::Call);
1480  } else {
1481  // Create the new edge.
1482  SourceN.Edges.emplace_back(TargetN, Edge::Call);
1483  }
1484 
1485  // Now that we have the edge, handle the graph fallout.
1486  handleTrivialEdgeInsertion(SourceN, TargetN);
1487 }
1488 
1490 #ifndef NDEBUG
1491  // Check that the RefSCC is still valid when we finish.
1492  auto ExitVerifier = make_scope_exit([this] { verify(); });
1493 
1494  // Check that we aren't breaking some invariants of the RefSCC graph.
1495  RefSCC &SourceRC = *G->lookupRefSCC(SourceN);
1496  RefSCC &TargetRC = *G->lookupRefSCC(TargetN);
1497  if (&SourceRC != &TargetRC)
1498  assert(SourceRC.isAncestorOf(TargetRC) &&
1499  "Ref edge is not trivial in the RefSCC graph!");
1500 #endif
1501  // First insert it into the source or find the existing edge.
1502  auto InsertResult = SourceN.EdgeIndexMap.insert(
1503  {&TargetN.getFunction(), SourceN.Edges.size()});
1504  if (!InsertResult.second)
1505  // Already an edge, we're done.
1506  return;
1507 
1508  // Create the new edge.
1509  SourceN.Edges.emplace_back(TargetN, Edge::Ref);
1510 
1511  // Now that we have the edge, handle the graph fallout.
1512  handleTrivialEdgeInsertion(SourceN, TargetN);
1513 }
1514 
1515 void LazyCallGraph::insertEdge(Node &SourceN, Function &Target, Edge::Kind EK) {
1516  assert(SCCMap.empty() && DFSStack.empty() &&
1517  "This method cannot be called after SCCs have been formed!");
1518 
1519  return SourceN.insertEdgeInternal(Target, EK);
1520 }
1521 
1522 void LazyCallGraph::removeEdge(Node &SourceN, Function &Target) {
1523  assert(SCCMap.empty() && DFSStack.empty() &&
1524  "This method cannot be called after SCCs have been formed!");
1525 
1526  return SourceN.removeEdgeInternal(Target);
1527 }
1528 
1530  // FIXME: This is unnecessarily restrictive. We should be able to remove
1531  // functions which recursively call themselves.
1532  assert(F.use_empty() &&
1533  "This routine should only be called on trivially dead functions!");
1534 
1535  auto EII = EntryIndexMap.find(&F);
1536  if (EII != EntryIndexMap.end()) {
1537  EntryEdges[EII->second] = Edge();
1538  EntryIndexMap.erase(EII);
1539  }
1540 
1541  // It's safe to just remove un-visited functions from the RefSCC entry list.
1542  // FIXME: This is a linear operation which could become hot and benefit from
1543  // an index map.
1544  auto RENI = find(RefSCCEntryNodes, &F);
1545  if (RENI != RefSCCEntryNodes.end())
1546  RefSCCEntryNodes.erase(RENI);
1547 
1548  auto NI = NodeMap.find(&F);
1549  if (NI == NodeMap.end())
1550  // Not in the graph at all!
1551  return;
1552 
1553  Node &N = *NI->second;
1554  NodeMap.erase(NI);
1555 
1556  if (SCCMap.empty() && DFSStack.empty()) {
1557  // No SCC walk has begun, so removing this is fine and there is nothing
1558  // else necessary at this point but clearing out the node.
1559  N.clear();
1560  return;
1561  }
1562 
1563  // Check that we aren't going to break the DFS walk.
1564  assert(all_of(DFSStack,
1565  [&N](const std::pair<Node *, edge_iterator> &Element) {
1566  return Element.first != &N;
1567  }) &&
1568  "Tried to remove a function currently in the DFS stack!");
1569  assert(find(PendingRefSCCStack, &N) == PendingRefSCCStack.end() &&
1570  "Tried to remove a function currently pending to add to a RefSCC!");
1571 
1572  // Cannot remove a function which has yet to be visited in the DFS walk, so
1573  // if we have a node at all then we must have an SCC and RefSCC.
1574  auto CI = SCCMap.find(&N);
1575  assert(CI != SCCMap.end() &&
1576  "Tried to remove a node without an SCC after DFS walk started!");
1577  SCC &C = *CI->second;
1578  SCCMap.erase(CI);
1579  RefSCC &RC = C.getOuterRefSCC();
1580 
1581  // This node must be the only member of its SCC as it has no callers, and
1582  // that SCC must be the only member of a RefSCC as it has no references.
1583  // Validate these properties first.
1584  assert(C.size() == 1 && "Dead functions must be in a singular SCC");
1585  assert(RC.size() == 1 && "Dead functions must be in a singular RefSCC");
1586  assert(RC.Parents.empty() && "Cannot have parents of a dead RefSCC!");
1587 
1588  // Now remove this RefSCC from any parents sets and the leaf list.
1589  for (Edge &E : N)
1590  if (Node *TargetN = E.getNode())
1591  if (RefSCC *TargetRC = lookupRefSCC(*TargetN))
1592  TargetRC->Parents.erase(&RC);
1593  // FIXME: This is a linear operation which could become hot and benefit from
1594  // an index map.
1595  auto LRI = find(LeafRefSCCs, &RC);
1596  if (LRI != LeafRefSCCs.end())
1597  LeafRefSCCs.erase(LRI);
1598 
1599  auto RCIndexI = RefSCCIndices.find(&RC);
1600  int RCIndex = RCIndexI->second;
1601  PostOrderRefSCCs.erase(PostOrderRefSCCs.begin() + RCIndex);
1602  RefSCCIndices.erase(RCIndexI);
1603  for (int i = RCIndex, Size = PostOrderRefSCCs.size(); i < Size; ++i)
1604  RefSCCIndices[PostOrderRefSCCs[i]] = i;
1605 
1606  // Finally clear out all the data structures from the node down through the
1607  // components.
1608  N.clear();
1609  C.clear();
1610  RC.clear();
1611 
1612  // Nothing to delete as all the objects are allocated in stable bump pointer
1613  // allocators.
1614 }
1615 
1616 LazyCallGraph::Node &LazyCallGraph::insertInto(Function &F, Node *&MappedN) {
1617  return *new (MappedN = BPA.Allocate()) Node(*this, F);
1618 }
1619 
1620 void LazyCallGraph::updateGraphPtrs() {
1621  // Process all nodes updating the graph pointers.
1622  {
1623  SmallVector<Node *, 16> Worklist;
1624  for (Edge &E : EntryEdges)
1625  if (Node *EntryN = E.getNode())
1626  Worklist.push_back(EntryN);
1627 
1628  while (!Worklist.empty()) {
1629  Node *N = Worklist.pop_back_val();
1630  N->G = this;
1631  for (Edge &E : N->Edges)
1632  if (Node *TargetN = E.getNode())
1633  Worklist.push_back(TargetN);
1634  }
1635  }
1636 
1637  // Process all SCCs updating the graph pointers.
1638  {
1639  SmallVector<RefSCC *, 16> Worklist(LeafRefSCCs.begin(), LeafRefSCCs.end());
1640 
1641  while (!Worklist.empty()) {
1642  RefSCC &C = *Worklist.pop_back_val();
1643  C.G = this;
1644  for (RefSCC &ParentC : C.parents())
1645  Worklist.push_back(&ParentC);
1646  }
1647  }
1648 }
1649 
1650 /// Build the internal SCCs for a RefSCC from a sequence of nodes.
1651 ///
1652 /// Appends the SCCs to the provided vector and updates the map with their
1653 /// indices. Both the vector and map must be empty when passed into this
1654 /// routine.
1655 void LazyCallGraph::buildSCCs(RefSCC &RC, node_stack_range Nodes) {
1656  assert(RC.SCCs.empty() && "Already built SCCs!");
1657  assert(RC.SCCIndices.empty() && "Already mapped SCC indices!");
1658 
1659  for (Node *N : Nodes) {
1660  assert(N->LowLink >= (*Nodes.begin())->LowLink &&
1661  "We cannot have a low link in an SCC lower than its root on the "
1662  "stack!");
1663 
1664  // This node will go into the next RefSCC, clear out its DFS and low link
1665  // as we scan.
1666  N->DFSNumber = N->LowLink = 0;
1667  }
1668 
1669  // Each RefSCC contains a DAG of the call SCCs. To build these, we do
1670  // a direct walk of the call edges using Tarjan's algorithm. We reuse the
1671  // internal storage as we won't need it for the outer graph's DFS any longer.
1672 
1674  SmallVector<Node *, 16> PendingSCCStack;
1675 
1676  // Scan down the stack and DFS across the call edges.
1677  for (Node *RootN : Nodes) {
1678  assert(DFSStack.empty() &&
1679  "Cannot begin a new root with a non-empty DFS stack!");
1680  assert(PendingSCCStack.empty() &&
1681  "Cannot begin a new root with pending nodes for an SCC!");
1682 
1683  // Skip any nodes we've already reached in the DFS.
1684  if (RootN->DFSNumber != 0) {
1685  assert(RootN->DFSNumber == -1 &&
1686  "Shouldn't have any mid-DFS root nodes!");
1687  continue;
1688  }
1689 
1690  RootN->DFSNumber = RootN->LowLink = 1;
1691  int NextDFSNumber = 2;
1692 
1693  DFSStack.push_back({RootN, RootN->call_begin()});
1694  do {
1695  Node *N;
1696  call_edge_iterator I;
1697  std::tie(N, I) = DFSStack.pop_back_val();
1698  auto E = N->call_end();
1699  while (I != E) {
1700  Node &ChildN = *I->getNode();
1701  if (ChildN.DFSNumber == 0) {
1702  // We haven't yet visited this child, so descend, pushing the current
1703  // node onto the stack.
1704  DFSStack.push_back({N, I});
1705 
1706  assert(!lookupSCC(ChildN) &&
1707  "Found a node with 0 DFS number but already in an SCC!");
1708  ChildN.DFSNumber = ChildN.LowLink = NextDFSNumber++;
1709  N = &ChildN;
1710  I = N->call_begin();
1711  E = N->call_end();
1712  continue;
1713  }
1714 
1715  // If the child has already been added to some child component, it
1716  // couldn't impact the low-link of this parent because it isn't
1717  // connected, and thus its low-link isn't relevant so skip it.
1718  if (ChildN.DFSNumber == -1) {
1719  ++I;
1720  continue;
1721  }
1722 
1723  // Track the lowest linked child as the lowest link for this node.
1724  assert(ChildN.LowLink > 0 && "Must have a positive low-link number!");
1725  if (ChildN.LowLink < N->LowLink)
1726  N->LowLink = ChildN.LowLink;
1727 
1728  // Move to the next edge.
1729  ++I;
1730  }
1731 
1732  // We've finished processing N and its descendents, put it on our pending
1733  // SCC stack to eventually get merged into an SCC of nodes.
1734  PendingSCCStack.push_back(N);
1735 
1736  // If this node is linked to some lower entry, continue walking up the
1737  // stack.
1738  if (N->LowLink != N->DFSNumber)
1739  continue;
1740 
1741  // Otherwise, we've completed an SCC. Append it to our post order list of
1742  // SCCs.
1743  int RootDFSNumber = N->DFSNumber;
1744  // Find the range of the node stack by walking down until we pass the
1745  // root DFS number.
1746  auto SCCNodes = make_range(
1747  PendingSCCStack.rbegin(),
1748  find_if(reverse(PendingSCCStack), [RootDFSNumber](const Node *N) {
1749  return N->DFSNumber < RootDFSNumber;
1750  }));
1751  // Form a new SCC out of these nodes and then clear them off our pending
1752  // stack.
1753  RC.SCCs.push_back(createSCC(RC, SCCNodes));
1754  for (Node &N : *RC.SCCs.back()) {
1755  N.DFSNumber = N.LowLink = -1;
1756  SCCMap[&N] = RC.SCCs.back();
1757  }
1758  PendingSCCStack.erase(SCCNodes.end().base(), PendingSCCStack.end());
1759  } while (!DFSStack.empty());
1760  }
1761 
1762  // Wire up the SCC indices.
1763  for (int i = 0, Size = RC.SCCs.size(); i < Size; ++i)
1764  RC.SCCIndices[RC.SCCs[i]] = i;
1765 }
1766 
1767 // FIXME: We should move callers of this to embed the parent linking and leaf
1768 // tracking into their DFS in order to remove a full walk of all edges.
1769 void LazyCallGraph::connectRefSCC(RefSCC &RC) {
1770  // Walk all edges in the RefSCC (this remains linear as we only do this once
1771  // when we build the RefSCC) to connect it to the parent sets of its
1772  // children.
1773  bool IsLeaf = true;
1774  for (SCC &C : RC)
1775  for (Node &N : C)
1776  for (Edge &E : N) {
1777  assert(E.getNode() &&
1778  "Cannot have a missing node in a visited part of the graph!");
1779  RefSCC &ChildRC = *lookupRefSCC(*E.getNode());
1780  if (&ChildRC == &RC)
1781  continue;
1782  ChildRC.Parents.insert(&RC);
1783  IsLeaf = false;
1784  }
1785 
1786  // For the SCCs where we find no child SCCs, add them to the leaf list.
1787  if (IsLeaf)
1788  LeafRefSCCs.push_back(&RC);
1789 }
1790 
1791 bool LazyCallGraph::buildNextRefSCCInPostOrder() {
1792  if (DFSStack.empty()) {
1793  Node *N;
1794  do {
1795  // If we've handled all candidate entry nodes to the SCC forest, we're
1796  // done.
1797  if (RefSCCEntryNodes.empty())
1798  return false;
1799 
1800  N = &get(*RefSCCEntryNodes.pop_back_val());
1801  } while (N->DFSNumber != 0);
1802 
1803  // Found a new root, begin the DFS here.
1804  N->LowLink = N->DFSNumber = 1;
1805  NextDFSNumber = 2;
1806  DFSStack.push_back({N, N->begin()});
1807  }
1808 
1809  for (;;) {
1810  Node *N;
1811  edge_iterator I;
1812  std::tie(N, I) = DFSStack.pop_back_val();
1813 
1814  assert(N->DFSNumber > 0 && "We should always assign a DFS number "
1815  "before placing a node onto the stack.");
1816 
1817  auto E = N->end();
1818  while (I != E) {
1819  Node &ChildN = I->getNode(*this);
1820  if (ChildN.DFSNumber == 0) {
1821  // We haven't yet visited this child, so descend, pushing the current
1822  // node onto the stack.
1823  DFSStack.push_back({N, N->begin()});
1824 
1825  assert(!SCCMap.count(&ChildN) &&
1826  "Found a node with 0 DFS number but already in an SCC!");
1827  ChildN.LowLink = ChildN.DFSNumber = NextDFSNumber++;
1828  N = &ChildN;
1829  I = N->begin();
1830  E = N->end();
1831  continue;
1832  }
1833 
1834  // If the child has already been added to some child component, it
1835  // couldn't impact the low-link of this parent because it isn't
1836  // connected, and thus its low-link isn't relevant so skip it.
1837  if (ChildN.DFSNumber == -1) {
1838  ++I;
1839  continue;
1840  }
1841 
1842  // Track the lowest linked child as the lowest link for this node.
1843  assert(ChildN.LowLink > 0 && "Must have a positive low-link number!");
1844  if (ChildN.LowLink < N->LowLink)
1845  N->LowLink = ChildN.LowLink;
1846 
1847  // Move to the next edge.
1848  ++I;
1849  }
1850 
1851  // We've finished processing N and its descendents, put it on our pending
1852  // SCC stack to eventually get merged into an SCC of nodes.
1853  PendingRefSCCStack.push_back(N);
1854 
1855  // If this node is linked to some lower entry, continue walking up the
1856  // stack.
1857  if (N->LowLink != N->DFSNumber) {
1858  assert(!DFSStack.empty() &&
1859  "We never found a viable root for an SCC to pop off!");
1860  continue;
1861  }
1862 
1863  // Otherwise, form a new RefSCC from the top of the pending node stack.
1864  int RootDFSNumber = N->DFSNumber;
1865  // Find the range of the node stack by walking down until we pass the
1866  // root DFS number.
1867  auto RefSCCNodes = node_stack_range(
1868  PendingRefSCCStack.rbegin(),
1869  find_if(reverse(PendingRefSCCStack), [RootDFSNumber](const Node *N) {
1870  return N->DFSNumber < RootDFSNumber;
1871  }));
1872  // Form a new RefSCC out of these nodes and then clear them off our pending
1873  // stack.
1874  RefSCC *NewRC = createRefSCC(*this);
1875  buildSCCs(*NewRC, RefSCCNodes);
1876  connectRefSCC(*NewRC);
1877  PendingRefSCCStack.erase(RefSCCNodes.end().base(),
1878  PendingRefSCCStack.end());
1879 
1880  // Push the new node into the postorder list and return true indicating we
1881  // successfully grew the postorder sequence by one.
1882  bool Inserted =
1883  RefSCCIndices.insert({NewRC, PostOrderRefSCCs.size()}).second;
1884  (void)Inserted;
1885  assert(Inserted && "Cannot already have this RefSCC in the index map!");
1886  PostOrderRefSCCs.push_back(NewRC);
1887  return true;
1888  }
1889 }
1890 
1891 AnalysisKey LazyCallGraphAnalysis::Key;
1892 
1894 
1896  OS << " Edges in function: " << N.getFunction().getName() << "\n";
1897  for (const LazyCallGraph::Edge &E : N)
1898  OS << " " << (E.isCall() ? "call" : "ref ") << " -> "
1899  << E.getFunction().getName() << "\n";
1900 
1901  OS << "\n";
1902 }
1903 
1905  ptrdiff_t Size = std::distance(C.begin(), C.end());
1906  OS << " SCC with " << Size << " functions:\n";
1907 
1908  for (LazyCallGraph::Node &N : C)
1909  OS << " " << N.getFunction().getName() << "\n";
1910 }
1911 
1913  ptrdiff_t Size = std::distance(C.begin(), C.end());
1914  OS << " RefSCC with " << Size << " call SCCs:\n";
1915 
1916  for (LazyCallGraph::SCC &InnerC : C)
1917  printSCC(OS, InnerC);
1918 
1919  OS << "\n";
1920 }
1921 
1923  ModuleAnalysisManager &AM) {
1925 
1926  OS << "Printing the call graph for module: " << M.getModuleIdentifier()
1927  << "\n\n";
1928 
1929  for (Function &F : M)
1930  printNode(OS, G.get(F));
1931 
1933  printRefSCC(OS, C);
1934 
1935  return PreservedAnalyses::all();
1936 }
1937 
1939  : OS(OS) {}
1940 
1942  std::string Name = "\"" + DOT::EscapeString(N.getFunction().getName()) + "\"";
1943 
1944  for (const LazyCallGraph::Edge &E : N) {
1945  OS << " " << Name << " -> \""
1946  << DOT::EscapeString(E.getFunction().getName()) << "\"";
1947  if (!E.isCall()) // It is a ref edge.
1948  OS << " [style=dashed,label=\"ref\"]";
1949  OS << ";\n";
1950  }
1951 
1952  OS << "\n";
1953 }
1954 
1956  ModuleAnalysisManager &AM) {
1958 
1959  OS << "digraph \"" << DOT::EscapeString(M.getModuleIdentifier()) << "\" {\n";
1960 
1961  for (Function &F : M)
1962  printNodeDOT(OS, G.get(F));
1963 
1964  OS << "}\n";
1965 
1966  return PreservedAnalyses::all();
1967 }
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
This routine provides some synthesis utilities to produce sequences of values.
bool isChildOf(const RefSCC &C) const
Test if this RefSCC is a child of C.
void removeOutgoingEdge(Node &SourceN, Node &TargetN)
Remove an edge whose source is in this RefSCC and target is not.
size_t i
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:162
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:52
auto remove_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:776
Kind
The kind of edge in the graph.
iterator begin() const
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
Implements a lazy call graph analysis and related passes for the new pass manager.
static void addEdge(SmallVectorImpl< LazyCallGraph::Edge > &Edges, DenseMap< Function *, int > &EdgeIndexMap, Function &F, LazyCallGraph::Edge::Kind EK)
size_type count(PtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:380
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:47
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:736
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
static void printNode(raw_ostream &OS, LazyCallGraph::Node &N)
edge_iterator begin() const
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:345
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:172
SmallVector< SCC *, 1 > switchInternalEdgeToCall(Node &SourceN, Node &TargetN)
Make an existing internal ref edge into a call edge.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:32
bool isAncestorOf(const SCC &C) const
Test if this SCC is an ancestor of C.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:750
const std::string & getModuleIdentifier() const
Get the module identifier which is, essentially, the name of the module.
Definition: Module.h:193
bool isDescendantOf(const RefSCC &C) const
Test if this RefSCC is a descendant of C.
edge_iterator end() const
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:241
A RefSCC of the call graph.
#define F(x, y, z)
Definition: MD5.cpp:51
bool isCall() const
Test whether the edge represents a direct call to a function.
void insertTrivialCallEdge(Node &SourceN, Node &TargetN)
A convenience wrapper around the above to handle trivial cases of inserting a new call edge...
A lazily constructed view of the call graph of a module.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
iterator_range< parent_iterator > parents() const
LazyCallGraph & operator=(LazyCallGraph &&RHS)
void removeEdge(Node &Caller, Function &Callee)
Update the call graph after deleting an edge.
void switchOutgoingEdgeToCall(Node &SourceN, Node &TargetN)
Make an existing outgoing ref edge into a call edge.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
bool verify(const TargetRegisterInfo &TRI) const
Check that information hold by this instance make sense for the given TRI.
void insertEdge(Node &Caller, Function &Callee, Edge::Kind EK)
Update the call graph after inserting a new edge.
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:107
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:653
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
LazyCallGraphDOTPrinterPass(raw_ostream &OS)
std::string EscapeString(const std::string &Label)
Definition: GraphWriter.cpp:24
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:675
This is an important base class in LLVM.
Definition: Constant.h:42
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
void switchOutgoingEdgeToRef(Node &SourceN, Node &TargetN)
Make an existing outgoing call edge into a ref edge.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
void insertInternalRefEdge(Node &SourceN, Node &TargetN)
Insert a ref edge from one node in this RefSCC to another in this RefSCC.
static void printNodeDOT(raw_ostream &OS, LazyCallGraph::Node &N)
SmallVector< RefSCC *, 1 > removeInternalRefEdge(Node &SourceN, Node &TargetN)
Remove a ref edge which is entirely within this RefSCC.
SmallVector< RefSCC *, 1 > insertIncomingRefEdge(Node &SourceN, Node &TargetN)
Insert an edge whose source is in a descendant RefSCC and target is in this RefSCC.
RefSCC & getOuterRefSCC() const
A node in the call graph.
A class used to represent edges in the call graph.
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:392
iterator erase(const_iterator CI)
Definition: SmallVector.h:431
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:113
A lazy iterator used for both the entry nodes and child nodes.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Function & getFunction() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:425
void removeDeadFunction(Function &F)
Remove a dead function from the call graph (typically to delete it).
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:757
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
const DataFlowGraph & G
Definition: RDFGraph.cpp:206
static const char * Target
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:382
IteratorT end() const
IteratorT begin() const
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
A range adaptor for a pair of iterators.
Target - Wrapper for Target specific information.
void insertOutgoingEdge(Node &SourceN, Node &TargetN, Edge::Kind EK)
Insert an edge whose parent is in this RefSCC and child is in some child RefSCC.
void switchTrivialInternalEdgeToRef(Node &SourceN, Node &TargetN)
Make an existing internal call edge between separate SCCs into a ref edge.
iterator_range< iterator > switchInternalEdgeToRef(Node &SourceN, Node &TargetN)
Make an existing internal call edge within a single SCC into a ref edge.
LazyCallGraphPrinterPass(raw_ostream &OS)
Node * lookup(const Function &F) const
Lookup a function in the graph which has already been scanned and added.
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
void emplace_back(ArgTypes &&...Args)
Definition: SmallVector.h:635
iterator end() const
call_edge_iterator call_end() const
static void printRefSCC(raw_ostream &OS, LazyCallGraph::RefSCC &C)
#define I(x, y, z)
Definition: MD5.cpp:54
#define N
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
static void printSCC(raw_ostream &OS, LazyCallGraph::SCC &C)
An analysis pass which computes the call graph for a module.
void insertTrivialRefEdge(Node &SourceN, Node &TargetN)
A convenience wrapper around the above to handle trivial cases of inserting a new ref edge...
const unsigned Kind
bool use_empty() const
Definition: Value.h:299
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A lazy iterator over specifically call edges.
static iterator_range< typename PostorderSequenceT::iterator > updatePostorderSequenceForEdgeInsertion(SCCT &SourceSCC, SCCT &TargetSCC, PostorderSequenceT &SCCs, SCCIndexMapT &SCCIndices, ComputeSourceConnectedSetCallableT ComputeSourceConnectedSet, ComputeTargetConnectedSetCallableT ComputeTargetConnectedSet)
Generic helper that updates a postorder sequence of SCCs for a potentially cycle-introducing edge ins...
LLVM Value Representation.
Definition: Value.h:71
An SCC of the call graph.
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:44
#define DEBUG(X)
Definition: Debug.h:100
A container for analyses that lazily runs them and caches their results.
bool isParentOf(const SCC &C) const
Test if this SCC is a parent of C.
RefSCC * lookupRefSCC(Node &N) const
Lookup a function's RefSCC in the graph.
This header defines various interfaces for pass management in LLVM.
auto find_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:764
LazyCallGraph(Module &M)
Construct a graph for the given module.
iterator_range< postorder_ref_scc_iterator > postorder_ref_sccs()
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: PassManager.h:64
call_edge_iterator call_begin() const
iterator_range< call_edge_iterator > calls() const
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:783