LLVM  3.7.0
RewriteStatepointsForGC.cpp
Go to the documentation of this file.
1 //===- RewriteStatepointsForGC.cpp - Make GC relocations explicit ---------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Rewrite an existing set of gc.statepoints such that they make potential
11 // relocations performed by the garbage collector explicit in the IR.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/Pass.h"
16 #include "llvm/Analysis/CFG.h"
18 #include "llvm/ADT/SetOperations.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/ADT/DenseSet.h"
21 #include "llvm/ADT/SetVector.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/IR/BasicBlock.h"
24 #include "llvm/IR/CallSite.h"
25 #include "llvm/IR/Dominators.h"
26 #include "llvm/IR/Function.h"
27 #include "llvm/IR/IRBuilder.h"
28 #include "llvm/IR/InstIterator.h"
29 #include "llvm/IR/Instructions.h"
30 #include "llvm/IR/Intrinsics.h"
31 #include "llvm/IR/IntrinsicInst.h"
32 #include "llvm/IR/Module.h"
33 #include "llvm/IR/MDBuilder.h"
34 #include "llvm/IR/Statepoint.h"
35 #include "llvm/IR/Value.h"
36 #include "llvm/IR/Verifier.h"
37 #include "llvm/Support/Debug.h"
39 #include "llvm/Transforms/Scalar.h"
44 
45 #define DEBUG_TYPE "rewrite-statepoints-for-gc"
46 
47 using namespace llvm;
48 
49 // Print tracing output
50 static cl::opt<bool> TraceLSP("trace-rewrite-statepoints", cl::Hidden,
51  cl::init(false));
52 
53 // Print the liveset found at the insert location
54 static cl::opt<bool> PrintLiveSet("spp-print-liveset", cl::Hidden,
55  cl::init(false));
56 static cl::opt<bool> PrintLiveSetSize("spp-print-liveset-size", cl::Hidden,
57  cl::init(false));
58 // Print out the base pointers for debugging
59 static cl::opt<bool> PrintBasePointers("spp-print-base-pointers", cl::Hidden,
60  cl::init(false));
61 
62 // Cost threshold measuring when it is profitable to rematerialize value instead
63 // of relocating it
64 static cl::opt<unsigned>
65 RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden,
66  cl::init(6));
67 
68 #ifdef XDEBUG
69 static bool ClobberNonLive = true;
70 #else
71 static bool ClobberNonLive = false;
72 #endif
73 static cl::opt<bool, true> ClobberNonLiveOverride("rs4gc-clobber-non-live",
75  cl::Hidden);
76 
77 namespace {
78 struct RewriteStatepointsForGC : public ModulePass {
79  static char ID; // Pass identification, replacement for typeid
80 
81  RewriteStatepointsForGC() : ModulePass(ID) {
83  }
84  bool runOnFunction(Function &F);
85  bool runOnModule(Module &M) override {
86  bool Changed = false;
87  for (Function &F : M)
88  Changed |= runOnFunction(F);
89 
90  if (Changed) {
91  // stripDereferenceabilityInfo asserts that shouldRewriteStatepointsIn
92  // returns true for at least one function in the module. Since at least
93  // one function changed, we know that the precondition is satisfied.
94  stripDereferenceabilityInfo(M);
95  }
96 
97  return Changed;
98  }
99 
100  void getAnalysisUsage(AnalysisUsage &AU) const override {
101  // We add and rewrite a bunch of instructions, but don't really do much
102  // else. We could in theory preserve a lot more analyses here.
105  }
106 
107  /// The IR fed into RewriteStatepointsForGC may have had attributes implying
108  /// dereferenceability that are no longer valid/correct after
109  /// RewriteStatepointsForGC has run. This is because semantically, after
110  /// RewriteStatepointsForGC runs, all calls to gc.statepoint "free" the entire
111  /// heap. stripDereferenceabilityInfo (conservatively) restores correctness
112  /// by erasing all attributes in the module that externally imply
113  /// dereferenceability.
114  ///
115  void stripDereferenceabilityInfo(Module &M);
116 
117  // Helpers for stripDereferenceabilityInfo
118  void stripDereferenceabilityInfoFromBody(Function &F);
119  void stripDereferenceabilityInfoFromPrototype(Function &F);
120 };
121 } // namespace
122 
124 
126  return new RewriteStatepointsForGC();
127 }
128 
129 INITIALIZE_PASS_BEGIN(RewriteStatepointsForGC, "rewrite-statepoints-for-gc",
130  "Make relocations explicit at statepoints", false, false)
132 INITIALIZE_PASS_END(RewriteStatepointsForGC, "rewrite-statepoints-for-gc",
133  "Make relocations explicit at statepoints", false, false)
134 
135 namespace {
137  /// Values defined in this block.
139  /// Values used in this block (and thus live); does not included values
140  /// killed within this block.
142 
143  /// Values live into this basic block (i.e. used by any
144  /// instruction in this basic block or ones reachable from here)
146 
147  /// Values live out of this basic block (i.e. live into
148  /// any successor block)
150 };
151 
152 // The type of the internal cache used inside the findBasePointers family
153 // of functions. From the callers perspective, this is an opaque type and
154 // should not be inspected.
155 //
156 // In the actual implementation this caches two relations:
157 // - The base relation itself (i.e. this pointer is based on that one)
158 // - The base defining value relation (i.e. before base_phi insertion)
159 // Generally, after the execution of a full findBasePointer call, only the
160 // base relation will remain. Internally, we add a mixture of the two
161 // types, then update all the second type to the first type
165 
167  /// The set of values known to be live accross this safepoint
168  StatepointLiveSetTy liveset;
169 
170  /// Mapping from live pointers to a base-defining-value
172 
173  /// The *new* gc.statepoint instruction itself. This produces the token
174  /// that normal path gc.relocates and the gc.result are tied to.
176 
177  /// Instruction to which exceptional gc relocates are attached
178  /// Makes it easier to iterate through them during relocationViaAlloca.
180 
181  /// Record live values we are rematerialized instead of relocating.
182  /// They are not included into 'liveset' field.
183  /// Maps rematerialized copy to it's original value.
184  RematerializedValueMapTy RematerializedValues;
185 };
186 }
187 
188 /// Compute the live-in set for every basic block in the function
189 static void computeLiveInValues(DominatorTree &DT, Function &F,
190  GCPtrLivenessData &Data);
191 
192 /// Given results from the dataflow liveness computation, find the set of live
193 /// Values at a particular instruction.
194 static void findLiveSetAtInst(Instruction *inst, GCPtrLivenessData &Data,
195  StatepointLiveSetTy &out);
196 
197 // TODO: Once we can get to the GCStrategy, this becomes
198 // Optional<bool> isGCManagedPointer(const Value *V) const override {
199 
200 static bool isGCPointerType(const Type *T) {
201  if (const PointerType *PT = dyn_cast<PointerType>(T))
202  // For the sake of this example GC, we arbitrarily pick addrspace(1) as our
203  // GC managed heap. We know that a pointer into this heap needs to be
204  // updated and that no other pointer does.
205  return (1 == PT->getAddressSpace());
206  return false;
207 }
208 
209 // Return true if this type is one which a) is a gc pointer or contains a GC
210 // pointer and b) is of a type this code expects to encounter as a live value.
211 // (The insertion code will assert that a type which matches (a) and not (b)
212 // is not encountered.)
214  // We fully support gc pointers
215  if (isGCPointerType(T))
216  return true;
217  // We partially support vectors of gc pointers. The code will assert if it
218  // can't handle something.
219  if (auto VT = dyn_cast<VectorType>(T))
220  if (isGCPointerType(VT->getElementType()))
221  return true;
222  return false;
223 }
224 
225 #ifndef NDEBUG
226 /// Returns true if this type contains a gc pointer whether we know how to
227 /// handle that type or not.
228 static bool containsGCPtrType(Type *Ty) {
229  if (isGCPointerType(Ty))
230  return true;
231  if (VectorType *VT = dyn_cast<VectorType>(Ty))
232  return isGCPointerType(VT->getScalarType());
233  if (ArrayType *AT = dyn_cast<ArrayType>(Ty))
234  return containsGCPtrType(AT->getElementType());
235  if (StructType *ST = dyn_cast<StructType>(Ty))
236  return std::any_of(
237  ST->subtypes().begin(), ST->subtypes().end(),
238  [](Type *SubType) { return containsGCPtrType(SubType); });
239  return false;
240 }
241 
242 // Returns true if this is a type which a) is a gc pointer or contains a GC
243 // pointer and b) is of a type which the code doesn't expect (i.e. first class
244 // aggregates). Used to trip assertions.
245 static bool isUnhandledGCPointerType(Type *Ty) {
246  return containsGCPtrType(Ty) && !isHandledGCPointerType(Ty);
247 }
248 #endif
249 
250 static bool order_by_name(llvm::Value *a, llvm::Value *b) {
251  if (a->hasName() && b->hasName()) {
252  return -1 == a->getName().compare(b->getName());
253  } else if (a->hasName() && !b->hasName()) {
254  return true;
255  } else if (!a->hasName() && b->hasName()) {
256  return false;
257  } else {
258  // Better than nothing, but not stable
259  return a < b;
260  }
261 }
262 
263 // Conservatively identifies any definitions which might be live at the
264 // given instruction. The analysis is performed immediately before the
265 // given instruction. Values defined by that instruction are not considered
266 // live. Values used by that instruction are considered live.
268  DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData,
269  const CallSite &CS, PartiallyConstructedSafepointRecord &result) {
270  Instruction *inst = CS.getInstruction();
271 
272  StatepointLiveSetTy liveset;
273  findLiveSetAtInst(inst, OriginalLivenessData, liveset);
274 
275  if (PrintLiveSet) {
276  // Note: This output is used by several of the test cases
277  // The order of elemtns in a set is not stable, put them in a vec and sort
278  // by name
280  temp.insert(temp.end(), liveset.begin(), liveset.end());
281  std::sort(temp.begin(), temp.end(), order_by_name);
282  errs() << "Live Variables:\n";
283  for (Value *V : temp) {
284  errs() << " " << V->getName(); // no newline
285  V->dump();
286  }
287  }
288  if (PrintLiveSetSize) {
289  errs() << "Safepoint For: " << CS.getCalledValue()->getName() << "\n";
290  errs() << "Number live values: " << liveset.size() << "\n";
291  }
292  result.liveset = liveset;
293 }
294 
296 
297 /// Return a base defining value for the 'Index' element of the given vector
298 /// instruction 'I'. If Index is null, returns a BDV for the entire vector
299 /// 'I'. As an optimization, this method will try to determine when the
300 /// element is known to already be a base pointer. If this can be established,
301 /// the second value in the returned pair will be true. Note that either a
302 /// vector or a pointer typed value can be returned. For the former, the
303 /// vector returned is a BDV (and possibly a base) of the entire vector 'I'.
304 /// If the later, the return pointer is a BDV (or possibly a base) for the
305 /// particular element in 'I'.
306 static std::pair<Value *, bool>
308  assert(I->getType()->isVectorTy() &&
309  cast<VectorType>(I->getType())->getElementType()->isPointerTy() &&
310  "Illegal to ask for the base pointer of a non-pointer type");
311 
312  // Each case parallels findBaseDefiningValue below, see that code for
313  // detailed motivation.
314 
315  if (isa<Argument>(I))
316  // An incoming argument to the function is a base pointer
317  return std::make_pair(I, true);
318 
319  // We shouldn't see the address of a global as a vector value?
320  assert(!isa<GlobalVariable>(I) &&
321  "unexpected global variable found in base of vector");
322 
323  // inlining could possibly introduce phi node that contains
324  // undef if callee has multiple returns
325  if (isa<UndefValue>(I))
326  // utterly meaningless, but useful for dealing with partially optimized
327  // code.
328  return std::make_pair(I, true);
329 
330  // Due to inheritance, this must be _after_ the global variable and undef
331  // checks
332  if (Constant *Con = dyn_cast<Constant>(I)) {
333  assert(!isa<GlobalVariable>(I) && !isa<UndefValue>(I) &&
334  "order of checks wrong!");
335  assert(Con->isNullValue() && "null is the only case which makes sense");
336  return std::make_pair(Con, true);
337  }
338 
339  if (isa<LoadInst>(I))
340  return std::make_pair(I, true);
341 
342  // For an insert element, we might be able to look through it if we know
343  // something about the indexes.
344  if (InsertElementInst *IEI = dyn_cast<InsertElementInst>(I)) {
345  if (Index) {
346  Value *InsertIndex = IEI->getOperand(2);
347  // This index is inserting the value, look for its BDV
348  if (InsertIndex == Index)
349  return std::make_pair(findBaseDefiningValue(IEI->getOperand(1)), false);
350  // Both constant, and can't be equal per above. This insert is definitely
351  // not relevant, look back at the rest of the vector and keep trying.
352  if (isa<ConstantInt>(Index) && isa<ConstantInt>(InsertIndex))
353  return findBaseDefiningValueOfVector(IEI->getOperand(0), Index);
354  }
355 
356  // We don't know whether this vector contains entirely base pointers or
357  // not. To be conservatively correct, we treat it as a BDV and will
358  // duplicate code as needed to construct a parallel vector of bases.
359  return std::make_pair(IEI, false);
360  }
361 
362  if (isa<ShuffleVectorInst>(I))
363  // We don't know whether this vector contains entirely base pointers or
364  // not. To be conservatively correct, we treat it as a BDV and will
365  // duplicate code as needed to construct a parallel vector of bases.
366  // TODO: There a number of local optimizations which could be applied here
367  // for particular sufflevector patterns.
368  return std::make_pair(I, false);
369 
370  // A PHI or Select is a base defining value. The outer findBasePointer
371  // algorithm is responsible for constructing a base value for this BDV.
372  assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
373  "unknown vector instruction - no base found for vector element");
374  return std::make_pair(I, false);
375 }
376 
377 static bool isKnownBaseResult(Value *V);
378 
379 /// Helper function for findBasePointer - Will return a value which either a)
380 /// defines the base pointer for the input or b) blocks the simple search
381 /// (i.e. a PHI or Select of two derived pointers)
383  if (I->getType()->isVectorTy())
384  return findBaseDefiningValueOfVector(I).first;
385 
386  assert(I->getType()->isPointerTy() &&
387  "Illegal to ask for the base pointer of a non-pointer type");
388 
389  // This case is a bit of a hack - it only handles extracts from vectors which
390  // trivially contain only base pointers or cases where we can directly match
391  // the index of the original extract element to an insertion into the vector.
392  // See note inside the function for how to improve this.
393  if (auto *EEI = dyn_cast<ExtractElementInst>(I)) {
394  Value *VectorOperand = EEI->getVectorOperand();
395  Value *Index = EEI->getIndexOperand();
396  std::pair<Value *, bool> pair =
397  findBaseDefiningValueOfVector(VectorOperand, Index);
398  Value *VectorBase = pair.first;
399  if (VectorBase->getType()->isPointerTy())
400  // We found a BDV for this specific element with the vector. This is an
401  // optimization, but in practice it covers most of the useful cases
402  // created via scalarization.
403  return VectorBase;
404  else {
405  assert(VectorBase->getType()->isVectorTy());
406  if (pair.second)
407  // If the entire vector returned is known to be entirely base pointers,
408  // then the extractelement is valid base for this value.
409  return EEI;
410  else {
411  // Otherwise, we have an instruction which potentially produces a
412  // derived pointer and we need findBasePointers to clone code for us
413  // such that we can create an instruction which produces the
414  // accompanying base pointer.
415  // Note: This code is currently rather incomplete. We don't currently
416  // support the general form of shufflevector of insertelement.
417  // Conceptually, these are just 'base defining values' of the same
418  // variety as phi or select instructions. We need to update the
419  // findBasePointers algorithm to insert new 'base-only' versions of the
420  // original instructions. This is relative straight forward to do, but
421  // the case which would motivate the work hasn't shown up in real
422  // workloads yet.
423  assert((isa<PHINode>(VectorBase) || isa<SelectInst>(VectorBase)) &&
424  "need to extend findBasePointers for generic vector"
425  "instruction cases");
426  return VectorBase;
427  }
428  }
429  }
430 
431  if (isa<Argument>(I))
432  // An incoming argument to the function is a base pointer
433  // We should have never reached here if this argument isn't an gc value
434  return I;
435 
436  if (isa<GlobalVariable>(I))
437  // base case
438  return I;
439 
440  // inlining could possibly introduce phi node that contains
441  // undef if callee has multiple returns
442  if (isa<UndefValue>(I))
443  // utterly meaningless, but useful for dealing with
444  // partially optimized code.
445  return I;
446 
447  // Due to inheritance, this must be _after_ the global variable and undef
448  // checks
449  if (Constant *Con = dyn_cast<Constant>(I)) {
450  assert(!isa<GlobalVariable>(I) && !isa<UndefValue>(I) &&
451  "order of checks wrong!");
452  // Note: Finding a constant base for something marked for relocation
453  // doesn't really make sense. The most likely case is either a) some
454  // screwed up the address space usage or b) your validating against
455  // compiled C++ code w/o the proper separation. The only real exception
456  // is a null pointer. You could have generic code written to index of
457  // off a potentially null value and have proven it null. We also use
458  // null pointers in dead paths of relocation phis (which we might later
459  // want to find a base pointer for).
460  assert(isa<ConstantPointerNull>(Con) &&
461  "null is the only case which makes sense");
462  return Con;
463  }
464 
465  if (CastInst *CI = dyn_cast<CastInst>(I)) {
466  Value *Def = CI->stripPointerCasts();
467  // If we find a cast instruction here, it means we've found a cast which is
468  // not simply a pointer cast (i.e. an inttoptr). We don't know how to
469  // handle int->ptr conversion.
470  assert(!isa<CastInst>(Def) && "shouldn't find another cast here");
471  return findBaseDefiningValue(Def);
472  }
473 
474  if (isa<LoadInst>(I))
475  return I; // The value loaded is an gc base itself
476 
477  if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I))
478  // The base of this GEP is the base
479  return findBaseDefiningValue(GEP->getPointerOperand());
480 
481  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
482  switch (II->getIntrinsicID()) {
483  case Intrinsic::experimental_gc_result_ptr:
484  default:
485  // fall through to general call handling
486  break;
487  case Intrinsic::experimental_gc_statepoint:
488  case Intrinsic::experimental_gc_result_float:
489  case Intrinsic::experimental_gc_result_int:
490  llvm_unreachable("these don't produce pointers");
491  case Intrinsic::experimental_gc_relocate: {
492  // Rerunning safepoint insertion after safepoints are already
493  // inserted is not supported. It could probably be made to work,
494  // but why are you doing this? There's no good reason.
495  llvm_unreachable("repeat safepoint insertion is not supported");
496  }
497  case Intrinsic::gcroot:
498  // Currently, this mechanism hasn't been extended to work with gcroot.
499  // There's no reason it couldn't be, but I haven't thought about the
500  // implications much.
502  "interaction with the gcroot mechanism is not supported");
503  }
504  }
505  // We assume that functions in the source language only return base
506  // pointers. This should probably be generalized via attributes to support
507  // both source language and internal functions.
508  if (isa<CallInst>(I) || isa<InvokeInst>(I))
509  return I;
510 
511  // I have absolutely no idea how to implement this part yet. It's not
512  // neccessarily hard, I just haven't really looked at it yet.
513  assert(!isa<LandingPadInst>(I) && "Landing Pad is unimplemented");
514 
515  if (isa<AtomicCmpXchgInst>(I))
516  // A CAS is effectively a atomic store and load combined under a
517  // predicate. From the perspective of base pointers, we just treat it
518  // like a load.
519  return I;
520 
521  assert(!isa<AtomicRMWInst>(I) && "Xchg handled above, all others are "
522  "binary ops which don't apply to pointers");
523 
524  // The aggregate ops. Aggregates can either be in the heap or on the
525  // stack, but in either case, this is simply a field load. As a result,
526  // this is a defining definition of the base just like a load is.
527  if (isa<ExtractValueInst>(I))
528  return I;
529 
530  // We should never see an insert vector since that would require we be
531  // tracing back a struct value not a pointer value.
532  assert(!isa<InsertValueInst>(I) &&
533  "Base pointer for a struct is meaningless");
534 
535  // The last two cases here don't return a base pointer. Instead, they
536  // return a value which dynamically selects from amoung several base
537  // derived pointers (each with it's own base potentially). It's the job of
538  // the caller to resolve these.
539  assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
540  "missing instruction case in findBaseDefiningValing");
541  return I;
542 }
543 
544 /// Returns the base defining value for this value.
546  Value *&Cached = Cache[I];
547  if (!Cached) {
548  Cached = findBaseDefiningValue(I);
549  }
550  assert(Cache[I] != nullptr);
551 
552  if (TraceLSP) {
553  dbgs() << "fBDV-cached: " << I->getName() << " -> " << Cached->getName()
554  << "\n";
555  }
556  return Cached;
557 }
558 
559 /// Return a base pointer for this value if known. Otherwise, return it's
560 /// base defining value.
563  auto Found = Cache.find(Def);
564  if (Found != Cache.end()) {
565  // Either a base-of relation, or a self reference. Caller must check.
566  return Found->second;
567  }
568  // Only a BDV available
569  return Def;
570 }
571 
572 /// Given the result of a call to findBaseDefiningValue, or findBaseOrBDV,
573 /// is it known to be a base pointer? Or do we need to continue searching.
574 static bool isKnownBaseResult(Value *V) {
575  if (!isa<PHINode>(V) && !isa<SelectInst>(V)) {
576  // no recursion possible
577  return true;
578  }
579  if (isa<Instruction>(V) &&
580  cast<Instruction>(V)->getMetadata("is_base_value")) {
581  // This is a previously inserted base phi or select. We know
582  // that this is a base value.
583  return true;
584  }
585 
586  // We need to keep searching
587  return false;
588 }
589 
590 // TODO: find a better name for this
591 namespace {
592 class PhiState {
593 public:
594  enum Status { Unknown, Base, Conflict };
595 
596  PhiState(Status s, Value *b = nullptr) : status(s), base(b) {
597  assert(status != Base || b);
598  }
599  PhiState(Value *b) : status(Base), base(b) {}
600  PhiState() : status(Unknown), base(nullptr) {}
601 
602  Status getStatus() const { return status; }
603  Value *getBase() const { return base; }
604 
605  bool isBase() const { return getStatus() == Base; }
606  bool isUnknown() const { return getStatus() == Unknown; }
607  bool isConflict() const { return getStatus() == Conflict; }
608 
609  bool operator==(const PhiState &other) const {
610  return base == other.base && status == other.status;
611  }
612 
613  bool operator!=(const PhiState &other) const { return !(*this == other); }
614 
615  void dump() {
616  errs() << status << " (" << base << " - "
617  << (base ? base->getName() : "nullptr") << "): ";
618  }
619 
620 private:
621  Status status;
622  Value *base; // non null only if status == base
623 };
624 
625 typedef DenseMap<Value *, PhiState> ConflictStateMapTy;
626 // Values of type PhiState form a lattice, and this is a helper
627 // class that implementes the meet operation. The meat of the meet
628 // operation is implemented in MeetPhiStates::pureMeet
629 class MeetPhiStates {
630 public:
631  // phiStates is a mapping from PHINodes and SelectInst's to PhiStates.
632  explicit MeetPhiStates(const ConflictStateMapTy &phiStates)
633  : phiStates(phiStates) {}
634 
635  // Destructively meet the current result with the base V. V can
636  // either be a merge instruction (SelectInst / PHINode), in which
637  // case its status is looked up in the phiStates map; or a regular
638  // SSA value, in which case it is assumed to be a base.
639  void meetWith(Value *V) {
640  PhiState otherState = getStateForBDV(V);
641  assert((MeetPhiStates::pureMeet(otherState, currentResult) ==
642  MeetPhiStates::pureMeet(currentResult, otherState)) &&
643  "math is wrong: meet does not commute!");
644  currentResult = MeetPhiStates::pureMeet(otherState, currentResult);
645  }
646 
647  PhiState getResult() const { return currentResult; }
648 
649 private:
650  const ConflictStateMapTy &phiStates;
651  PhiState currentResult;
652 
653  /// Return a phi state for a base defining value. We'll generate a new
654  /// base state for known bases and expect to find a cached state otherwise
655  PhiState getStateForBDV(Value *baseValue) {
656  if (isKnownBaseResult(baseValue)) {
657  return PhiState(baseValue);
658  } else {
659  return lookupFromMap(baseValue);
660  }
661  }
662 
663  PhiState lookupFromMap(Value *V) {
664  auto I = phiStates.find(V);
665  assert(I != phiStates.end() && "lookup failed!");
666  return I->second;
667  }
668 
669  static PhiState pureMeet(const PhiState &stateA, const PhiState &stateB) {
670  switch (stateA.getStatus()) {
671  case PhiState::Unknown:
672  return stateB;
673 
674  case PhiState::Base:
675  assert(stateA.getBase() && "can't be null");
676  if (stateB.isUnknown())
677  return stateA;
678 
679  if (stateB.isBase()) {
680  if (stateA.getBase() == stateB.getBase()) {
681  assert(stateA == stateB && "equality broken!");
682  return stateA;
683  }
684  return PhiState(PhiState::Conflict);
685  }
686  assert(stateB.isConflict() && "only three states!");
687  return PhiState(PhiState::Conflict);
688 
689  case PhiState::Conflict:
690  return stateA;
691  }
692  llvm_unreachable("only three states!");
693  }
694 };
695 }
696 /// For a given value or instruction, figure out what base ptr it's derived
697 /// from. For gc objects, this is simply itself. On success, returns a value
698 /// which is the base pointer. (This is reliable and can be used for
699 /// relocation.) On failure, returns nullptr.
701  Value *def = findBaseOrBDV(I, cache);
702 
703  if (isKnownBaseResult(def)) {
704  return def;
705  }
706 
707  // Here's the rough algorithm:
708  // - For every SSA value, construct a mapping to either an actual base
709  // pointer or a PHI which obscures the base pointer.
710  // - Construct a mapping from PHI to unknown TOP state. Use an
711  // optimistic algorithm to propagate base pointer information. Lattice
712  // looks like:
713  // UNKNOWN
714  // b1 b2 b3 b4
715  // CONFLICT
716  // When algorithm terminates, all PHIs will either have a single concrete
717  // base or be in a conflict state.
718  // - For every conflict, insert a dummy PHI node without arguments. Add
719  // these to the base[Instruction] = BasePtr mapping. For every
720  // non-conflict, add the actual base.
721  // - For every conflict, add arguments for the base[a] of each input
722  // arguments.
723  //
724  // Note: A simpler form of this would be to add the conflict form of all
725  // PHIs without running the optimistic algorithm. This would be
726  // analougous to pessimistic data flow and would likely lead to an
727  // overall worse solution.
728 
729  ConflictStateMapTy states;
730  states[def] = PhiState();
731  // Recursively fill in all phis & selects reachable from the initial one
732  // for which we don't already know a definite base value for
733  // TODO: This should be rewritten with a worklist
734  bool done = false;
735  while (!done) {
736  done = true;
737  // Since we're adding elements to 'states' as we run, we can't keep
738  // iterators into the set.
740  Keys.reserve(states.size());
741  for (auto Pair : states) {
742  Value *V = Pair.first;
743  Keys.push_back(V);
744  }
745  for (Value *v : Keys) {
746  assert(!isKnownBaseResult(v) && "why did it get added?");
747  if (PHINode *phi = dyn_cast<PHINode>(v)) {
748  assert(phi->getNumIncomingValues() > 0 &&
749  "zero input phis are illegal");
750  for (Value *InVal : phi->incoming_values()) {
751  Value *local = findBaseOrBDV(InVal, cache);
752  if (!isKnownBaseResult(local) && states.find(local) == states.end()) {
753  states[local] = PhiState();
754  done = false;
755  }
756  }
757  } else if (SelectInst *sel = dyn_cast<SelectInst>(v)) {
758  Value *local = findBaseOrBDV(sel->getTrueValue(), cache);
759  if (!isKnownBaseResult(local) && states.find(local) == states.end()) {
760  states[local] = PhiState();
761  done = false;
762  }
763  local = findBaseOrBDV(sel->getFalseValue(), cache);
764  if (!isKnownBaseResult(local) && states.find(local) == states.end()) {
765  states[local] = PhiState();
766  done = false;
767  }
768  }
769  }
770  }
771 
772  if (TraceLSP) {
773  errs() << "States after initialization:\n";
774  for (auto Pair : states) {
775  Instruction *v = cast<Instruction>(Pair.first);
776  PhiState state = Pair.second;
777  state.dump();
778  v->dump();
779  }
780  }
781 
782  // TODO: come back and revisit the state transitions around inputs which
783  // have reached conflict state. The current version seems too conservative.
784 
785  bool progress = true;
786  while (progress) {
787 #ifndef NDEBUG
788  size_t oldSize = states.size();
789 #endif
790  progress = false;
791  // We're only changing keys in this loop, thus safe to keep iterators
792  for (auto Pair : states) {
793  MeetPhiStates calculateMeet(states);
794  Value *v = Pair.first;
795  assert(!isKnownBaseResult(v) && "why did it get added?");
796  if (SelectInst *select = dyn_cast<SelectInst>(v)) {
797  calculateMeet.meetWith(findBaseOrBDV(select->getTrueValue(), cache));
798  calculateMeet.meetWith(findBaseOrBDV(select->getFalseValue(), cache));
799  } else
800  for (Value *Val : cast<PHINode>(v)->incoming_values())
801  calculateMeet.meetWith(findBaseOrBDV(Val, cache));
802 
803  PhiState oldState = states[v];
804  PhiState newState = calculateMeet.getResult();
805  if (oldState != newState) {
806  progress = true;
807  states[v] = newState;
808  }
809  }
810 
811  assert(oldSize <= states.size());
812  assert(oldSize == states.size() || progress);
813  }
814 
815  if (TraceLSP) {
816  errs() << "States after meet iteration:\n";
817  for (auto Pair : states) {
818  Instruction *v = cast<Instruction>(Pair.first);
819  PhiState state = Pair.second;
820  state.dump();
821  v->dump();
822  }
823  }
824 
825  // Insert Phis for all conflicts
826  // We want to keep naming deterministic in the loop that follows, so
827  // sort the keys before iteration. This is useful in allowing us to
828  // write stable tests. Note that there is no invalidation issue here.
830  Keys.reserve(states.size());
831  for (auto Pair : states) {
832  Value *V = Pair.first;
833  Keys.push_back(V);
834  }
835  std::sort(Keys.begin(), Keys.end(), order_by_name);
836  // TODO: adjust naming patterns to avoid this order of iteration dependency
837  for (Value *V : Keys) {
838  Instruction *v = cast<Instruction>(V);
839  PhiState state = states[V];
840  assert(!isKnownBaseResult(v) && "why did it get added?");
841  assert(!state.isUnknown() && "Optimistic algorithm didn't complete!");
842  if (!state.isConflict())
843  continue;
844 
845  if (isa<PHINode>(v)) {
846  int num_preds =
847  std::distance(pred_begin(v->getParent()), pred_end(v->getParent()));
848  assert(num_preds > 0 && "how did we reach here");
849  PHINode *phi = PHINode::Create(v->getType(), num_preds, "base_phi", v);
850  // Add metadata marking this as a base value
851  auto *const_1 = ConstantInt::get(
853  v->getParent()->getParent()->getParent()->getContext()),
854  1);
855  auto MDConst = ConstantAsMetadata::get(const_1);
856  MDNode *md = MDNode::get(
857  v->getParent()->getParent()->getParent()->getContext(), MDConst);
858  phi->setMetadata("is_base_value", md);
859  states[v] = PhiState(PhiState::Conflict, phi);
860  } else {
861  SelectInst *sel = cast<SelectInst>(v);
862  // The undef will be replaced later
863  UndefValue *undef = UndefValue::get(sel->getType());
864  SelectInst *basesel = SelectInst::Create(sel->getCondition(), undef,
865  undef, "base_select", sel);
866  // Add metadata marking this as a base value
867  auto *const_1 = ConstantInt::get(
869  v->getParent()->getParent()->getParent()->getContext()),
870  1);
871  auto MDConst = ConstantAsMetadata::get(const_1);
872  MDNode *md = MDNode::get(
873  v->getParent()->getParent()->getParent()->getContext(), MDConst);
874  basesel->setMetadata("is_base_value", md);
875  states[v] = PhiState(PhiState::Conflict, basesel);
876  }
877  }
878 
879  // Fixup all the inputs of the new PHIs
880  for (auto Pair : states) {
881  Instruction *v = cast<Instruction>(Pair.first);
882  PhiState state = Pair.second;
883 
884  assert(!isKnownBaseResult(v) && "why did it get added?");
885  assert(!state.isUnknown() && "Optimistic algorithm didn't complete!");
886  if (!state.isConflict())
887  continue;
888 
889  if (PHINode *basephi = dyn_cast<PHINode>(state.getBase())) {
890  PHINode *phi = cast<PHINode>(v);
891  unsigned NumPHIValues = phi->getNumIncomingValues();
892  for (unsigned i = 0; i < NumPHIValues; i++) {
893  Value *InVal = phi->getIncomingValue(i);
894  BasicBlock *InBB = phi->getIncomingBlock(i);
895 
896  // If we've already seen InBB, add the same incoming value
897  // we added for it earlier. The IR verifier requires phi
898  // nodes with multiple entries from the same basic block
899  // to have the same incoming value for each of those
900  // entries. If we don't do this check here and basephi
901  // has a different type than base, we'll end up adding two
902  // bitcasts (and hence two distinct values) as incoming
903  // values for the same basic block.
904 
905  int blockIndex = basephi->getBasicBlockIndex(InBB);
906  if (blockIndex != -1) {
907  Value *oldBase = basephi->getIncomingValue(blockIndex);
908  basephi->addIncoming(oldBase, InBB);
909 #ifndef NDEBUG
910  Value *base = findBaseOrBDV(InVal, cache);
911  if (!isKnownBaseResult(base)) {
912  // Either conflict or base.
913  assert(states.count(base));
914  base = states[base].getBase();
915  assert(base != nullptr && "unknown PhiState!");
916  }
917 
918  // In essense this assert states: the only way two
919  // values incoming from the same basic block may be
920  // different is by being different bitcasts of the same
921  // value. A cleanup that remains TODO is changing
922  // findBaseOrBDV to return an llvm::Value of the correct
923  // type (and still remain pure). This will remove the
924  // need to add bitcasts.
925  assert(base->stripPointerCasts() == oldBase->stripPointerCasts() &&
926  "sanity -- findBaseOrBDV should be pure!");
927 #endif
928  continue;
929  }
930 
931  // Find either the defining value for the PHI or the normal base for
932  // a non-phi node
933  Value *base = findBaseOrBDV(InVal, cache);
934  if (!isKnownBaseResult(base)) {
935  // Either conflict or base.
936  assert(states.count(base));
937  base = states[base].getBase();
938  assert(base != nullptr && "unknown PhiState!");
939  }
940  assert(base && "can't be null");
941  // Must use original input BB since base may not be Instruction
942  // The cast is needed since base traversal may strip away bitcasts
943  if (base->getType() != basephi->getType()) {
944  base = new BitCastInst(base, basephi->getType(), "cast",
945  InBB->getTerminator());
946  }
947  basephi->addIncoming(base, InBB);
948  }
949  assert(basephi->getNumIncomingValues() == NumPHIValues);
950  } else {
951  SelectInst *basesel = cast<SelectInst>(state.getBase());
952  SelectInst *sel = cast<SelectInst>(v);
953  // Operand 1 & 2 are true, false path respectively. TODO: refactor to
954  // something more safe and less hacky.
955  for (int i = 1; i <= 2; i++) {
956  Value *InVal = sel->getOperand(i);
957  // Find either the defining value for the PHI or the normal base for
958  // a non-phi node
959  Value *base = findBaseOrBDV(InVal, cache);
960  if (!isKnownBaseResult(base)) {
961  // Either conflict or base.
962  assert(states.count(base));
963  base = states[base].getBase();
964  assert(base != nullptr && "unknown PhiState!");
965  }
966  assert(base && "can't be null");
967  // Must use original input BB since base may not be Instruction
968  // The cast is needed since base traversal may strip away bitcasts
969  if (base->getType() != basesel->getType()) {
970  base = new BitCastInst(base, basesel->getType(), "cast", basesel);
971  }
972  basesel->setOperand(i, base);
973  }
974  }
975  }
976 
977  // Cache all of our results so we can cheaply reuse them
978  // NOTE: This is actually two caches: one of the base defining value
979  // relation and one of the base pointer relation! FIXME
980  for (auto item : states) {
981  Value *v = item.first;
982  Value *base = item.second.getBase();
983  assert(v && base);
984  assert(!isKnownBaseResult(v) && "why did it get added?");
985 
986  if (TraceLSP) {
987  std::string fromstr =
988  cache.count(v) ? (cache[v]->hasName() ? cache[v]->getName() : "")
989  : "none";
990  errs() << "Updating base value cache"
991  << " for: " << (v->hasName() ? v->getName() : "")
992  << " from: " << fromstr
993  << " to: " << (base->hasName() ? base->getName() : "") << "\n";
994  }
995 
996  assert(isKnownBaseResult(base) &&
997  "must be something we 'know' is a base pointer");
998  if (cache.count(v)) {
999  // Once we transition from the BDV relation being store in the cache to
1000  // the base relation being stored, it must be stable
1001  assert((!isKnownBaseResult(cache[v]) || cache[v] == base) &&
1002  "base relation should be stable");
1003  }
1004  cache[v] = base;
1005  }
1006  assert(cache.find(def) != cache.end());
1007  return cache[def];
1008 }
1009 
1010 // For a set of live pointers (base and/or derived), identify the base
1011 // pointer of the object which they are derived from. This routine will
1012 // mutate the IR graph as needed to make the 'base' pointer live at the
1013 // definition site of 'derived'. This ensures that any use of 'derived' can
1014 // also use 'base'. This may involve the insertion of a number of
1015 // additional PHI nodes.
1016 //
1017 // preconditions: live is a set of pointer type Values
1018 //
1019 // side effects: may insert PHI nodes into the existing CFG, will preserve
1020 // CFG, will not remove or mutate any existing nodes
1021 //
1022 // post condition: PointerToBase contains one (derived, base) pair for every
1023 // pointer in live. Note that derived can be equal to base if the original
1024 // pointer was a base pointer.
1025 static void
1028  DominatorTree *DT, DefiningValueMapTy &DVCache) {
1029  // For the naming of values inserted to be deterministic - which makes for
1030  // much cleaner and more stable tests - we need to assign an order to the
1031  // live values. DenseSets do not provide a deterministic order across runs.
1033  Temp.insert(Temp.end(), live.begin(), live.end());
1034  std::sort(Temp.begin(), Temp.end(), order_by_name);
1035  for (Value *ptr : Temp) {
1036  Value *base = findBasePointer(ptr, DVCache);
1037  assert(base && "failed to find base pointer");
1038  PointerToBase[ptr] = base;
1039  assert((!isa<Instruction>(base) || !isa<Instruction>(ptr) ||
1040  DT->dominates(cast<Instruction>(base)->getParent(),
1041  cast<Instruction>(ptr)->getParent())) &&
1042  "The base we found better dominate the derived pointer");
1043 
1044  // If you see this trip and like to live really dangerously, the code should
1045  // be correct, just with idioms the verifier can't handle. You can try
1046  // disabling the verifier at your own substaintial risk.
1047  assert(!isa<ConstantPointerNull>(base) &&
1048  "the relocation code needs adjustment to handle the relocation of "
1049  "a null pointer constant without causing false positives in the "
1050  "safepoint ir verifier.");
1051  }
1052 }
1053 
1054 /// Find the required based pointers (and adjust the live set) for the given
1055 /// parse point.
1057  const CallSite &CS,
1058  PartiallyConstructedSafepointRecord &result) {
1060  findBasePointers(result.liveset, PointerToBase, &DT, DVCache);
1061 
1062  if (PrintBasePointers) {
1063  // Note: Need to print these in a stable order since this is checked in
1064  // some tests.
1065  errs() << "Base Pairs (w/o Relocation):\n";
1067  Temp.reserve(PointerToBase.size());
1068  for (auto Pair : PointerToBase) {
1069  Temp.push_back(Pair.first);
1070  }
1071  std::sort(Temp.begin(), Temp.end(), order_by_name);
1072  for (Value *Ptr : Temp) {
1073  Value *Base = PointerToBase[Ptr];
1074  errs() << " derived %" << Ptr->getName() << " base %" << Base->getName()
1075  << "\n";
1076  }
1077  }
1078 
1079  result.PointerToBase = PointerToBase;
1080 }
1081 
1082 /// Given an updated version of the dataflow liveness results, update the
1083 /// liveset and base pointer maps for the call site CS.
1084 static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
1085  const CallSite &CS,
1086  PartiallyConstructedSafepointRecord &result);
1087 
1089  Function &F, DominatorTree &DT, Pass *P, ArrayRef<CallSite> toUpdate,
1091  // TODO-PERF: reuse the original liveness, then simply run the dataflow
1092  // again. The old values are still live and will help it stablize quickly.
1093  GCPtrLivenessData RevisedLivenessData;
1094  computeLiveInValues(DT, F, RevisedLivenessData);
1095  for (size_t i = 0; i < records.size(); i++) {
1096  struct PartiallyConstructedSafepointRecord &info = records[i];
1097  const CallSite &CS = toUpdate[i];
1098  recomputeLiveInValues(RevisedLivenessData, CS, info);
1099  }
1100 }
1101 
1102 // When inserting gc.relocate calls, we need to ensure there are no uses
1103 // of the original value between the gc.statepoint and the gc.relocate call.
1104 // One case which can arise is a phi node starting one of the successor blocks.
1105 // We also need to be able to insert the gc.relocates only on the path which
1106 // goes through the statepoint. We might need to split an edge to make this
1107 // possible.
1108 static BasicBlock *
1110  DominatorTree &DT) {
1111  BasicBlock *Ret = BB;
1112  if (!BB->getUniquePredecessor()) {
1113  Ret = SplitBlockPredecessors(BB, InvokeParent, "", nullptr, &DT);
1114  }
1115 
1116  // Now that 'ret' has unique predecessor we can safely remove all phi nodes
1117  // from it
1119  assert(!isa<PHINode>(Ret->begin()));
1120 
1121  // At this point, we can safely insert a gc.relocate as the first instruction
1122  // in Ret if needed.
1123  return Ret;
1124 }
1125 
1126 static int find_index(ArrayRef<Value *> livevec, Value *val) {
1127  auto itr = std::find(livevec.begin(), livevec.end(), val);
1128  assert(livevec.end() != itr);
1129  size_t index = std::distance(livevec.begin(), itr);
1130  assert(index < livevec.size());
1131  return index;
1132 }
1133 
1134 // Create new attribute set containing only attributes which can be transfered
1135 // from original call to the safepoint.
1137  AttributeSet ret;
1138 
1139  for (unsigned Slot = 0; Slot < AS.getNumSlots(); Slot++) {
1140  unsigned index = AS.getSlotIndex(Slot);
1141 
1142  if (index == AttributeSet::ReturnIndex ||
1143  index == AttributeSet::FunctionIndex) {
1144 
1145  for (auto it = AS.begin(Slot), it_end = AS.end(Slot); it != it_end;
1146  ++it) {
1147  Attribute attr = *it;
1148 
1149  // Do not allow certain attributes - just skip them
1150  // Safepoint can not be read only or read none.
1151  if (attr.hasAttribute(Attribute::ReadNone) ||
1153  continue;
1154 
1155  ret = ret.addAttributes(
1156  AS.getContext(), index,
1157  AttributeSet::get(AS.getContext(), index, AttrBuilder(attr)));
1158  }
1159  }
1160 
1161  // Just skip parameter attributes for now
1162  }
1163 
1164  return ret;
1165 }
1166 
1167 /// Helper function to place all gc relocates necessary for the given
1168 /// statepoint.
1169 /// Inputs:
1170 /// liveVariables - list of variables to be relocated.
1171 /// liveStart - index of the first live variable.
1172 /// basePtrs - base pointers.
1173 /// statepointToken - statepoint instruction to which relocates should be
1174 /// bound.
1175 /// Builder - Llvm IR builder to be used to construct new calls.
1177  const int LiveStart,
1178  ArrayRef<llvm::Value *> BasePtrs,
1179  Instruction *StatepointToken,
1180  IRBuilder<> Builder) {
1182  NewDefs.reserve(LiveVariables.size());
1183 
1184  Module *M = StatepointToken->getParent()->getParent()->getParent();
1185 
1186  for (unsigned i = 0; i < LiveVariables.size(); i++) {
1187  // We generate a (potentially) unique declaration for every pointer type
1188  // combination. This results is some blow up the function declarations in
1189  // the IR, but removes the need for argument bitcasts which shrinks the IR
1190  // greatly and makes it much more readable.
1191  SmallVector<Type *, 1> Types; // one per 'any' type
1192  // All gc_relocate are set to i8 addrspace(1)* type. This could help avoid
1193  // cases where the actual value's type mangling is not supported by llvm. A
1194  // bitcast is added later to convert gc_relocate to the actual value's type.
1195  Types.push_back(Type::getInt8PtrTy(M->getContext(), 1));
1196  Value *GCRelocateDecl = Intrinsic::getDeclaration(
1197  M, Intrinsic::experimental_gc_relocate, Types);
1198 
1199  // Generate the gc.relocate call and save the result
1200  Value *BaseIdx =
1202  LiveStart + find_index(LiveVariables, BasePtrs[i]));
1203  Value *LiveIdx = ConstantInt::get(
1205  LiveStart + find_index(LiveVariables, LiveVariables[i]));
1206 
1207  // only specify a debug name if we can give a useful one
1208  Value *Reloc = Builder.CreateCall(
1209  GCRelocateDecl, {StatepointToken, BaseIdx, LiveIdx},
1210  LiveVariables[i]->hasName() ? LiveVariables[i]->getName() + ".relocated"
1211  : "");
1212  // Trick CodeGen into thinking there are lots of free registers at this
1213  // fake call.
1214  cast<CallInst>(Reloc)->setCallingConv(CallingConv::Cold);
1215 
1216  NewDefs.push_back(cast<Instruction>(Reloc));
1217  }
1218  assert(NewDefs.size() == LiveVariables.size() &&
1219  "missing or extra redefinition at safepoint");
1220 }
1221 
1222 static void
1223 makeStatepointExplicitImpl(const CallSite &CS, /* to replace */
1224  const SmallVectorImpl<llvm::Value *> &basePtrs,
1225  const SmallVectorImpl<llvm::Value *> &liveVariables,
1226  Pass *P,
1227  PartiallyConstructedSafepointRecord &result) {
1228  assert(basePtrs.size() == liveVariables.size());
1229  assert(isStatepoint(CS) &&
1230  "This method expects to be rewriting a statepoint");
1231 
1232  BasicBlock *BB = CS.getInstruction()->getParent();
1233  assert(BB);
1234  Function *F = BB->getParent();
1235  assert(F && "must be set");
1236  Module *M = F->getParent();
1237  (void)M;
1238  assert(M && "must be set");
1239 
1240  // We're not changing the function signature of the statepoint since the gc
1241  // arguments go into the var args section.
1242  Function *gc_statepoint_decl = CS.getCalledFunction();
1243 
1244  // Then go ahead and use the builder do actually do the inserts. We insert
1245  // immediately before the previous instruction under the assumption that all
1246  // arguments will be available here. We can't insert afterwards since we may
1247  // be replacing a terminator.
1248  Instruction *insertBefore = CS.getInstruction();
1249  IRBuilder<> Builder(insertBefore);
1250  // Copy all of the arguments from the original statepoint - this includes the
1251  // target, call args, and deopt args
1253  args.insert(args.end(), CS.arg_begin(), CS.arg_end());
1254  // TODO: Clear the 'needs rewrite' flag
1255 
1256  // add all the pointers to be relocated (gc arguments)
1257  // Capture the start of the live variable list for use in the gc_relocates
1258  const int live_start = args.size();
1259  args.insert(args.end(), liveVariables.begin(), liveVariables.end());
1260 
1261  // Create the statepoint given all the arguments
1262  Instruction *token = nullptr;
1263  AttributeSet return_attributes;
1264  if (CS.isCall()) {
1265  CallInst *toReplace = cast<CallInst>(CS.getInstruction());
1266  CallInst *call =
1267  Builder.CreateCall(gc_statepoint_decl, args, "safepoint_token");
1268  call->setTailCall(toReplace->isTailCall());
1269  call->setCallingConv(toReplace->getCallingConv());
1270 
1271  // Currently we will fail on parameter attributes and on certain
1272  // function attributes.
1273  AttributeSet new_attrs = legalizeCallAttributes(toReplace->getAttributes());
1274  // In case if we can handle this set of sttributes - set up function attrs
1275  // directly on statepoint and return attrs later for gc_result intrinsic.
1276  call->setAttributes(new_attrs.getFnAttributes());
1277  return_attributes = new_attrs.getRetAttributes();
1278 
1279  token = call;
1280 
1281  // Put the following gc_result and gc_relocate calls immediately after the
1282  // the old call (which we're about to delete)
1283  BasicBlock::iterator next(toReplace);
1284  assert(BB->end() != next && "not a terminator, must have next");
1285  next++;
1286  Instruction *IP = &*(next);
1287  Builder.SetInsertPoint(IP);
1288  Builder.SetCurrentDebugLocation(IP->getDebugLoc());
1289 
1290  } else {
1291  InvokeInst *toReplace = cast<InvokeInst>(CS.getInstruction());
1292 
1293  // Insert the new invoke into the old block. We'll remove the old one in a
1294  // moment at which point this will become the new terminator for the
1295  // original block.
1296  InvokeInst *invoke = InvokeInst::Create(
1297  gc_statepoint_decl, toReplace->getNormalDest(),
1298  toReplace->getUnwindDest(), args, "", toReplace->getParent());
1299  invoke->setCallingConv(toReplace->getCallingConv());
1300 
1301  // Currently we will fail on parameter attributes and on certain
1302  // function attributes.
1303  AttributeSet new_attrs = legalizeCallAttributes(toReplace->getAttributes());
1304  // In case if we can handle this set of sttributes - set up function attrs
1305  // directly on statepoint and return attrs later for gc_result intrinsic.
1306  invoke->setAttributes(new_attrs.getFnAttributes());
1307  return_attributes = new_attrs.getRetAttributes();
1308 
1309  token = invoke;
1310 
1311  // Generate gc relocates in exceptional path
1312  BasicBlock *unwindBlock = toReplace->getUnwindDest();
1313  assert(!isa<PHINode>(unwindBlock->begin()) &&
1314  unwindBlock->getUniquePredecessor() &&
1315  "can't safely insert in this block!");
1316 
1317  Instruction *IP = &*(unwindBlock->getFirstInsertionPt());
1318  Builder.SetInsertPoint(IP);
1319  Builder.SetCurrentDebugLocation(toReplace->getDebugLoc());
1320 
1321  // Extract second element from landingpad return value. We will attach
1322  // exceptional gc relocates to it.
1323  const unsigned idx = 1;
1324  Instruction *exceptional_token =
1325  cast<Instruction>(Builder.CreateExtractValue(
1326  unwindBlock->getLandingPadInst(), idx, "relocate_token"));
1327  result.UnwindToken = exceptional_token;
1328 
1329  // Just throw away return value. We will use the one we got for normal
1330  // block.
1331  (void)CreateGCRelocates(liveVariables, live_start, basePtrs,
1332  exceptional_token, Builder);
1333 
1334  // Generate gc relocates and returns for normal block
1335  BasicBlock *normalDest = toReplace->getNormalDest();
1336  assert(!isa<PHINode>(normalDest->begin()) &&
1337  normalDest->getUniquePredecessor() &&
1338  "can't safely insert in this block!");
1339 
1340  IP = &*(normalDest->getFirstInsertionPt());
1341  Builder.SetInsertPoint(IP);
1342 
1343  // gc relocates will be generated later as if it were regular call
1344  // statepoint
1345  }
1346  assert(token);
1347 
1348  // Take the name of the original value call if it had one.
1349  token->takeName(CS.getInstruction());
1350 
1351 // The GCResult is already inserted, we just need to find it
1352 #ifndef NDEBUG
1353  Instruction *toReplace = CS.getInstruction();
1354  assert((toReplace->hasNUses(0) || toReplace->hasNUses(1)) &&
1355  "only valid use before rewrite is gc.result");
1356  assert(!toReplace->hasOneUse() ||
1357  isGCResult(cast<Instruction>(*toReplace->user_begin())));
1358 #endif
1359 
1360  // Update the gc.result of the original statepoint (if any) to use the newly
1361  // inserted statepoint. This is safe to do here since the token can't be
1362  // considered a live reference.
1363  CS.getInstruction()->replaceAllUsesWith(token);
1364 
1365  result.StatepointToken = token;
1366 
1367  // Second, create a gc.relocate for every live variable
1368  CreateGCRelocates(liveVariables, live_start, basePtrs, token, Builder);
1369 }
1370 
1371 namespace {
1372 struct name_ordering {
1373  Value *base;
1374  Value *derived;
1375  bool operator()(name_ordering const &a, name_ordering const &b) {
1376  return -1 == a.derived->getName().compare(b.derived->getName());
1377  }
1378 };
1379 }
1381  SmallVectorImpl<Value *> &livevec) {
1382  assert(basevec.size() == livevec.size());
1383 
1385  for (size_t i = 0; i < basevec.size(); i++) {
1386  name_ordering v;
1387  v.base = basevec[i];
1388  v.derived = livevec[i];
1389  temp.push_back(v);
1390  }
1391  std::sort(temp.begin(), temp.end(), name_ordering());
1392  for (size_t i = 0; i < basevec.size(); i++) {
1393  basevec[i] = temp[i].base;
1394  livevec[i] = temp[i].derived;
1395  }
1396 }
1397 
1398 // Replace an existing gc.statepoint with a new one and a set of gc.relocates
1399 // which make the relocations happening at this safepoint explicit.
1400 //
1401 // WARNING: Does not do any fixup to adjust users of the original live
1402 // values. That's the callers responsibility.
1403 static void
1405  PartiallyConstructedSafepointRecord &result) {
1406  auto liveset = result.liveset;
1407  auto PointerToBase = result.PointerToBase;
1408 
1409  // Convert to vector for efficient cross referencing.
1410  SmallVector<Value *, 64> basevec, livevec;
1411  livevec.reserve(liveset.size());
1412  basevec.reserve(liveset.size());
1413  for (Value *L : liveset) {
1414  livevec.push_back(L);
1415 
1416  assert(PointerToBase.find(L) != PointerToBase.end());
1417  Value *base = PointerToBase[L];
1418  basevec.push_back(base);
1419  }
1420  assert(livevec.size() == basevec.size());
1421 
1422  // To make the output IR slightly more stable (for use in diffs), ensure a
1423  // fixed order of the values in the safepoint (by sorting the value name).
1424  // The order is otherwise meaningless.
1425  stablize_order(basevec, livevec);
1426 
1427  // Do the actual rewriting and delete the old statepoint
1428  makeStatepointExplicitImpl(CS, basevec, livevec, P, result);
1430 }
1431 
1432 // Helper function for the relocationViaAlloca.
1433 // It receives iterator to the statepoint gc relocates and emits store to the
1434 // assigned
1435 // location (via allocaMap) for the each one of them.
1436 // Add visited values into the visitedLiveValues set we will later use them
1437 // for sanity check.
1438 static void
1440  DenseMap<Value *, Value *> &AllocaMap,
1441  DenseSet<Value *> &VisitedLiveValues) {
1442 
1443  for (User *U : GCRelocs) {
1444  if (!isa<IntrinsicInst>(U))
1445  continue;
1446 
1447  IntrinsicInst *RelocatedValue = cast<IntrinsicInst>(U);
1448 
1449  // We only care about relocates
1450  if (RelocatedValue->getIntrinsicID() !=
1451  Intrinsic::experimental_gc_relocate) {
1452  continue;
1453  }
1454 
1455  GCRelocateOperands RelocateOperands(RelocatedValue);
1456  Value *OriginalValue =
1457  const_cast<Value *>(RelocateOperands.getDerivedPtr());
1458  assert(AllocaMap.count(OriginalValue));
1459  Value *Alloca = AllocaMap[OriginalValue];
1460 
1461  // Emit store into the related alloca
1462  // All gc_relocate are i8 addrspace(1)* typed, and it must be bitcasted to
1463  // the correct type according to alloca.
1464  assert(RelocatedValue->getNextNode() && "Should always have one since it's not a terminator");
1465  IRBuilder<> Builder(RelocatedValue->getNextNode());
1466  Value *CastedRelocatedValue =
1467  Builder.CreateBitCast(RelocatedValue, cast<AllocaInst>(Alloca)->getAllocatedType(),
1468  RelocatedValue->hasName() ? RelocatedValue->getName() + ".casted" : "");
1469 
1470  StoreInst *Store = new StoreInst(CastedRelocatedValue, Alloca);
1471  Store->insertAfter(cast<Instruction>(CastedRelocatedValue));
1472 
1473 #ifndef NDEBUG
1474  VisitedLiveValues.insert(OriginalValue);
1475 #endif
1476  }
1477 }
1478 
1479 // Helper function for the "relocationViaAlloca". Similar to the
1480 // "insertRelocationStores" but works for rematerialized values.
1481 static void
1483  RematerializedValueMapTy RematerializedValues,
1484  DenseMap<Value *, Value *> &AllocaMap,
1485  DenseSet<Value *> &VisitedLiveValues) {
1486 
1487  for (auto RematerializedValuePair: RematerializedValues) {
1488  Instruction *RematerializedValue = RematerializedValuePair.first;
1489  Value *OriginalValue = RematerializedValuePair.second;
1490 
1491  assert(AllocaMap.count(OriginalValue) &&
1492  "Can not find alloca for rematerialized value");
1493  Value *Alloca = AllocaMap[OriginalValue];
1494 
1495  StoreInst *Store = new StoreInst(RematerializedValue, Alloca);
1496  Store->insertAfter(RematerializedValue);
1497 
1498 #ifndef NDEBUG
1499  VisitedLiveValues.insert(OriginalValue);
1500 #endif
1501  }
1502 }
1503 
1504 /// do all the relocation update via allocas and mem2reg
1508 #ifndef NDEBUG
1509  // record initial number of (static) allocas; we'll check we have the same
1510  // number when we get done.
1511  int InitialAllocaNum = 0;
1512  for (auto I = F.getEntryBlock().begin(), E = F.getEntryBlock().end(); I != E;
1513  I++)
1514  if (isa<AllocaInst>(*I))
1515  InitialAllocaNum++;
1516 #endif
1517 
1518  // TODO-PERF: change data structures, reserve
1519  DenseMap<Value *, Value *> AllocaMap;
1520  SmallVector<AllocaInst *, 200> PromotableAllocas;
1521  // Used later to chack that we have enough allocas to store all values
1522  std::size_t NumRematerializedValues = 0;
1523  PromotableAllocas.reserve(Live.size());
1524 
1525  // Emit alloca for "LiveValue" and record it in "allocaMap" and
1526  // "PromotableAllocas"
1527  auto emitAllocaFor = [&](Value *LiveValue) {
1528  AllocaInst *Alloca = new AllocaInst(LiveValue->getType(), "",
1530  AllocaMap[LiveValue] = Alloca;
1531  PromotableAllocas.push_back(Alloca);
1532  };
1533 
1534  // emit alloca for each live gc pointer
1535  for (unsigned i = 0; i < Live.size(); i++) {
1536  emitAllocaFor(Live[i]);
1537  }
1538 
1539  // emit allocas for rematerialized values
1540  for (size_t i = 0; i < Records.size(); i++) {
1541  const struct PartiallyConstructedSafepointRecord &Info = Records[i];
1542 
1543  for (auto RematerializedValuePair : Info.RematerializedValues) {
1544  Value *OriginalValue = RematerializedValuePair.second;
1545  if (AllocaMap.count(OriginalValue) != 0)
1546  continue;
1547 
1548  emitAllocaFor(OriginalValue);
1549  ++NumRematerializedValues;
1550  }
1551  }
1552 
1553  // The next two loops are part of the same conceptual operation. We need to
1554  // insert a store to the alloca after the original def and at each
1555  // redefinition. We need to insert a load before each use. These are split
1556  // into distinct loops for performance reasons.
1557 
1558  // update gc pointer after each statepoint
1559  // either store a relocated value or null (if no relocated value found for
1560  // this gc pointer and it is not a gc_result)
1561  // this must happen before we update the statepoint with load of alloca
1562  // otherwise we lose the link between statepoint and old def
1563  for (size_t i = 0; i < Records.size(); i++) {
1564  const struct PartiallyConstructedSafepointRecord &Info = Records[i];
1565  Value *Statepoint = Info.StatepointToken;
1566 
1567  // This will be used for consistency check
1568  DenseSet<Value *> VisitedLiveValues;
1569 
1570  // Insert stores for normal statepoint gc relocates
1571  insertRelocationStores(Statepoint->users(), AllocaMap, VisitedLiveValues);
1572 
1573  // In case if it was invoke statepoint
1574  // we will insert stores for exceptional path gc relocates.
1575  if (isa<InvokeInst>(Statepoint)) {
1576  insertRelocationStores(Info.UnwindToken->users(), AllocaMap,
1577  VisitedLiveValues);
1578  }
1579 
1580  // Do similar thing with rematerialized values
1581  insertRematerializationStores(Info.RematerializedValues, AllocaMap,
1582  VisitedLiveValues);
1583 
1584  if (ClobberNonLive) {
1585  // As a debuging aid, pretend that an unrelocated pointer becomes null at
1586  // the gc.statepoint. This will turn some subtle GC problems into
1587  // slightly easier to debug SEGVs. Note that on large IR files with
1588  // lots of gc.statepoints this is extremely costly both memory and time
1589  // wise.
1591  for (auto Pair : AllocaMap) {
1592  Value *Def = Pair.first;
1593  AllocaInst *Alloca = cast<AllocaInst>(Pair.second);
1594 
1595  // This value was relocated
1596  if (VisitedLiveValues.count(Def)) {
1597  continue;
1598  }
1599  ToClobber.push_back(Alloca);
1600  }
1601 
1602  auto InsertClobbersAt = [&](Instruction *IP) {
1603  for (auto *AI : ToClobber) {
1604  auto AIType = cast<PointerType>(AI->getType());
1605  auto PT = cast<PointerType>(AIType->getElementType());
1607  StoreInst *Store = new StoreInst(CPN, AI);
1608  Store->insertBefore(IP);
1609  }
1610  };
1611 
1612  // Insert the clobbering stores. These may get intermixed with the
1613  // gc.results and gc.relocates, but that's fine.
1614  if (auto II = dyn_cast<InvokeInst>(Statepoint)) {
1615  InsertClobbersAt(II->getNormalDest()->getFirstInsertionPt());
1616  InsertClobbersAt(II->getUnwindDest()->getFirstInsertionPt());
1617  } else {
1618  BasicBlock::iterator Next(cast<CallInst>(Statepoint));
1619  Next++;
1620  InsertClobbersAt(Next);
1621  }
1622  }
1623  }
1624  // update use with load allocas and add store for gc_relocated
1625  for (auto Pair : AllocaMap) {
1626  Value *Def = Pair.first;
1627  Value *Alloca = Pair.second;
1628 
1629  // we pre-record the uses of allocas so that we dont have to worry about
1630  // later update
1631  // that change the user information.
1633  // PERF: trade a linear scan for repeated reallocation
1634  Uses.reserve(std::distance(Def->user_begin(), Def->user_end()));
1635  for (User *U : Def->users()) {
1636  if (!isa<ConstantExpr>(U)) {
1637  // If the def has a ConstantExpr use, then the def is either a
1638  // ConstantExpr use itself or null. In either case
1639  // (recursively in the first, directly in the second), the oop
1640  // it is ultimately dependent on is null and this particular
1641  // use does not need to be fixed up.
1642  Uses.push_back(cast<Instruction>(U));
1643  }
1644  }
1645 
1646  std::sort(Uses.begin(), Uses.end());
1647  auto Last = std::unique(Uses.begin(), Uses.end());
1648  Uses.erase(Last, Uses.end());
1649 
1650  for (Instruction *Use : Uses) {
1651  if (isa<PHINode>(Use)) {
1652  PHINode *Phi = cast<PHINode>(Use);
1653  for (unsigned i = 0; i < Phi->getNumIncomingValues(); i++) {
1654  if (Def == Phi->getIncomingValue(i)) {
1655  LoadInst *Load = new LoadInst(
1656  Alloca, "", Phi->getIncomingBlock(i)->getTerminator());
1657  Phi->setIncomingValue(i, Load);
1658  }
1659  }
1660  } else {
1661  LoadInst *Load = new LoadInst(Alloca, "", Use);
1662  Use->replaceUsesOfWith(Def, Load);
1663  }
1664  }
1665 
1666  // emit store for the initial gc value
1667  // store must be inserted after load, otherwise store will be in alloca's
1668  // use list and an extra load will be inserted before it
1669  StoreInst *Store = new StoreInst(Def, Alloca);
1670  if (Instruction *Inst = dyn_cast<Instruction>(Def)) {
1671  if (InvokeInst *Invoke = dyn_cast<InvokeInst>(Inst)) {
1672  // InvokeInst is a TerminatorInst so the store need to be inserted
1673  // into its normal destination block.
1674  BasicBlock *NormalDest = Invoke->getNormalDest();
1675  Store->insertBefore(NormalDest->getFirstNonPHI());
1676  } else {
1677  assert(!Inst->isTerminator() &&
1678  "The only TerminatorInst that can produce a value is "
1679  "InvokeInst which is handled above.");
1680  Store->insertAfter(Inst);
1681  }
1682  } else {
1683  assert(isa<Argument>(Def));
1684  Store->insertAfter(cast<Instruction>(Alloca));
1685  }
1686  }
1687 
1688  assert(PromotableAllocas.size() == Live.size() + NumRematerializedValues &&
1689  "we must have the same allocas with lives");
1690  if (!PromotableAllocas.empty()) {
1691  // apply mem2reg to promote alloca to SSA
1692  PromoteMemToReg(PromotableAllocas, DT);
1693  }
1694 
1695 #ifndef NDEBUG
1696  for (auto I = F.getEntryBlock().begin(), E = F.getEntryBlock().end(); I != E;
1697  I++)
1698  if (isa<AllocaInst>(*I))
1699  InitialAllocaNum--;
1700  assert(InitialAllocaNum == 0 && "We must not introduce any extra allocas");
1701 #endif
1702 }
1703 
1704 /// Implement a unique function which doesn't require we sort the input
1705 /// vector. Doing so has the effect of changing the output of a couple of
1706 /// tests in ways which make them less useful in testing fused safepoints.
1707 template <typename T> static void unique_unsorted(SmallVectorImpl<T> &Vec) {
1708  SmallSet<T, 8> Seen;
1709  Vec.erase(std::remove_if(Vec.begin(), Vec.end(), [&](const T &V) {
1710  return !Seen.insert(V).second;
1711  }), Vec.end());
1712 }
1713 
1714 /// Insert holders so that each Value is obviously live through the entire
1715 /// lifetime of the call.
1716 static void insertUseHolderAfter(CallSite &CS, const ArrayRef<Value *> Values,
1717  SmallVectorImpl<CallInst *> &Holders) {
1718  if (Values.empty())
1719  // No values to hold live, might as well not insert the empty holder
1720  return;
1721 
1722  Module *M = CS.getInstruction()->getParent()->getParent()->getParent();
1723  // Use a dummy vararg function to actually hold the values live
1724  Function *Func = cast<Function>(M->getOrInsertFunction(
1725  "__tmp_use", FunctionType::get(Type::getVoidTy(M->getContext()), true)));
1726  if (CS.isCall()) {
1727  // For call safepoints insert dummy calls right after safepoint
1729  Next++;
1730  Holders.push_back(CallInst::Create(Func, Values, "", Next));
1731  return;
1732  }
1733  // For invoke safepooints insert dummy calls both in normal and
1734  // exceptional destination blocks
1735  auto *II = cast<InvokeInst>(CS.getInstruction());
1736  Holders.push_back(CallInst::Create(
1737  Func, Values, "", II->getNormalDest()->getFirstInsertionPt()));
1738  Holders.push_back(CallInst::Create(
1739  Func, Values, "", II->getUnwindDest()->getFirstInsertionPt()));
1740 }
1741 
1743  Function &F, DominatorTree &DT, Pass *P, ArrayRef<CallSite> toUpdate,
1745  GCPtrLivenessData OriginalLivenessData;
1746  computeLiveInValues(DT, F, OriginalLivenessData);
1747  for (size_t i = 0; i < records.size(); i++) {
1748  struct PartiallyConstructedSafepointRecord &info = records[i];
1749  const CallSite &CS = toUpdate[i];
1750  analyzeParsePointLiveness(DT, OriginalLivenessData, CS, info);
1751  }
1752 }
1753 
1754 /// Remove any vector of pointers from the liveset by scalarizing them over the
1755 /// statepoint instruction. Adds the scalarized pieces to the liveset. It
1756 /// would be preferrable to include the vector in the statepoint itself, but
1757 /// the lowering code currently does not handle that. Extending it would be
1758 /// slightly non-trivial since it requires a format change. Given how rare
1759 /// such cases are (for the moment?) scalarizing is an acceptable comprimise.
1760 static void splitVectorValues(Instruction *StatepointInst,
1761  StatepointLiveSetTy &LiveSet,
1762  DenseMap<Value *, Value *>& PointerToBase,
1763  DominatorTree &DT) {
1764  SmallVector<Value *, 16> ToSplit;
1765  for (Value *V : LiveSet)
1766  if (isa<VectorType>(V->getType()))
1767  ToSplit.push_back(V);
1768 
1769  if (ToSplit.empty())
1770  return;
1771 
1773 
1774  Function &F = *(StatepointInst->getParent()->getParent());
1775 
1777  // First is normal return, second is exceptional return (invoke only)
1779  for (Value *V : ToSplit) {
1780  AllocaInst *Alloca =
1781  new AllocaInst(V->getType(), "", F.getEntryBlock().getFirstNonPHI());
1782  AllocaMap[V] = Alloca;
1783 
1784  VectorType *VT = cast<VectorType>(V->getType());
1785  IRBuilder<> Builder(StatepointInst);
1786  SmallVector<Value *, 16> Elements;
1787  for (unsigned i = 0; i < VT->getNumElements(); i++)
1788  Elements.push_back(Builder.CreateExtractElement(V, Builder.getInt32(i)));
1789  ElementMapping[V] = Elements;
1790 
1791  auto InsertVectorReform = [&](Instruction *IP) {
1792  Builder.SetInsertPoint(IP);
1793  Builder.SetCurrentDebugLocation(IP->getDebugLoc());
1794  Value *ResultVec = UndefValue::get(VT);
1795  for (unsigned i = 0; i < VT->getNumElements(); i++)
1796  ResultVec = Builder.CreateInsertElement(ResultVec, Elements[i],
1797  Builder.getInt32(i));
1798  return ResultVec;
1799  };
1800 
1801  if (isa<CallInst>(StatepointInst)) {
1802  BasicBlock::iterator Next(StatepointInst);
1803  Next++;
1804  Instruction *IP = &*(Next);
1805  Replacements[V].first = InsertVectorReform(IP);
1806  Replacements[V].second = nullptr;
1807  } else {
1808  InvokeInst *Invoke = cast<InvokeInst>(StatepointInst);
1809  // We've already normalized - check that we don't have shared destination
1810  // blocks
1811  BasicBlock *NormalDest = Invoke->getNormalDest();
1812  assert(!isa<PHINode>(NormalDest->begin()));
1813  BasicBlock *UnwindDest = Invoke->getUnwindDest();
1814  assert(!isa<PHINode>(UnwindDest->begin()));
1815  // Insert insert element sequences in both successors
1816  Instruction *IP = &*(NormalDest->getFirstInsertionPt());
1817  Replacements[V].first = InsertVectorReform(IP);
1818  IP = &*(UnwindDest->getFirstInsertionPt());
1819  Replacements[V].second = InsertVectorReform(IP);
1820  }
1821  }
1822 
1823  for (Value *V : ToSplit) {
1824  AllocaInst *Alloca = AllocaMap[V];
1825 
1826  // Capture all users before we start mutating use lists
1828  for (User *U : V->users())
1829  Users.push_back(cast<Instruction>(U));
1830 
1831  for (Instruction *I : Users) {
1832  if (auto Phi = dyn_cast<PHINode>(I)) {
1833  for (unsigned i = 0; i < Phi->getNumIncomingValues(); i++)
1834  if (V == Phi->getIncomingValue(i)) {
1835  LoadInst *Load = new LoadInst(
1836  Alloca, "", Phi->getIncomingBlock(i)->getTerminator());
1837  Phi->setIncomingValue(i, Load);
1838  }
1839  } else {
1840  LoadInst *Load = new LoadInst(Alloca, "", I);
1841  I->replaceUsesOfWith(V, Load);
1842  }
1843  }
1844 
1845  // Store the original value and the replacement value into the alloca
1846  StoreInst *Store = new StoreInst(V, Alloca);
1847  if (auto I = dyn_cast<Instruction>(V))
1848  Store->insertAfter(I);
1849  else
1850  Store->insertAfter(Alloca);
1851 
1852  // Normal return for invoke, or call return
1853  Instruction *Replacement = cast<Instruction>(Replacements[V].first);
1854  (new StoreInst(Replacement, Alloca))->insertAfter(Replacement);
1855  // Unwind return for invoke only
1856  Replacement = cast_or_null<Instruction>(Replacements[V].second);
1857  if (Replacement)
1858  (new StoreInst(Replacement, Alloca))->insertAfter(Replacement);
1859  }
1860 
1861  // apply mem2reg to promote alloca to SSA
1863  for (Value *V : ToSplit)
1864  Allocas.push_back(AllocaMap[V]);
1865  PromoteMemToReg(Allocas, DT);
1866 
1867  // Update our tracking of live pointers and base mappings to account for the
1868  // changes we just made.
1869  for (Value *V : ToSplit) {
1870  auto &Elements = ElementMapping[V];
1871 
1872  LiveSet.erase(V);
1873  LiveSet.insert(Elements.begin(), Elements.end());
1874  // We need to update the base mapping as well.
1875  assert(PointerToBase.count(V));
1876  Value *OldBase = PointerToBase[V];
1877  auto &BaseElements = ElementMapping[OldBase];
1878  PointerToBase.erase(V);
1879  assert(Elements.size() == BaseElements.size());
1880  for (unsigned i = 0; i < Elements.size(); i++) {
1881  Value *Elem = Elements[i];
1882  PointerToBase[Elem] = BaseElements[i];
1883  }
1884  }
1885 }
1886 
1887 // Helper function for the "rematerializeLiveValues". It walks use chain
1888 // starting from the "CurrentValue" until it meets "BaseValue". Only "simple"
1889 // values are visited (currently it is GEP's and casts). Returns true if it
1890 // sucessfully reached "BaseValue" and false otherwise.
1891 // Fills "ChainToBase" array with all visited values. "BaseValue" is not
1892 // recorded.
1894  SmallVectorImpl<Instruction*> &ChainToBase,
1895  Value *CurrentValue, Value *BaseValue) {
1896 
1897  // We have found a base value
1898  if (CurrentValue == BaseValue) {
1899  return true;
1900  }
1901 
1902  if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(CurrentValue)) {
1903  ChainToBase.push_back(GEP);
1904  return findRematerializableChainToBasePointer(ChainToBase,
1905  GEP->getPointerOperand(),
1906  BaseValue);
1907  }
1908 
1909  if (CastInst *CI = dyn_cast<CastInst>(CurrentValue)) {
1910  Value *Def = CI->stripPointerCasts();
1911 
1912  // This two checks are basically similar. First one is here for the
1913  // consistency with findBasePointers logic.
1914  assert(!isa<CastInst>(Def) && "not a pointer cast found");
1915  if (!CI->isNoopCast(CI->getModule()->getDataLayout()))
1916  return false;
1917 
1918  ChainToBase.push_back(CI);
1919  return findRematerializableChainToBasePointer(ChainToBase, Def, BaseValue);
1920  }
1921 
1922  // Not supported instruction in the chain
1923  return false;
1924 }
1925 
1926 // Helper function for the "rematerializeLiveValues". Compute cost of the use
1927 // chain we are going to rematerialize.
1928 static unsigned
1930  TargetTransformInfo &TTI) {
1931  unsigned Cost = 0;
1932 
1933  for (Instruction *Instr : Chain) {
1934  if (CastInst *CI = dyn_cast<CastInst>(Instr)) {
1935  assert(CI->isNoopCast(CI->getModule()->getDataLayout()) &&
1936  "non noop cast is found during rematerialization");
1937 
1938  Type *SrcTy = CI->getOperand(0)->getType();
1939  Cost += TTI.getCastInstrCost(CI->getOpcode(), CI->getType(), SrcTy);
1940 
1941  } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Instr)) {
1942  // Cost of the address calculation
1943  Type *ValTy = GEP->getPointerOperandType()->getPointerElementType();
1944  Cost += TTI.getAddressComputationCost(ValTy);
1945 
1946  // And cost of the GEP itself
1947  // TODO: Use TTI->getGEPCost here (it exists, but appears to be not
1948  // allowed for the external usage)
1949  if (!GEP->hasAllConstantIndices())
1950  Cost += 2;
1951 
1952  } else {
1953  llvm_unreachable("unsupported instruciton type during rematerialization");
1954  }
1955  }
1956 
1957  return Cost;
1958 }
1959 
1960 // From the statepoint liveset pick values that are cheaper to recompute then to
1961 // relocate. Remove this values from the liveset, rematerialize them after
1962 // statepoint and record them in "Info" structure. Note that similar to
1963 // relocated values we don't do any user adjustments here.
1965  PartiallyConstructedSafepointRecord &Info,
1966  TargetTransformInfo &TTI) {
1967  const unsigned int ChainLengthThreshold = 10;
1968 
1969  // Record values we are going to delete from this statepoint live set.
1970  // We can not di this in following loop due to iterator invalidation.
1971  SmallVector<Value *, 32> LiveValuesToBeDeleted;
1972 
1973  for (Value *LiveValue: Info.liveset) {
1974  // For each live pointer find it's defining chain
1975  SmallVector<Instruction *, 3> ChainToBase;
1976  assert(Info.PointerToBase.find(LiveValue) != Info.PointerToBase.end());
1977  bool FoundChain =
1979  LiveValue,
1980  Info.PointerToBase[LiveValue]);
1981  // Nothing to do, or chain is too long
1982  if (!FoundChain ||
1983  ChainToBase.size() == 0 ||
1984  ChainToBase.size() > ChainLengthThreshold)
1985  continue;
1986 
1987  // Compute cost of this chain
1988  unsigned Cost = chainToBasePointerCost(ChainToBase, TTI);
1989  // TODO: We can also account for cases when we will be able to remove some
1990  // of the rematerialized values by later optimization passes. I.e if
1991  // we rematerialized several intersecting chains. Or if original values
1992  // don't have any uses besides this statepoint.
1993 
1994  // For invokes we need to rematerialize each chain twice - for normal and
1995  // for unwind basic blocks. Model this by multiplying cost by two.
1996  if (CS.isInvoke()) {
1997  Cost *= 2;
1998  }
1999  // If it's too expensive - skip it
2000  if (Cost >= RematerializationThreshold)
2001  continue;
2002 
2003  // Remove value from the live set
2004  LiveValuesToBeDeleted.push_back(LiveValue);
2005 
2006  // Clone instructions and record them inside "Info" structure
2007 
2008  // Walk backwards to visit top-most instructions first
2009  std::reverse(ChainToBase.begin(), ChainToBase.end());
2010 
2011  // Utility function which clones all instructions from "ChainToBase"
2012  // and inserts them before "InsertBefore". Returns rematerialized value
2013  // which should be used after statepoint.
2014  auto rematerializeChain = [&ChainToBase](Instruction *InsertBefore) {
2015  Instruction *LastClonedValue = nullptr;
2016  Instruction *LastValue = nullptr;
2017  for (Instruction *Instr: ChainToBase) {
2018  // Only GEP's and casts are suported as we need to be careful to not
2019  // introduce any new uses of pointers not in the liveset.
2020  // Note that it's fine to introduce new uses of pointers which were
2021  // otherwise not used after this statepoint.
2022  assert(isa<GetElementPtrInst>(Instr) || isa<CastInst>(Instr));
2023 
2024  Instruction *ClonedValue = Instr->clone();
2025  ClonedValue->insertBefore(InsertBefore);
2026  ClonedValue->setName(Instr->getName() + ".remat");
2027 
2028  // If it is not first instruction in the chain then it uses previously
2029  // cloned value. We should update it to use cloned value.
2030  if (LastClonedValue) {
2031  assert(LastValue);
2032  ClonedValue->replaceUsesOfWith(LastValue, LastClonedValue);
2033 #ifndef NDEBUG
2034  // Assert that cloned instruction does not use any instructions from
2035  // this chain other than LastClonedValue
2036  for (auto OpValue : ClonedValue->operand_values()) {
2037  assert(std::find(ChainToBase.begin(), ChainToBase.end(), OpValue) ==
2038  ChainToBase.end() &&
2039  "incorrect use in rematerialization chain");
2040  }
2041 #endif
2042  }
2043 
2044  LastClonedValue = ClonedValue;
2045  LastValue = Instr;
2046  }
2047  assert(LastClonedValue);
2048  return LastClonedValue;
2049  };
2050 
2051  // Different cases for calls and invokes. For invokes we need to clone
2052  // instructions both on normal and unwind path.
2053  if (CS.isCall()) {
2054  Instruction *InsertBefore = CS.getInstruction()->getNextNode();
2055  assert(InsertBefore);
2056  Instruction *RematerializedValue = rematerializeChain(InsertBefore);
2057  Info.RematerializedValues[RematerializedValue] = LiveValue;
2058  } else {
2059  InvokeInst *Invoke = cast<InvokeInst>(CS.getInstruction());
2060 
2061  Instruction *NormalInsertBefore =
2062  Invoke->getNormalDest()->getFirstInsertionPt();
2063  Instruction *UnwindInsertBefore =
2064  Invoke->getUnwindDest()->getFirstInsertionPt();
2065 
2066  Instruction *NormalRematerializedValue =
2067  rematerializeChain(NormalInsertBefore);
2068  Instruction *UnwindRematerializedValue =
2069  rematerializeChain(UnwindInsertBefore);
2070 
2071  Info.RematerializedValues[NormalRematerializedValue] = LiveValue;
2072  Info.RematerializedValues[UnwindRematerializedValue] = LiveValue;
2073  }
2074  }
2075 
2076  // Remove rematerializaed values from the live set
2077  for (auto LiveValue: LiveValuesToBeDeleted) {
2078  Info.liveset.erase(LiveValue);
2079  }
2080 }
2081 
2083  SmallVectorImpl<CallSite> &toUpdate) {
2084 #ifndef NDEBUG
2085  // sanity check the input
2086  std::set<CallSite> uniqued;
2087  uniqued.insert(toUpdate.begin(), toUpdate.end());
2088  assert(uniqued.size() == toUpdate.size() && "no duplicates please!");
2089 
2090  for (size_t i = 0; i < toUpdate.size(); i++) {
2091  CallSite &CS = toUpdate[i];
2092  assert(CS.getInstruction()->getParent()->getParent() == &F);
2093  assert(isStatepoint(CS) && "expected to already be a deopt statepoint");
2094  }
2095 #endif
2096 
2097  // When inserting gc.relocates for invokes, we need to be able to insert at
2098  // the top of the successor blocks. See the comment on
2099  // normalForInvokeSafepoint on exactly what is needed. Note that this step
2100  // may restructure the CFG.
2101  for (CallSite CS : toUpdate) {
2102  if (!CS.isInvoke())
2103  continue;
2104  InvokeInst *invoke = cast<InvokeInst>(CS.getInstruction());
2106  DT);
2108  DT);
2109  }
2110 
2111  // A list of dummy calls added to the IR to keep various values obviously
2112  // live in the IR. We'll remove all of these when done.
2114 
2115  // Insert a dummy call with all of the arguments to the vm_state we'll need
2116  // for the actual safepoint insertion. This ensures reference arguments in
2117  // the deopt argument list are considered live through the safepoint (and
2118  // thus makes sure they get relocated.)
2119  for (size_t i = 0; i < toUpdate.size(); i++) {
2120  CallSite &CS = toUpdate[i];
2121  Statepoint StatepointCS(CS);
2122 
2123  SmallVector<Value *, 64> DeoptValues;
2124  for (Use &U : StatepointCS.vm_state_args()) {
2125  Value *Arg = cast<Value>(&U);
2126  assert(!isUnhandledGCPointerType(Arg->getType()) &&
2127  "support for FCA unimplemented");
2128  if (isHandledGCPointerType(Arg->getType()))
2129  DeoptValues.push_back(Arg);
2130  }
2131  insertUseHolderAfter(CS, DeoptValues, holders);
2132  }
2133 
2135  records.reserve(toUpdate.size());
2136  for (size_t i = 0; i < toUpdate.size(); i++) {
2137  struct PartiallyConstructedSafepointRecord info;
2138  records.push_back(info);
2139  }
2140  assert(records.size() == toUpdate.size());
2141 
2142  // A) Identify all gc pointers which are staticly live at the given call
2143  // site.
2144  findLiveReferences(F, DT, P, toUpdate, records);
2145 
2146  // B) Find the base pointers for each live pointer
2147  /* scope for caching */ {
2148  // Cache the 'defining value' relation used in the computation and
2149  // insertion of base phis and selects. This ensures that we don't insert
2150  // large numbers of duplicate base_phis.
2151  DefiningValueMapTy DVCache;
2152 
2153  for (size_t i = 0; i < records.size(); i++) {
2154  struct PartiallyConstructedSafepointRecord &info = records[i];
2155  CallSite &CS = toUpdate[i];
2156  findBasePointers(DT, DVCache, CS, info);
2157  }
2158  } // end of cache scope
2159 
2160  // The base phi insertion logic (for any safepoint) may have inserted new
2161  // instructions which are now live at some safepoint. The simplest such
2162  // example is:
2163  // loop:
2164  // phi a <-- will be a new base_phi here
2165  // safepoint 1 <-- that needs to be live here
2166  // gep a + 1
2167  // safepoint 2
2168  // br loop
2169  // We insert some dummy calls after each safepoint to definitely hold live
2170  // the base pointers which were identified for that safepoint. We'll then
2171  // ask liveness for _every_ base inserted to see what is now live. Then we
2172  // remove the dummy calls.
2173  holders.reserve(holders.size() + records.size());
2174  for (size_t i = 0; i < records.size(); i++) {
2175  struct PartiallyConstructedSafepointRecord &info = records[i];
2176  CallSite &CS = toUpdate[i];
2177 
2179  for (auto Pair : info.PointerToBase) {
2180  Bases.push_back(Pair.second);
2181  }
2182  insertUseHolderAfter(CS, Bases, holders);
2183  }
2184 
2185  // By selecting base pointers, we've effectively inserted new uses. Thus, we
2186  // need to rerun liveness. We may *also* have inserted new defs, but that's
2187  // not the key issue.
2188  recomputeLiveInValues(F, DT, P, toUpdate, records);
2189 
2190  if (PrintBasePointers) {
2191  for (size_t i = 0; i < records.size(); i++) {
2192  struct PartiallyConstructedSafepointRecord &info = records[i];
2193  errs() << "Base Pairs: (w/Relocation)\n";
2194  for (auto Pair : info.PointerToBase) {
2195  errs() << " derived %" << Pair.first->getName() << " base %"
2196  << Pair.second->getName() << "\n";
2197  }
2198  }
2199  }
2200  for (size_t i = 0; i < holders.size(); i++) {
2201  holders[i]->eraseFromParent();
2202  holders[i] = nullptr;
2203  }
2204  holders.clear();
2205 
2206  // Do a limited scalarization of any live at safepoint vector values which
2207  // contain pointers. This enables this pass to run after vectorization at
2208  // the cost of some possible performance loss. TODO: it would be nice to
2209  // natively support vectors all the way through the backend so we don't need
2210  // to scalarize here.
2211  for (size_t i = 0; i < records.size(); i++) {
2212  struct PartiallyConstructedSafepointRecord &info = records[i];
2213  Instruction *statepoint = toUpdate[i].getInstruction();
2214  splitVectorValues(cast<Instruction>(statepoint), info.liveset,
2215  info.PointerToBase, DT);
2216  }
2217 
2218  // In order to reduce live set of statepoint we might choose to rematerialize
2219  // some values instead of relocating them. This is purelly an optimization and
2220  // does not influence correctness.
2221  TargetTransformInfo &TTI =
2223 
2224  for (size_t i = 0; i < records.size(); i++) {
2225  struct PartiallyConstructedSafepointRecord &info = records[i];
2226  CallSite &CS = toUpdate[i];
2227 
2228  rematerializeLiveValues(CS, info, TTI);
2229  }
2230 
2231  // Now run through and replace the existing statepoints with new ones with
2232  // the live variables listed. We do not yet update uses of the values being
2233  // relocated. We have references to live variables that need to
2234  // survive to the last iteration of this loop. (By construction, the
2235  // previous statepoint can not be a live variable, thus we can and remove
2236  // the old statepoint calls as we go.)
2237  for (size_t i = 0; i < records.size(); i++) {
2238  struct PartiallyConstructedSafepointRecord &info = records[i];
2239  CallSite &CS = toUpdate[i];
2240  makeStatepointExplicit(DT, CS, P, info);
2241  }
2242  toUpdate.clear(); // prevent accident use of invalid CallSites
2243 
2244  // Do all the fixups of the original live variables to their relocated selves
2246  for (size_t i = 0; i < records.size(); i++) {
2247  struct PartiallyConstructedSafepointRecord &info = records[i];
2248  // We can't simply save the live set from the original insertion. One of
2249  // the live values might be the result of a call which needs a safepoint.
2250  // That Value* no longer exists and we need to use the new gc_result.
2251  // Thankfully, the liveset is embedded in the statepoint (and updated), so
2252  // we just grab that.
2253  Statepoint statepoint(info.StatepointToken);
2254  live.insert(live.end(), statepoint.gc_args_begin(),
2255  statepoint.gc_args_end());
2256 #ifndef NDEBUG
2257  // Do some basic sanity checks on our liveness results before performing
2258  // relocation. Relocation can and will turn mistakes in liveness results
2259  // into non-sensical code which is must harder to debug.
2260  // TODO: It would be nice to test consistency as well
2261  assert(DT.isReachableFromEntry(info.StatepointToken->getParent()) &&
2262  "statepoint must be reachable or liveness is meaningless");
2263  for (Value *V : statepoint.gc_args()) {
2264  if (!isa<Instruction>(V))
2265  // Non-instruction values trivial dominate all possible uses
2266  continue;
2267  auto LiveInst = cast<Instruction>(V);
2268  assert(DT.isReachableFromEntry(LiveInst->getParent()) &&
2269  "unreachable values should never be live");
2270  assert(DT.dominates(LiveInst, info.StatepointToken) &&
2271  "basic SSA liveness expectation violated by liveness analysis");
2272  }
2273 #endif
2274  }
2275  unique_unsorted(live);
2276 
2277 #ifndef NDEBUG
2278  // sanity check
2279  for (auto ptr : live) {
2280  assert(isGCPointerType(ptr->getType()) && "must be a gc pointer type");
2281  }
2282 #endif
2283 
2284  relocationViaAlloca(F, DT, live, records);
2285  return !records.empty();
2286 }
2287 
2288 // Handles both return values and arguments for Functions and CallSites.
2289 template <typename AttrHolder>
2290 static void RemoveDerefAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH,
2291  unsigned Index) {
2292  AttrBuilder R;
2293  if (AH.getDereferenceableBytes(Index))
2295  AH.getDereferenceableBytes(Index)));
2296  if (AH.getDereferenceableOrNullBytes(Index))
2298  AH.getDereferenceableOrNullBytes(Index)));
2299 
2300  if (!R.empty())
2301  AH.setAttributes(AH.getAttributes().removeAttributes(
2302  Ctx, Index, AttributeSet::get(Ctx, Index, R)));
2303 }
2304 
2305 void
2306 RewriteStatepointsForGC::stripDereferenceabilityInfoFromPrototype(Function &F) {
2307  LLVMContext &Ctx = F.getContext();
2308 
2309  for (Argument &A : F.args())
2310  if (isa<PointerType>(A.getType()))
2311  RemoveDerefAttrAtIndex(Ctx, F, A.getArgNo() + 1);
2312 
2313  if (isa<PointerType>(F.getReturnType()))
2315 }
2316 
2317 void RewriteStatepointsForGC::stripDereferenceabilityInfoFromBody(Function &F) {
2318  if (F.empty())
2319  return;
2320 
2321  LLVMContext &Ctx = F.getContext();
2322  MDBuilder Builder(Ctx);
2323 
2324  for (Instruction &I : inst_range(F)) {
2325  if (const MDNode *MD = I.getMetadata(LLVMContext::MD_tbaa)) {
2326  assert(MD->getNumOperands() < 5 && "unrecognized metadata shape!");
2327  bool IsImmutableTBAA =
2328  MD->getNumOperands() == 4 &&
2329  mdconst::extract<ConstantInt>(MD->getOperand(3))->getValue() == 1;
2330 
2331  if (!IsImmutableTBAA)
2332  continue; // no work to do, MD_tbaa is already marked mutable
2333 
2334  MDNode *Base = cast<MDNode>(MD->getOperand(0));
2335  MDNode *Access = cast<MDNode>(MD->getOperand(1));
2336  uint64_t Offset =
2337  mdconst::extract<ConstantInt>(MD->getOperand(2))->getZExtValue();
2338 
2339  MDNode *MutableTBAA =
2340  Builder.createTBAAStructTagNode(Base, Access, Offset);
2341  I.setMetadata(LLVMContext::MD_tbaa, MutableTBAA);
2342  }
2343 
2344  if (CallSite CS = CallSite(&I)) {
2345  for (int i = 0, e = CS.arg_size(); i != e; i++)
2346  if (isa<PointerType>(CS.getArgument(i)->getType()))
2347  RemoveDerefAttrAtIndex(Ctx, CS, i + 1);
2348  if (isa<PointerType>(CS.getType()))
2350  }
2351  }
2352 }
2353 
2354 /// Returns true if this function should be rewritten by this pass. The main
2355 /// point of this function is as an extension point for custom logic.
2357  // TODO: This should check the GCStrategy
2358  if (F.hasGC()) {
2359  const char *FunctionGCName = F.getGC();
2360  const StringRef StatepointExampleName("statepoint-example");
2361  const StringRef CoreCLRName("coreclr");
2362  return (StatepointExampleName == FunctionGCName) ||
2363  (CoreCLRName == FunctionGCName);
2364  } else
2365  return false;
2366 }
2367 
2368 void RewriteStatepointsForGC::stripDereferenceabilityInfo(Module &M) {
2369 #ifndef NDEBUG
2370  assert(std::any_of(M.begin(), M.end(), shouldRewriteStatepointsIn) &&
2371  "precondition!");
2372 #endif
2373 
2374  for (Function &F : M)
2375  stripDereferenceabilityInfoFromPrototype(F);
2376 
2377  for (Function &F : M)
2378  stripDereferenceabilityInfoFromBody(F);
2379 }
2380 
2381 bool RewriteStatepointsForGC::runOnFunction(Function &F) {
2382  // Nothing to do for declarations.
2383  if (F.isDeclaration() || F.empty())
2384  return false;
2385 
2386  // Policy choice says not to rewrite - the most common reason is that we're
2387  // compiling code without a GCStrategy.
2389  return false;
2390 
2391  DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(F).getDomTree();
2392 
2393  // Gather all the statepoints which need rewritten. Be careful to only
2394  // consider those in reachable code since we need to ask dominance queries
2395  // when rewriting. We'll delete the unreachable ones in a moment.
2396  SmallVector<CallSite, 64> ParsePointNeeded;
2397  bool HasUnreachableStatepoint = false;
2398  for (Instruction &I : inst_range(F)) {
2399  // TODO: only the ones with the flag set!
2400  if (isStatepoint(I)) {
2401  if (DT.isReachableFromEntry(I.getParent()))
2402  ParsePointNeeded.push_back(CallSite(&I));
2403  else
2404  HasUnreachableStatepoint = true;
2405  }
2406  }
2407 
2408  bool MadeChange = false;
2409 
2410  // Delete any unreachable statepoints so that we don't have unrewritten
2411  // statepoints surviving this pass. This makes testing easier and the
2412  // resulting IR less confusing to human readers. Rather than be fancy, we
2413  // just reuse a utility function which removes the unreachable blocks.
2414  if (HasUnreachableStatepoint)
2415  MadeChange |= removeUnreachableBlocks(F);
2416 
2417  // Return early if no work to do.
2418  if (ParsePointNeeded.empty())
2419  return MadeChange;
2420 
2421  // As a prepass, go ahead and aggressively destroy single entry phi nodes.
2422  // These are created by LCSSA. They have the effect of increasing the size
2423  // of liveness sets for no good reason. It may be harder to do this post
2424  // insertion since relocations and base phis can confuse things.
2425  for (BasicBlock &BB : F)
2426  if (BB.getUniquePredecessor()) {
2427  MadeChange = true;
2429  }
2430 
2431  MadeChange |= insertParsePoints(F, DT, this, ParsePointNeeded);
2432  return MadeChange;
2433 }
2434 
2435 // liveness computation via standard dataflow
2436 // -------------------------------------------------------------------
2437 
2438 // TODO: Consider using bitvectors for liveness, the set of potentially
2439 // interesting values should be small and easy to pre-compute.
2440 
2441 /// Compute the live-in set for the location rbegin starting from
2442 /// the live-out set of the basic block
2445  DenseSet<Value *> &LiveTmp) {
2446 
2447  for (BasicBlock::reverse_iterator ritr = rbegin; ritr != rend; ritr++) {
2448  Instruction *I = &*ritr;
2449 
2450  // KILL/Def - Remove this definition from LiveIn
2451  LiveTmp.erase(I);
2452 
2453  // Don't consider *uses* in PHI nodes, we handle their contribution to
2454  // predecessor blocks when we seed the LiveOut sets
2455  if (isa<PHINode>(I))
2456  continue;
2457 
2458  // USE - Add to the LiveIn set for this instruction
2459  for (Value *V : I->operands()) {
2460  assert(!isUnhandledGCPointerType(V->getType()) &&
2461  "support for FCA unimplemented");
2462  if (isHandledGCPointerType(V->getType()) && !isa<Constant>(V)) {
2463  // The choice to exclude all things constant here is slightly subtle.
2464  // There are two idependent reasons:
2465  // - We assume that things which are constant (from LLVM's definition)
2466  // do not move at runtime. For example, the address of a global
2467  // variable is fixed, even though it's contents may not be.
2468  // - Second, we can't disallow arbitrary inttoptr constants even
2469  // if the language frontend does. Optimization passes are free to
2470  // locally exploit facts without respect to global reachability. This
2471  // can create sections of code which are dynamically unreachable and
2472  // contain just about anything. (see constants.ll in tests)
2473  LiveTmp.insert(V);
2474  }
2475  }
2476  }
2477 }
2478 
2480 
2481  for (BasicBlock *Succ : successors(BB)) {
2482  const BasicBlock::iterator E(Succ->getFirstNonPHI());
2483  for (BasicBlock::iterator I = Succ->begin(); I != E; I++) {
2484  PHINode *Phi = cast<PHINode>(&*I);
2485  Value *V = Phi->getIncomingValueForBlock(BB);
2486  assert(!isUnhandledGCPointerType(V->getType()) &&
2487  "support for FCA unimplemented");
2488  if (isHandledGCPointerType(V->getType()) && !isa<Constant>(V)) {
2489  LiveTmp.insert(V);
2490  }
2491  }
2492  }
2493 }
2494 
2496  DenseSet<Value *> KillSet;
2497  for (Instruction &I : *BB)
2498  if (isHandledGCPointerType(I.getType()))
2499  KillSet.insert(&I);
2500  return KillSet;
2501 }
2502 
2503 #ifndef NDEBUG
2504 /// Check that the items in 'Live' dominate 'TI'. This is used as a basic
2505 /// sanity check for the liveness computation.
2507  TerminatorInst *TI, bool TermOkay = false) {
2508  for (Value *V : Live) {
2509  if (auto *I = dyn_cast<Instruction>(V)) {
2510  // The terminator can be a member of the LiveOut set. LLVM's definition
2511  // of instruction dominance states that V does not dominate itself. As
2512  // such, we need to special case this to allow it.
2513  if (TermOkay && TI == I)
2514  continue;
2515  assert(DT.dominates(I, TI) &&
2516  "basic SSA liveness expectation violated by liveness analysis");
2517  }
2518  }
2519 }
2520 
2521 /// Check that all the liveness sets used during the computation of liveness
2522 /// obey basic SSA properties. This is useful for finding cases where we miss
2523 /// a def.
2524 static void checkBasicSSA(DominatorTree &DT, GCPtrLivenessData &Data,
2525  BasicBlock &BB) {
2526  checkBasicSSA(DT, Data.LiveSet[&BB], BB.getTerminator());
2527  checkBasicSSA(DT, Data.LiveOut[&BB], BB.getTerminator(), true);
2528  checkBasicSSA(DT, Data.LiveIn[&BB], BB.getTerminator());
2529 }
2530 #endif
2531 
2533  GCPtrLivenessData &Data) {
2534 
2536  auto AddPredsToWorklist = [&](BasicBlock *BB) {
2537  // We use a SetVector so that we don't have duplicates in the worklist.
2538  Worklist.insert(pred_begin(BB), pred_end(BB));
2539  };
2540  auto NextItem = [&]() {
2541  BasicBlock *BB = Worklist.back();
2542  Worklist.pop_back();
2543  return BB;
2544  };
2545 
2546  // Seed the liveness for each individual block
2547  for (BasicBlock &BB : F) {
2548  Data.KillSet[&BB] = computeKillSet(&BB);
2549  Data.LiveSet[&BB].clear();
2550  computeLiveInValues(BB.rbegin(), BB.rend(), Data.LiveSet[&BB]);
2551 
2552 #ifndef NDEBUG
2553  for (Value *Kill : Data.KillSet[&BB])
2554  assert(!Data.LiveSet[&BB].count(Kill) && "live set contains kill");
2555 #endif
2556 
2557  Data.LiveOut[&BB] = DenseSet<Value *>();
2558  computeLiveOutSeed(&BB, Data.LiveOut[&BB]);
2559  Data.LiveIn[&BB] = Data.LiveSet[&BB];
2560  set_union(Data.LiveIn[&BB], Data.LiveOut[&BB]);
2561  set_subtract(Data.LiveIn[&BB], Data.KillSet[&BB]);
2562  if (!Data.LiveIn[&BB].empty())
2563  AddPredsToWorklist(&BB);
2564  }
2565 
2566  // Propagate that liveness until stable
2567  while (!Worklist.empty()) {
2568  BasicBlock *BB = NextItem();
2569 
2570  // Compute our new liveout set, then exit early if it hasn't changed
2571  // despite the contribution of our successor.
2572  DenseSet<Value *> LiveOut = Data.LiveOut[BB];
2573  const auto OldLiveOutSize = LiveOut.size();
2574  for (BasicBlock *Succ : successors(BB)) {
2575  assert(Data.LiveIn.count(Succ));
2576  set_union(LiveOut, Data.LiveIn[Succ]);
2577  }
2578  // assert OutLiveOut is a subset of LiveOut
2579  if (OldLiveOutSize == LiveOut.size()) {
2580  // If the sets are the same size, then we didn't actually add anything
2581  // when unioning our successors LiveIn Thus, the LiveIn of this block
2582  // hasn't changed.
2583  continue;
2584  }
2585  Data.LiveOut[BB] = LiveOut;
2586 
2587  // Apply the effects of this basic block
2588  DenseSet<Value *> LiveTmp = LiveOut;
2589  set_union(LiveTmp, Data.LiveSet[BB]);
2590  set_subtract(LiveTmp, Data.KillSet[BB]);
2591 
2592  assert(Data.LiveIn.count(BB));
2593  const DenseSet<Value *> &OldLiveIn = Data.LiveIn[BB];
2594  // assert: OldLiveIn is a subset of LiveTmp
2595  if (OldLiveIn.size() != LiveTmp.size()) {
2596  Data.LiveIn[BB] = LiveTmp;
2597  AddPredsToWorklist(BB);
2598  }
2599  } // while( !worklist.empty() )
2600 
2601 #ifndef NDEBUG
2602  // Sanity check our ouput against SSA properties. This helps catch any
2603  // missing kills during the above iteration.
2604  for (BasicBlock &BB : F) {
2605  checkBasicSSA(DT, Data, BB);
2606  }
2607 #endif
2608 }
2609 
2610 static void findLiveSetAtInst(Instruction *Inst, GCPtrLivenessData &Data,
2611  StatepointLiveSetTy &Out) {
2612 
2613  BasicBlock *BB = Inst->getParent();
2614 
2615  // Note: The copy is intentional and required
2616  assert(Data.LiveOut.count(BB));
2617  DenseSet<Value *> LiveOut = Data.LiveOut[BB];
2618 
2619  // We want to handle the statepoint itself oddly. It's
2620  // call result is not live (normal), nor are it's arguments
2621  // (unless they're used again later). This adjustment is
2622  // specifically what we need to relocate
2624  computeLiveInValues(BB->rbegin(), rend, LiveOut);
2625  LiveOut.erase(Inst);
2626  Out.insert(LiveOut.begin(), LiveOut.end());
2627 }
2628 
2629 static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
2630  const CallSite &CS,
2631  PartiallyConstructedSafepointRecord &Info) {
2632  Instruction *Inst = CS.getInstruction();
2633  StatepointLiveSetTy Updated;
2634  findLiveSetAtInst(Inst, RevisedLivenessData, Updated);
2635 
2636 #ifndef NDEBUG
2637  DenseSet<Value *> Bases;
2638  for (auto KVPair : Info.PointerToBase) {
2639  Bases.insert(KVPair.second);
2640  }
2641 #endif
2642  // We may have base pointers which are now live that weren't before. We need
2643  // to update the PointerToBase structure to reflect this.
2644  for (auto V : Updated)
2645  if (!Info.PointerToBase.count(V)) {
2646  assert(Bases.count(V) && "can't find base for unexpected live value");
2647  Info.PointerToBase[V] = V;
2648  continue;
2649  }
2650 
2651 #ifndef NDEBUG
2652  for (auto V : Updated) {
2653  assert(Info.PointerToBase.count(V) &&
2654  "must be able to find base for live value");
2655  }
2656 #endif
2657 
2658  // Remove any stale base mappings - this can happen since our liveness is
2659  // more precise then the one inherent in the base pointer analysis
2660  DenseSet<Value *> ToErase;
2661  for (auto KVPair : Info.PointerToBase)
2662  if (!Updated.count(KVPair.first))
2663  ToErase.insert(KVPair.first);
2664  for (auto V : ToErase)
2665  Info.PointerToBase.erase(V);
2666 
2667 #ifndef NDEBUG
2668  for (auto KVPair : Info.PointerToBase)
2669  assert(Updated.count(KVPair.first) && "record for non-live value");
2670 #endif
2671 
2672  Info.liveset = Updated;
2673 }
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:82
static std::pair< Value *, bool > findBaseDefiningValueOfVector(Value *I, Value *Index=nullptr)
Return a base defining value for the 'Index' element of the given vector instruction 'I'...
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
Definition: Instruction.cpp:70
static void unique_unsorted(SmallVectorImpl< T > &Vec)
Implement a unique function which doesn't require we sort the input vector.
static bool isHandledGCPointerType(Type *T)
BasicBlock * getUniquePredecessor()
Return the predecessor of this block if it has a unique predecessor block.
Definition: BasicBlock.cpp:224
static cl::opt< bool, true > ClobberNonLiveOverride("rs4gc-clobber-non-live", cl::location(ClobberNonLive), cl::Hidden)
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:223
LLVM Argument representation.
Definition: Argument.h:35
static Value * findBaseDefiningValueCached(Value *I, DefiningValueMapTy &Cache)
Returns the base defining value for this value.
bool hasName() const
Definition: Value.h:228
Instruction * StatepointToken
The new gc.statepoint instruction itself.
static void Found()
static void RemoveDerefAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH, unsigned Index)
static int find_index(ArrayRef< Value * > livevec, Value *val)
Instruction * UnwindToken
Instruction to which exceptional gc relocates are attached Makes it easier to iterate through them du...
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:114
static bool findRematerializableChainToBasePointer(SmallVectorImpl< Instruction * > &ChainToBase, Value *CurrentValue, Value *BaseValue)
InstrTy * getInstruction() const
Definition: CallSite.h:82
Intrinsic::ID getIntrinsicID() const
getIntrinsicID - Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:44
bool hasNUses(unsigned N) const
Return true if this Value has exactly N users.
Definition: Value.cpp:96
size_type size() const
Definition: DenseSet.h:53
static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, const CallSite &CS, PartiallyConstructedSafepointRecord &result)
Given an updated version of the dataflow liveness results, update the liveset and base pointer maps f...
static DenseSet< Value * > computeKillSet(BasicBlock *BB)
iterator end() const
Definition: ArrayRef.h:123
static void rematerializeLiveValues(CallSite CS, PartiallyConstructedSafepointRecord &Info, TargetTransformInfo &TTI)
const char * getGC() const
Definition: Function.cpp:384
CallInst - This class represents a function call, abstracting a target machine's calling convention...
static void insertRematerializationStores(RematerializedValueMapTy RematerializedValues, DenseMap< Value *, Value * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)
iterator begin(unsigned Slot) const
Type * getReturnType() const
Definition: Function.cpp:233
static void stablize_order(SmallVectorImpl< Value * > &basevec, SmallVectorImpl< Value * > &livevec)
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:111
Metadata node.
Definition: Metadata.h:740
static Value * findBaseDefiningValue(Value *I)
Helper function for findBasePointer - Will return a value which either a) defines the base pointer fo...
F(f)
LoadInst - an instruction for reading from memory.
Definition: Instructions.h:177
reverse_iterator rbegin()
Definition: BasicBlock.h:236
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
Hexagon Common GEP
iv Induction Variable Users
Definition: IVUsers.cpp:43
void reserve(size_type N)
Definition: SmallVector.h:401
bool hasAttribute(AttrKind Val) const
Return true if the attribute is present.
Definition: Attributes.cpp:147
static void computeLiveInValues(DominatorTree &DT, Function &F, GCPtrLivenessData &Data)
Compute the live-in set for every basic block in the function.
static void makeStatepointExplicitImpl(const CallSite &CS, const SmallVectorImpl< llvm::Value * > &basePtrs, const SmallVectorImpl< llvm::Value * > &liveVariables, Pass *P, PartiallyConstructedSafepointRecord &result)
AttributeSet getRetAttributes() const
The attributes for the ret value are returned.
Definition: Attributes.cpp:938
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:188
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:231
int compare(StringRef RHS) const
compare - Compare two strings; the result is -1, 0, or 1 if this string is lexicographically less tha...
Definition: StringRef.h:148
lazy value info
IterTy arg_end() const
Definition: CallSite.h:157
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:1541
AnalysisUsage & addRequired()
rewrite statepoints for Make relocations at false
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:70
static unsigned chainToBasePointerCost(SmallVectorImpl< Instruction * > &Chain, TargetTransformInfo &TTI)
static void makeStatepointExplicit(DominatorTree &DT, const CallSite &CS, Pass *P, PartiallyConstructedSafepointRecord &result)
SelectInst - This class represents the LLVM 'select' instruction.
StatepointLiveSetTy liveset
The set of values known to be live accross this safepoint.
bool erase(const ValueT &V)
Definition: DenseSet.h:69
static CallInst * Create(Value *Func, ArrayRef< Value * > Args, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:389
NodeTy * getNextNode()
Get the next node, or 0 for the list tail.
Definition: ilist_node.h:80
UndefValue - 'undef' values are things that do not have specified contents.
Definition: Constants.h:1220
static bool isKnownBaseResult(Value *V)
Given the result of a call to findBaseDefiningValue, or findBaseOrBDV, is it known to be a base point...
StructType - Class to represent struct types.
Definition: DerivedTypes.h:191
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:98
A Use represents the edge between a Value definition and its users.
Definition: Use.h:69
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:75
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:165
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:517
static void CreateGCRelocates(ArrayRef< llvm::Value * > LiveVariables, const int LiveStart, ArrayRef< llvm::Value * > BasePtrs, Instruction *StatepointToken, IRBuilder<> Builder)
Helper function to place all gc relocates necessary for the given statepoint.
static Value * findBasePointer(Value *I, DefiningValueMapTy &cache)
For a given value or instruction, figure out what base ptr it's derived from.
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:250
unsigned getAddressComputationCost(Type *Ty, bool IsComplex=false) const
Instruction * clone() const
clone() - Create a copy of 'this' instruction that is identical in all ways except the following: ...
void pop_back()
Remove the last element of the SetVector.
Definition: SetVector.h:167
Function does not access memory.
Definition: Attributes.h:99
bool isStatepoint(const ImmutableCallSite &CS)
Definition: Statepoint.cpp:22
static cl::opt< bool > PrintLiveSet("spp-print-liveset", cl::Hidden, cl::init(false))
void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AliasSetTracker *AST=nullptr, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
#define states
Definition: regexec.c:106
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:102
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
bool LLVM_ATTRIBUTE_UNUSED_RESULT empty() const
Definition: SmallVector.h:57
static void insertUseHolderAfter(CallSite &CS, const ArrayRef< Value * > Values, SmallVectorImpl< CallInst * > &Holders)
Insert holders so that each Value is obviously live through the entire lifetime of the call...
ValTy * getCalledValue() const
getCalledValue - Return the pointer to function that is being called.
Definition: CallSite.h:91
DenseMap< BasicBlock *, DenseSet< Value * > > LiveIn
Values live into this basic block (i.e.
ArrayType - Class to represent array types.
Definition: DerivedTypes.h:336
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Definition: Dominators.cpp:265
This class represents a no-op cast from one type to another.
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:59
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
FunctionType::get - This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: ArrayRef.h:31
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:866
StoreInst - an instruction for storing to memory.
Definition: Instructions.h:316
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Definition: IRBuilder.h:109
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:351
unsigned getNumElements() const
Return the number of elements in the Vector type.
Definition: DerivedTypes.h:432
static void splitVectorValues(Instruction *StatepointInst, StatepointLiveSetTy &LiveSet, DenseMap< Value *, Value * > &PointerToBase, DominatorTree &DT)
Remove any vector of pointers from the liveset by scalarizing them over the statepoint instruction...
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:256
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:318
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:67
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:134
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Definition: IRBuilder.h:85
BasicBlock * getNormalDest() const
PointerType - Class to represent pointers.
Definition: DerivedTypes.h:449
unsigned getNumIncomingValues() const
getNumIncomingValues - Return the number of incoming edges
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition: User.cpp:24
void FoldSingleEntryPHINodes(BasicBlock *BB, AliasAnalysis *AA=nullptr, MemoryDependenceAnalysis *MemDep=nullptr)
FoldSingleEntryPHINodes - We know that BB has one predecessor.
InstListType::reverse_iterator reverse_iterator
Definition: BasicBlock.h:95
CallSiteTy::arg_iterator gc_args_begin() const
Definition: Statepoint.h:231
GetElementPtrInst - an instruction for type-safe pointer arithmetic to access elements of arrays and ...
Definition: Instructions.h:830
#define P(N)
iterator_range< arg_iterator > vm_state_args() const
range adapter for vm state arguments
Definition: Statepoint.h:227
unsigned getNumSlots() const
Return the number of slots used in this attribute list.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
InsertElementInst - This instruction inserts a single (scalar) element into a VectorType value...
static void checkBasicSSA(DominatorTree &DT, DenseSet< Value * > &Live, TerminatorInst *TI, bool TermOkay=false)
Check that the items in 'Live' dominate 'TI'.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:35
Wrapper pass for TargetTransformInfo.
static ConstantPointerNull * get(PointerType *T)
get() - Static factory methods - Return objects of the specified value
Definition: Constants.cpp:1455
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:221
Constant * getOrInsertFunction(StringRef Name, FunctionType *T, AttributeSet AttributeList)
Look up the specified function in the module symbol table.
Definition: Module.cpp:115
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
Definition: Instruction.cpp:76
LLVM Basic Block Representation.
Definition: BasicBlock.h:65
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:41
FunTy * getCalledFunction() const
getCalledFunction - Return the function being called if this is a direct call, otherwise return null ...
Definition: CallSite.h:99
size_type count(const ValueT &V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:65
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
Definition: Type.h:226
This is an important base class in LLVM.
Definition: Constant.h:41
CallSiteTy::arg_iterator gc_args_end() const
Definition: Statepoint.h:234
const Value * getCondition() const
static cl::opt< unsigned > RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden, cl::init(6))
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:32
LandingPadInst * getLandingPadInst()
Return the landingpad instruction associated with the landing pad.
Definition: BasicBlock.cpp:418
iterator_range< arg_iterator > gc_args() const
range adapter for gc arguments
Definition: Statepoint.h:239
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:114
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:230
DenseMap< BasicBlock *, DenseSet< Value * > > LiveSet
Values used in this block (and thus live); does not included values killed within this block...
Represent the analysis usage information of a pass.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:225
BasicBlock * getIncomingBlock(unsigned i) const
getIncomingBlock - Return incoming basic block number i.
static void insertRelocationStores(iterator_range< Value::user_iterator > GCRelocs, DenseMap< Value *, Value * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)
static bool insertParsePoints(Function &F, DominatorTree &DT, Pass *P, SmallVectorImpl< CallSite > &toUpdate)
DenseMap< Instruction *, Value * > RematerializedValueMapTy
RematerializedValueMapTy RematerializedValues
Record live values we are rematerialized instead of relocating.
DenseMap< llvm::Value *, llvm::Value * > PointerToBase
Mapping from live pointers to a base-defining-value.
iterator begin() const
Definition: ArrayRef.h:122
static void analyzeParsePointLiveness(DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, const CallSite &CS, PartiallyConstructedSafepointRecord &result)
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
A specialization of it's base class for read-write access to a gc.statepoint.
Definition: Statepoint.h:296
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:117
op_range operands()
Definition: User.h:191
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:69
Wraps a call to a gc.relocate and provides access to it's operands.
Definition: Statepoint.h:308
rewrite statepoints for gc
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:129
static bool containsGCPtrType(Type *Ty)
Returns true if this type contains a gc pointer whether we know how to handle that type or not...
unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) const
static bool order_by_name(llvm::Value *a, llvm::Value *b)
void setTailCall(bool isTC=true)
bool isPointerTy() const
isPointerTy - True if this is an instance of PointerType.
Definition: Type.h:217
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1473
iterator erase(iterator I)
Definition: SmallVector.h:455
static void findLiveSetAtInst(Instruction *inst, GCPtrLivenessData &Data, StatepointLiveSetTy &out)
Given results from the dataflow liveness computation, find the set of live Values at a particular ins...
CallInst * CreateCall(Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="")
Definition: IRBuilder.h:1467
INITIALIZE_PASS_BEGIN(RewriteStatepointsForGC,"rewrite-statepoints-for-gc","Make relocations explicit at statepoints", false, false) INITIALIZE_PASS_END(RewriteStatepointsForGC
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:283
static Value * findBaseOrBDV(Value *I, DefiningValueMapTy &Cache)
Return a base pointer for this value if known.
void setMetadata(unsigned KindID, MDNode *Node)
setMetadata - Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1083
static bool shouldRewriteStatepointsIn(Function &F)
Returns true if this function should be rewritten by this pass.
static void findLiveReferences(Function &F, DominatorTree &DT, Pass *P, ArrayRef< CallSite > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records)
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - Get or set the calling convention of this function call...
void dump() const
Support for debugging, callable in GDB: V->dump()
Definition: AsmWriter.cpp:3353
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
Definition: Dominators.cpp:214
LLVMContext & getContext() const
Retrieve the LLVM context.
Definition: Attributes.cpp:926
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
static cl::opt< bool > PrintBasePointers("spp-print-base-pointers", cl::Hidden, cl::init(false))
static InvokeInst * Create(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
rewrite statepoints for Make relocations at statepoints
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:217
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:147
BasicBlock * getUnwindDest() const
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Value * getIncomingValue(unsigned i) const
getIncomingValue - Return incoming value number x
iterator end()
Definition: BasicBlock.h:233
static AttributeSet legalizeCallAttributes(AttributeSet AS)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
DenseMap< Value *, Value * > DefiningValueMapTy
void initializeRewriteStatepointsForGCPass(PassRegistry &)
iterator end(unsigned Slot) const
bool empty() const
Return true if the builder contains no target-independent attributes.
Definition: Attributes.h:540
static void findBasePointers(const StatepointLiveSetTy &live, DenseMap< llvm::Value *, llvm::Value * > &PointerToBase, DominatorTree *DT, DefiningValueMapTy &DVCache)
Value * stripPointerCasts()
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:458
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:582
static BasicBlock * normalizeForInvokeSafepoint(BasicBlock *BB, BasicBlock *InvokeParent, DominatorTree &DT)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
const BasicBlock & getEntryBlock() const
Definition: Function.h:442
void setOperand(unsigned i, Value *Val)
Definition: User.h:122
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:119
DenseMap< BasicBlock *, DenseSet< Value * > > LiveOut
Values live out of this basic block (i.e.
static bool ClobberNonLive
A range adaptor for a pair of iterators.
VectorType - Class to represent vector types.
Definition: DerivedTypes.h:362
Value * getIncomingValueForBlock(const BasicBlock *BB) const
iterator_range< user_iterator > users()
Definition: Value.h:300
Function only reads from memory.
Definition: Attributes.h:100
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.cpp:379
bool empty() const
Definition: Function.h:463
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:481
const AttributeSet & getAttributes() const
getAttributes - Return the parameter attributes for this call.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1030
void set_subtract(S1Ty &S1, const S2Ty &S2)
set_subtract(A, B) - Compute A := A - B
Definition: SetOperations.h:63
bool operator!=(uint64_t V1, const APInt &V2)
Definition: APInt.h:1736
const AttributeSet & getAttributes() const
getAttributes - Return the parameter attributes for this invoke.
unsigned size() const
Definition: DenseMap.h:82
iterator end()
Definition: Module.h:571
static cl::opt< bool > PrintLiveSetSize("spp-print-liveset-size", cl::Hidden, cl::init(false))
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:239
iterator_range< inst_iterator > inst_range(Function *F)
Definition: InstIterator.h:129
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:128
unsigned getSlotIndex(unsigned Slot) const
Return the index for the given slot.
reverse_iterator rbegin(StringRef path)
Get reverse begin iterator over path.
Definition: Path.cpp:308
static void relocationViaAlloca(Function &F, DominatorTree &DT, ArrayRef< Value * > Live, ArrayRef< struct PartiallyConstructedSafepointRecord > Records)
do all the relocation update via allocas and mem2reg
static bool isUnhandledGCPointerType(Type *Ty)
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction...
Definition: Instruction.cpp:82
#define I(x, y, z)
Definition: MD5.cpp:54
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:311
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:236
iterator begin()
Definition: Module.h:569
bool isTailCall() const
iterator_range< value_op_iterator > operand_values()
Definition: User.h:215
const T & back() const
Return the last element of the SetVector.
Definition: SetVector.h:89
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:34
reverse_iterator rend(StringRef path)
Get reverse end iterator over path.
Definition: Path.cpp:315
static bool isGCPointerType(const Type *T)
user_iterator user_begin()
Definition: Value.h:294
Pointer is known to be dereferenceable.
Definition: Attributes.h:92
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:365
LLVM Value Representation.
Definition: Value.h:69
succ_range successors(BasicBlock *BB)
Definition: IR/CFG.h:271
ModulePass * createRewriteStatepointsForGCPass()
static void computeLiveOutSeed(BasicBlock *BB, DenseSet< Value * > &LiveTmp)
static const Function * getParent(const Value *V)
static cl::opt< bool > TraceLSP("trace-rewrite-statepoints", cl::Hidden, cl::init(false))
void setCallingConv(CallingConv::ID CC)
InvokeInst - Invoke instruction.
IterTy arg_begin() const
arg_begin/arg_end - Return iterators corresponding to the actual argument list for a call site...
Definition: CallSite.h:151
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:40
BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, AliasAnalysis *AA=nullptr, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, bool PreserveLCSSA=false)
SplitBlockPredecessors - This method introduces at least one new basic block into the function and mo...
bool removeUnreachableBlocks(Function &F)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:1254
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - Get or set the calling convention of this function call...
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:203
This pass exposes codegen information to IR-level passes.
bool operator==(uint64_t V1, const APInt &V2)
Definition: APInt.h:1734
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:194
std::error_code status(const Twine &path, file_status &result)
Get file status as if by POSIX stat().
bool isGCResult(const Value *V)
Definition: Statepoint.cpp:67
void setIncomingValue(unsigned i, Value *V)
bool isInvoke() const
isInvoke - true if a InvokeInst is enclosed.
Definition: CallSite.h:80
AttributeSet addAttributes(LLVMContext &C, unsigned Index, AttributeSet Attrs) const
Add attributes to the attribute set at the given index.
Definition: Attributes.cpp:773
bool isCall() const
isCall - true if a CallInst is enclosed.
Definition: CallSite.h:76
bool set_union(S1Ty &S1, const S2Ty &S2)
set_union(A, B) - Compute A := A u B, return whether A changed.
Definition: SetOperations.h:23
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:340
const BasicBlock * getParent() const
Definition: Instruction.h:72
Pointer is either null or dereferenceable.
Definition: Attributes.h:93
iterator_range< arg_iterator > args()
Definition: Function.h:489
IntrinsicInst - A useful wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:37
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:265
DenseMap< BasicBlock *, DenseSet< Value * > > KillSet
Values defined in this block.
AllocaInst - an instruction to allocate memory on the stack.
Definition: Instructions.h:76
DenseSet< llvm::Value * > StatepointLiveSetTy
AttributeSet getFnAttributes() const
The function attributes are returned.
Definition: Attributes.cpp:947
user_iterator user_end()
Definition: Value.h:296