LLVM  4.0.0
RewriteStatepointsForGC.cpp
Go to the documentation of this file.
1 //===- RewriteStatepointsForGC.cpp - Make GC relocations explicit ---------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Rewrite an existing set of gc.statepoints such that they make potential
11 // relocations performed by the garbage collector explicit in the IR.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "llvm/Pass.h"
16 #include "llvm/Analysis/CFG.h"
18 #include "llvm/ADT/SetOperations.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/ADT/DenseSet.h"
21 #include "llvm/ADT/SetVector.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/ADT/MapVector.h"
24 #include "llvm/IR/BasicBlock.h"
25 #include "llvm/IR/CallSite.h"
26 #include "llvm/IR/Dominators.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/IR/IRBuilder.h"
29 #include "llvm/IR/InstIterator.h"
30 #include "llvm/IR/Instructions.h"
31 #include "llvm/IR/Intrinsics.h"
32 #include "llvm/IR/IntrinsicInst.h"
33 #include "llvm/IR/Module.h"
34 #include "llvm/IR/MDBuilder.h"
35 #include "llvm/IR/Statepoint.h"
36 #include "llvm/IR/Value.h"
37 #include "llvm/IR/Verifier.h"
38 #include "llvm/Support/Debug.h"
40 #include "llvm/Transforms/Scalar.h"
45 
46 #define DEBUG_TYPE "rewrite-statepoints-for-gc"
47 
48 using namespace llvm;
49 
50 // Print the liveset found at the insert location
51 static cl::opt<bool> PrintLiveSet("spp-print-liveset", cl::Hidden,
52  cl::init(false));
53 static cl::opt<bool> PrintLiveSetSize("spp-print-liveset-size", cl::Hidden,
54  cl::init(false));
55 // Print out the base pointers for debugging
56 static cl::opt<bool> PrintBasePointers("spp-print-base-pointers", cl::Hidden,
57  cl::init(false));
58 
59 // Cost threshold measuring when it is profitable to rematerialize value instead
60 // of relocating it
61 static cl::opt<unsigned>
62 RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden,
63  cl::init(6));
64 
65 #ifdef EXPENSIVE_CHECKS
66 static bool ClobberNonLive = true;
67 #else
68 static bool ClobberNonLive = false;
69 #endif
70 static cl::opt<bool, true> ClobberNonLiveOverride("rs4gc-clobber-non-live",
72  cl::Hidden);
73 
74 static cl::opt<bool>
75  AllowStatepointWithNoDeoptInfo("rs4gc-allow-statepoint-with-no-deopt-info",
76  cl::Hidden, cl::init(true));
77 
78 namespace {
79 struct RewriteStatepointsForGC : public ModulePass {
80  static char ID; // Pass identification, replacement for typeid
81 
82  RewriteStatepointsForGC() : ModulePass(ID) {
84  }
85  bool runOnFunction(Function &F);
86  bool runOnModule(Module &M) override {
87  bool Changed = false;
88  for (Function &F : M)
89  Changed |= runOnFunction(F);
90 
91  if (Changed) {
92  // stripNonValidAttributes asserts that shouldRewriteStatepointsIn
93  // returns true for at least one function in the module. Since at least
94  // one function changed, we know that the precondition is satisfied.
95  stripNonValidAttributes(M);
96  }
97 
98  return Changed;
99  }
100 
101  void getAnalysisUsage(AnalysisUsage &AU) const override {
102  // We add and rewrite a bunch of instructions, but don't really do much
103  // else. We could in theory preserve a lot more analyses here.
106  }
107 
108  /// The IR fed into RewriteStatepointsForGC may have had attributes implying
109  /// dereferenceability that are no longer valid/correct after
110  /// RewriteStatepointsForGC has run. This is because semantically, after
111  /// RewriteStatepointsForGC runs, all calls to gc.statepoint "free" the entire
112  /// heap. stripNonValidAttributes (conservatively) restores correctness
113  /// by erasing all attributes in the module that externally imply
114  /// dereferenceability.
115  /// Similar reasoning also applies to the noalias attributes. gc.statepoint
116  /// can touch the entire heap including noalias objects.
117  void stripNonValidAttributes(Module &M);
118 
119  // Helpers for stripNonValidAttributes
120  void stripNonValidAttributesFromBody(Function &F);
121  void stripNonValidAttributesFromPrototype(Function &F);
122 };
123 } // namespace
124 
126 
128  return new RewriteStatepointsForGC();
129 }
130 
131 INITIALIZE_PASS_BEGIN(RewriteStatepointsForGC, "rewrite-statepoints-for-gc",
132  "Make relocations explicit at statepoints", false, false)
135 INITIALIZE_PASS_END(RewriteStatepointsForGC, "rewrite-statepoints-for-gc",
136  "Make relocations explicit at statepoints", false, false)
137 
138 namespace {
140  /// Values defined in this block.
142  /// Values used in this block (and thus live); does not included values
143  /// killed within this block.
145 
146  /// Values live into this basic block (i.e. used by any
147  /// instruction in this basic block or ones reachable from here)
149 
150  /// Values live out of this basic block (i.e. live into
151  /// any successor block)
153 };
154 
155 // The type of the internal cache used inside the findBasePointers family
156 // of functions. From the callers perspective, this is an opaque type and
157 // should not be inspected.
158 //
159 // In the actual implementation this caches two relations:
160 // - The base relation itself (i.e. this pointer is based on that one)
161 // - The base defining value relation (i.e. before base_phi insertion)
162 // Generally, after the execution of a full findBasePointer call, only the
163 // base relation will remain. Internally, we add a mixture of the two
164 // types, then update all the second type to the first type
169 
171  /// The set of values known to be live across this safepoint
172  StatepointLiveSetTy LiveSet;
173 
174  /// Mapping from live pointers to a base-defining-value
176 
177  /// The *new* gc.statepoint instruction itself. This produces the token
178  /// that normal path gc.relocates and the gc.result are tied to.
180 
181  /// Instruction to which exceptional gc relocates are attached
182  /// Makes it easier to iterate through them during relocationViaAlloca.
184 
185  /// Record live values we are rematerialized instead of relocating.
186  /// They are not included into 'LiveSet' field.
187  /// Maps rematerialized copy to it's original value.
189 };
190 }
191 
193  Optional<OperandBundleUse> DeoptBundle =
195 
196  if (!DeoptBundle.hasValue()) {
198  "Found non-leaf call without deopt info!");
199  return None;
200  }
201 
202  return DeoptBundle.getValue().Inputs;
203 }
204 
205 /// Compute the live-in set for every basic block in the function
206 static void computeLiveInValues(DominatorTree &DT, Function &F,
207  GCPtrLivenessData &Data);
208 
209 /// Given results from the dataflow liveness computation, find the set of live
210 /// Values at a particular instruction.
211 static void findLiveSetAtInst(Instruction *inst, GCPtrLivenessData &Data,
212  StatepointLiveSetTy &out);
213 
214 // TODO: Once we can get to the GCStrategy, this becomes
215 // Optional<bool> isGCManagedPointer(const Type *Ty) const override {
216 
217 static bool isGCPointerType(Type *T) {
218  if (auto *PT = dyn_cast<PointerType>(T))
219  // For the sake of this example GC, we arbitrarily pick addrspace(1) as our
220  // GC managed heap. We know that a pointer into this heap needs to be
221  // updated and that no other pointer does.
222  return PT->getAddressSpace() == 1;
223  return false;
224 }
225 
226 // Return true if this type is one which a) is a gc pointer or contains a GC
227 // pointer and b) is of a type this code expects to encounter as a live value.
228 // (The insertion code will assert that a type which matches (a) and not (b)
229 // is not encountered.)
231  // We fully support gc pointers
232  if (isGCPointerType(T))
233  return true;
234  // We partially support vectors of gc pointers. The code will assert if it
235  // can't handle something.
236  if (auto VT = dyn_cast<VectorType>(T))
237  if (isGCPointerType(VT->getElementType()))
238  return true;
239  return false;
240 }
241 
242 #ifndef NDEBUG
243 /// Returns true if this type contains a gc pointer whether we know how to
244 /// handle that type or not.
245 static bool containsGCPtrType(Type *Ty) {
246  if (isGCPointerType(Ty))
247  return true;
248  if (VectorType *VT = dyn_cast<VectorType>(Ty))
249  return isGCPointerType(VT->getScalarType());
250  if (ArrayType *AT = dyn_cast<ArrayType>(Ty))
251  return containsGCPtrType(AT->getElementType());
252  if (StructType *ST = dyn_cast<StructType>(Ty))
253  return any_of(ST->subtypes(), containsGCPtrType);
254  return false;
255 }
256 
257 // Returns true if this is a type which a) is a gc pointer or contains a GC
258 // pointer and b) is of a type which the code doesn't expect (i.e. first class
259 // aggregates). Used to trip assertions.
260 static bool isUnhandledGCPointerType(Type *Ty) {
261  return containsGCPtrType(Ty) && !isHandledGCPointerType(Ty);
262 }
263 #endif
264 
265 // Return the name of the value suffixed with the provided value, or if the
266 // value didn't have a name, the default value specified.
267 static std::string suffixed_name_or(Value *V, StringRef Suffix,
268  StringRef DefaultName) {
269  return V->hasName() ? (V->getName() + Suffix).str() : DefaultName.str();
270 }
271 
272 // Conservatively identifies any definitions which might be live at the
273 // given instruction. The analysis is performed immediately before the
274 // given instruction. Values defined by that instruction are not considered
275 // live. Values used by that instruction are considered live.
276 static void
278  GCPtrLivenessData &OriginalLivenessData, CallSite CS,
279  PartiallyConstructedSafepointRecord &Result) {
280  Instruction *Inst = CS.getInstruction();
281 
282  StatepointLiveSetTy LiveSet;
283  findLiveSetAtInst(Inst, OriginalLivenessData, LiveSet);
284 
285  if (PrintLiveSet) {
286  dbgs() << "Live Variables:\n";
287  for (Value *V : LiveSet)
288  dbgs() << " " << V->getName() << " " << *V << "\n";
289  }
290  if (PrintLiveSetSize) {
291  dbgs() << "Safepoint For: " << CS.getCalledValue()->getName() << "\n";
292  dbgs() << "Number live values: " << LiveSet.size() << "\n";
293  }
294  Result.LiveSet = LiveSet;
295 }
296 
297 static bool isKnownBaseResult(Value *V);
298 namespace {
299 /// A single base defining value - An immediate base defining value for an
300 /// instruction 'Def' is an input to 'Def' whose base is also a base of 'Def'.
301 /// For instructions which have multiple pointer [vector] inputs or that
302 /// transition between vector and scalar types, there is no immediate base
303 /// defining value. The 'base defining value' for 'Def' is the transitive
304 /// closure of this relation stopping at the first instruction which has no
305 /// immediate base defining value. The b.d.v. might itself be a base pointer,
306 /// but it can also be an arbitrary derived pointer.
307 struct BaseDefiningValueResult {
308  /// Contains the value which is the base defining value.
309  Value * const BDV;
310  /// True if the base defining value is also known to be an actual base
311  /// pointer.
312  const bool IsKnownBase;
313  BaseDefiningValueResult(Value *BDV, bool IsKnownBase)
314  : BDV(BDV), IsKnownBase(IsKnownBase) {
315 #ifndef NDEBUG
316  // Check consistency between new and old means of checking whether a BDV is
317  // a base.
318  bool MustBeBase = isKnownBaseResult(BDV);
319  assert(!MustBeBase || MustBeBase == IsKnownBase);
320 #endif
321  }
322 };
323 }
324 
325 static BaseDefiningValueResult findBaseDefiningValue(Value *I);
326 
327 /// Return a base defining value for the 'Index' element of the given vector
328 /// instruction 'I'. If Index is null, returns a BDV for the entire vector
329 /// 'I'. As an optimization, this method will try to determine when the
330 /// element is known to already be a base pointer. If this can be established,
331 /// the second value in the returned pair will be true. Note that either a
332 /// vector or a pointer typed value can be returned. For the former, the
333 /// vector returned is a BDV (and possibly a base) of the entire vector 'I'.
334 /// If the later, the return pointer is a BDV (or possibly a base) for the
335 /// particular element in 'I'.
336 static BaseDefiningValueResult
338  // Each case parallels findBaseDefiningValue below, see that code for
339  // detailed motivation.
340 
341  if (isa<Argument>(I))
342  // An incoming argument to the function is a base pointer
343  return BaseDefiningValueResult(I, true);
344 
345  if (isa<Constant>(I))
346  // Base of constant vector consists only of constant null pointers.
347  // For reasoning see similar case inside 'findBaseDefiningValue' function.
348  return BaseDefiningValueResult(ConstantAggregateZero::get(I->getType()),
349  true);
350 
351  if (isa<LoadInst>(I))
352  return BaseDefiningValueResult(I, true);
353 
354  if (isa<InsertElementInst>(I))
355  // We don't know whether this vector contains entirely base pointers or
356  // not. To be conservatively correct, we treat it as a BDV and will
357  // duplicate code as needed to construct a parallel vector of bases.
358  return BaseDefiningValueResult(I, false);
359 
360  if (isa<ShuffleVectorInst>(I))
361  // We don't know whether this vector contains entirely base pointers or
362  // not. To be conservatively correct, we treat it as a BDV and will
363  // duplicate code as needed to construct a parallel vector of bases.
364  // TODO: There a number of local optimizations which could be applied here
365  // for particular sufflevector patterns.
366  return BaseDefiningValueResult(I, false);
367 
368  // A PHI or Select is a base defining value. The outer findBasePointer
369  // algorithm is responsible for constructing a base value for this BDV.
370  assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
371  "unknown vector instruction - no base found for vector element");
372  return BaseDefiningValueResult(I, false);
373 }
374 
375 /// Helper function for findBasePointer - Will return a value which either a)
376 /// defines the base pointer for the input, b) blocks the simple search
377 /// (i.e. a PHI or Select of two derived pointers), or c) involves a change
378 /// from pointer to vector type or back.
379 static BaseDefiningValueResult findBaseDefiningValue(Value *I) {
381  "Illegal to ask for the base pointer of a non-pointer type");
382 
383  if (I->getType()->isVectorTy())
385 
386  if (isa<Argument>(I))
387  // An incoming argument to the function is a base pointer
388  // We should have never reached here if this argument isn't an gc value
389  return BaseDefiningValueResult(I, true);
390 
391  if (isa<Constant>(I)) {
392  // We assume that objects with a constant base (e.g. a global) can't move
393  // and don't need to be reported to the collector because they are always
394  // live. Besides global references, all kinds of constants (e.g. undef,
395  // constant expressions, null pointers) can be introduced by the inliner or
396  // the optimizer, especially on dynamically dead paths.
397  // Here we treat all of them as having single null base. By doing this we
398  // trying to avoid problems reporting various conflicts in a form of
399  // "phi (const1, const2)" or "phi (const, regular gc ptr)".
400  // See constant.ll file for relevant test cases.
401 
402  return BaseDefiningValueResult(
403  ConstantPointerNull::get(cast<PointerType>(I->getType())), true);
404  }
405 
406  if (CastInst *CI = dyn_cast<CastInst>(I)) {
407  Value *Def = CI->stripPointerCasts();
408  // If stripping pointer casts changes the address space there is an
409  // addrspacecast in between.
410  assert(cast<PointerType>(Def->getType())->getAddressSpace() ==
411  cast<PointerType>(CI->getType())->getAddressSpace() &&
412  "unsupported addrspacecast");
413  // If we find a cast instruction here, it means we've found a cast which is
414  // not simply a pointer cast (i.e. an inttoptr). We don't know how to
415  // handle int->ptr conversion.
416  assert(!isa<CastInst>(Def) && "shouldn't find another cast here");
417  return findBaseDefiningValue(Def);
418  }
419 
420  if (isa<LoadInst>(I))
421  // The value loaded is an gc base itself
422  return BaseDefiningValueResult(I, true);
423 
424 
425  if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I))
426  // The base of this GEP is the base
427  return findBaseDefiningValue(GEP->getPointerOperand());
428 
429  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
430  switch (II->getIntrinsicID()) {
431  default:
432  // fall through to general call handling
433  break;
434  case Intrinsic::experimental_gc_statepoint:
435  llvm_unreachable("statepoints don't produce pointers");
436  case Intrinsic::experimental_gc_relocate: {
437  // Rerunning safepoint insertion after safepoints are already
438  // inserted is not supported. It could probably be made to work,
439  // but why are you doing this? There's no good reason.
440  llvm_unreachable("repeat safepoint insertion is not supported");
441  }
442  case Intrinsic::gcroot:
443  // Currently, this mechanism hasn't been extended to work with gcroot.
444  // There's no reason it couldn't be, but I haven't thought about the
445  // implications much.
447  "interaction with the gcroot mechanism is not supported");
448  }
449  }
450  // We assume that functions in the source language only return base
451  // pointers. This should probably be generalized via attributes to support
452  // both source language and internal functions.
453  if (isa<CallInst>(I) || isa<InvokeInst>(I))
454  return BaseDefiningValueResult(I, true);
455 
456  // TODO: I have absolutely no idea how to implement this part yet. It's not
457  // necessarily hard, I just haven't really looked at it yet.
458  assert(!isa<LandingPadInst>(I) && "Landing Pad is unimplemented");
459 
460  if (isa<AtomicCmpXchgInst>(I))
461  // A CAS is effectively a atomic store and load combined under a
462  // predicate. From the perspective of base pointers, we just treat it
463  // like a load.
464  return BaseDefiningValueResult(I, true);
465 
466  assert(!isa<AtomicRMWInst>(I) && "Xchg handled above, all others are "
467  "binary ops which don't apply to pointers");
468 
469  // The aggregate ops. Aggregates can either be in the heap or on the
470  // stack, but in either case, this is simply a field load. As a result,
471  // this is a defining definition of the base just like a load is.
472  if (isa<ExtractValueInst>(I))
473  return BaseDefiningValueResult(I, true);
474 
475  // We should never see an insert vector since that would require we be
476  // tracing back a struct value not a pointer value.
477  assert(!isa<InsertValueInst>(I) &&
478  "Base pointer for a struct is meaningless");
479 
480  // An extractelement produces a base result exactly when it's input does.
481  // We may need to insert a parallel instruction to extract the appropriate
482  // element out of the base vector corresponding to the input. Given this,
483  // it's analogous to the phi and select case even though it's not a merge.
484  if (isa<ExtractElementInst>(I))
485  // Note: There a lot of obvious peephole cases here. This are deliberately
486  // handled after the main base pointer inference algorithm to make writing
487  // test cases to exercise that code easier.
488  return BaseDefiningValueResult(I, false);
489 
490  // The last two cases here don't return a base pointer. Instead, they
491  // return a value which dynamically selects from among several base
492  // derived pointers (each with it's own base potentially). It's the job of
493  // the caller to resolve these.
494  assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
495  "missing instruction case in findBaseDefiningValing");
496  return BaseDefiningValueResult(I, false);
497 }
498 
499 /// Returns the base defining value for this value.
501  Value *&Cached = Cache[I];
502  if (!Cached) {
503  Cached = findBaseDefiningValue(I).BDV;
504  DEBUG(dbgs() << "fBDV-cached: " << I->getName() << " -> "
505  << Cached->getName() << "\n");
506  }
507  assert(Cache[I] != nullptr);
508  return Cached;
509 }
510 
511 /// Return a base pointer for this value if known. Otherwise, return it's
512 /// base defining value.
515  auto Found = Cache.find(Def);
516  if (Found != Cache.end()) {
517  // Either a base-of relation, or a self reference. Caller must check.
518  return Found->second;
519  }
520  // Only a BDV available
521  return Def;
522 }
523 
524 /// Given the result of a call to findBaseDefiningValue, or findBaseOrBDV,
525 /// is it known to be a base pointer? Or do we need to continue searching.
526 static bool isKnownBaseResult(Value *V) {
527  if (!isa<PHINode>(V) && !isa<SelectInst>(V) &&
528  !isa<ExtractElementInst>(V) && !isa<InsertElementInst>(V) &&
529  !isa<ShuffleVectorInst>(V)) {
530  // no recursion possible
531  return true;
532  }
533  if (isa<Instruction>(V) &&
534  cast<Instruction>(V)->getMetadata("is_base_value")) {
535  // This is a previously inserted base phi or select. We know
536  // that this is a base value.
537  return true;
538  }
539 
540  // We need to keep searching
541  return false;
542 }
543 
544 namespace {
545 /// Models the state of a single base defining value in the findBasePointer
546 /// algorithm for determining where a new instruction is needed to propagate
547 /// the base of this BDV.
548 class BDVState {
549 public:
550  enum Status { Unknown, Base, Conflict };
551 
552  BDVState() : Status(Unknown), BaseValue(nullptr) {}
553 
554  explicit BDVState(Status Status, Value *BaseValue = nullptr)
555  : Status(Status), BaseValue(BaseValue) {
556  assert(Status != Base || BaseValue);
557  }
558 
559  explicit BDVState(Value *BaseValue) : Status(Base), BaseValue(BaseValue) {}
560 
561  Status getStatus() const { return Status; }
562  Value *getBaseValue() const { return BaseValue; }
563 
564  bool isBase() const { return getStatus() == Base; }
565  bool isUnknown() const { return getStatus() == Unknown; }
566  bool isConflict() const { return getStatus() == Conflict; }
567 
568  bool operator==(const BDVState &Other) const {
569  return BaseValue == Other.BaseValue && Status == Other.Status;
570  }
571 
572  bool operator!=(const BDVState &other) const { return !(*this == other); }
573 
575  void dump() const {
576  print(dbgs());
577  dbgs() << '\n';
578  }
579 
580  void print(raw_ostream &OS) const {
581  switch (getStatus()) {
582  case Unknown:
583  OS << "U";
584  break;
585  case Base:
586  OS << "B";
587  break;
588  case Conflict:
589  OS << "C";
590  break;
591  };
592  OS << " (" << getBaseValue() << " - "
593  << (getBaseValue() ? getBaseValue()->getName() : "nullptr") << "): ";
594  }
595 
596 private:
597  Status Status;
598  AssertingVH<Value> BaseValue; // Non-null only if Status == Base.
599 };
600 }
601 
602 #ifndef NDEBUG
603 static raw_ostream &operator<<(raw_ostream &OS, const BDVState &State) {
604  State.print(OS);
605  return OS;
606 }
607 #endif
608 
609 static BDVState meetBDVStateImpl(const BDVState &LHS, const BDVState &RHS) {
610  switch (LHS.getStatus()) {
611  case BDVState::Unknown:
612  return RHS;
613 
614  case BDVState::Base:
615  assert(LHS.getBaseValue() && "can't be null");
616  if (RHS.isUnknown())
617  return LHS;
618 
619  if (RHS.isBase()) {
620  if (LHS.getBaseValue() == RHS.getBaseValue()) {
621  assert(LHS == RHS && "equality broken!");
622  return LHS;
623  }
624  return BDVState(BDVState::Conflict);
625  }
626  assert(RHS.isConflict() && "only three states!");
627  return BDVState(BDVState::Conflict);
628 
629  case BDVState::Conflict:
630  return LHS;
631  }
632  llvm_unreachable("only three states!");
633 }
634 
635 // Values of type BDVState form a lattice, and this function implements the meet
636 // operation.
637 static BDVState meetBDVState(BDVState LHS, BDVState RHS) {
638  BDVState Result = meetBDVStateImpl(LHS, RHS);
639  assert(Result == meetBDVStateImpl(RHS, LHS) &&
640  "Math is wrong: meet does not commute!");
641  return Result;
642 }
643 
644 /// For a given value or instruction, figure out what base ptr its derived from.
645 /// For gc objects, this is simply itself. On success, returns a value which is
646 /// the base pointer. (This is reliable and can be used for relocation.) On
647 /// failure, returns nullptr.
649  Value *Def = findBaseOrBDV(I, Cache);
650 
651  if (isKnownBaseResult(Def))
652  return Def;
653 
654  // Here's the rough algorithm:
655  // - For every SSA value, construct a mapping to either an actual base
656  // pointer or a PHI which obscures the base pointer.
657  // - Construct a mapping from PHI to unknown TOP state. Use an
658  // optimistic algorithm to propagate base pointer information. Lattice
659  // looks like:
660  // UNKNOWN
661  // b1 b2 b3 b4
662  // CONFLICT
663  // When algorithm terminates, all PHIs will either have a single concrete
664  // base or be in a conflict state.
665  // - For every conflict, insert a dummy PHI node without arguments. Add
666  // these to the base[Instruction] = BasePtr mapping. For every
667  // non-conflict, add the actual base.
668  // - For every conflict, add arguments for the base[a] of each input
669  // arguments.
670  //
671  // Note: A simpler form of this would be to add the conflict form of all
672  // PHIs without running the optimistic algorithm. This would be
673  // analogous to pessimistic data flow and would likely lead to an
674  // overall worse solution.
675 
676 #ifndef NDEBUG
677  auto isExpectedBDVType = [](Value *BDV) {
678  return isa<PHINode>(BDV) || isa<SelectInst>(BDV) ||
679  isa<ExtractElementInst>(BDV) || isa<InsertElementInst>(BDV) ||
680  isa<ShuffleVectorInst>(BDV);
681  };
682 #endif
683 
684  // Once populated, will contain a mapping from each potentially non-base BDV
685  // to a lattice value (described above) which corresponds to that BDV.
686  // We use the order of insertion (DFS over the def/use graph) to provide a
687  // stable deterministic ordering for visiting DenseMaps (which are unordered)
688  // below. This is important for deterministic compilation.
690 
691  // Recursively fill in all base defining values reachable from the initial
692  // one for which we don't already know a definite base value for
693  /* scope */ {
694  SmallVector<Value*, 16> Worklist;
695  Worklist.push_back(Def);
696  States.insert({Def, BDVState()});
697  while (!Worklist.empty()) {
698  Value *Current = Worklist.pop_back_val();
699  assert(!isKnownBaseResult(Current) && "why did it get added?");
700 
701  auto visitIncomingValue = [&](Value *InVal) {
702  Value *Base = findBaseOrBDV(InVal, Cache);
703  if (isKnownBaseResult(Base))
704  // Known bases won't need new instructions introduced and can be
705  // ignored safely
706  return;
707  assert(isExpectedBDVType(Base) && "the only non-base values "
708  "we see should be base defining values");
709  if (States.insert(std::make_pair(Base, BDVState())).second)
710  Worklist.push_back(Base);
711  };
712  if (PHINode *PN = dyn_cast<PHINode>(Current)) {
713  for (Value *InVal : PN->incoming_values())
714  visitIncomingValue(InVal);
715  } else if (SelectInst *SI = dyn_cast<SelectInst>(Current)) {
716  visitIncomingValue(SI->getTrueValue());
717  visitIncomingValue(SI->getFalseValue());
718  } else if (auto *EE = dyn_cast<ExtractElementInst>(Current)) {
719  visitIncomingValue(EE->getVectorOperand());
720  } else if (auto *IE = dyn_cast<InsertElementInst>(Current)) {
721  visitIncomingValue(IE->getOperand(0)); // vector operand
722  visitIncomingValue(IE->getOperand(1)); // scalar operand
723  } else if (auto *SV = dyn_cast<ShuffleVectorInst>(Current)) {
724  visitIncomingValue(SV->getOperand(0));
725  visitIncomingValue(SV->getOperand(1));
726  }
727  else {
728  llvm_unreachable("Unimplemented instruction case");
729  }
730  }
731  }
732 
733 #ifndef NDEBUG
734  DEBUG(dbgs() << "States after initialization:\n");
735  for (auto Pair : States) {
736  DEBUG(dbgs() << " " << Pair.second << " for " << *Pair.first << "\n");
737  }
738 #endif
739 
740  // Return a phi state for a base defining value. We'll generate a new
741  // base state for known bases and expect to find a cached state otherwise.
742  auto getStateForBDV = [&](Value *baseValue) {
743  if (isKnownBaseResult(baseValue))
744  return BDVState(baseValue);
745  auto I = States.find(baseValue);
746  assert(I != States.end() && "lookup failed!");
747  return I->second;
748  };
749 
750  bool Progress = true;
751  while (Progress) {
752 #ifndef NDEBUG
753  const size_t OldSize = States.size();
754 #endif
755  Progress = false;
756  // We're only changing values in this loop, thus safe to keep iterators.
757  // Since this is computing a fixed point, the order of visit does not
758  // effect the result. TODO: We could use a worklist here and make this run
759  // much faster.
760  for (auto Pair : States) {
761  Value *BDV = Pair.first;
762  assert(!isKnownBaseResult(BDV) && "why did it get added?");
763 
764  // Given an input value for the current instruction, return a BDVState
765  // instance which represents the BDV of that value.
766  auto getStateForInput = [&](Value *V) mutable {
767  Value *BDV = findBaseOrBDV(V, Cache);
768  return getStateForBDV(BDV);
769  };
770 
771  BDVState NewState;
772  if (SelectInst *SI = dyn_cast<SelectInst>(BDV)) {
773  NewState = meetBDVState(NewState, getStateForInput(SI->getTrueValue()));
774  NewState =
775  meetBDVState(NewState, getStateForInput(SI->getFalseValue()));
776  } else if (PHINode *PN = dyn_cast<PHINode>(BDV)) {
777  for (Value *Val : PN->incoming_values())
778  NewState = meetBDVState(NewState, getStateForInput(Val));
779  } else if (auto *EE = dyn_cast<ExtractElementInst>(BDV)) {
780  // The 'meet' for an extractelement is slightly trivial, but it's still
781  // useful in that it drives us to conflict if our input is.
782  NewState =
783  meetBDVState(NewState, getStateForInput(EE->getVectorOperand()));
784  } else if (auto *IE = dyn_cast<InsertElementInst>(BDV)){
785  // Given there's a inherent type mismatch between the operands, will
786  // *always* produce Conflict.
787  NewState = meetBDVState(NewState, getStateForInput(IE->getOperand(0)));
788  NewState = meetBDVState(NewState, getStateForInput(IE->getOperand(1)));
789  } else {
790  // The only instance this does not return a Conflict is when both the
791  // vector operands are the same vector.
792  auto *SV = cast<ShuffleVectorInst>(BDV);
793  NewState = meetBDVState(NewState, getStateForInput(SV->getOperand(0)));
794  NewState = meetBDVState(NewState, getStateForInput(SV->getOperand(1)));
795  }
796 
797  BDVState OldState = States[BDV];
798  if (OldState != NewState) {
799  Progress = true;
800  States[BDV] = NewState;
801  }
802  }
803 
804  assert(OldSize == States.size() &&
805  "fixed point shouldn't be adding any new nodes to state");
806  }
807 
808 #ifndef NDEBUG
809  DEBUG(dbgs() << "States after meet iteration:\n");
810  for (auto Pair : States) {
811  DEBUG(dbgs() << " " << Pair.second << " for " << *Pair.first << "\n");
812  }
813 #endif
814 
815  // Insert Phis for all conflicts
816  // TODO: adjust naming patterns to avoid this order of iteration dependency
817  for (auto Pair : States) {
818  Instruction *I = cast<Instruction>(Pair.first);
819  BDVState State = Pair.second;
820  assert(!isKnownBaseResult(I) && "why did it get added?");
821  assert(!State.isUnknown() && "Optimistic algorithm didn't complete!");
822 
823  // extractelement instructions are a bit special in that we may need to
824  // insert an extract even when we know an exact base for the instruction.
825  // The problem is that we need to convert from a vector base to a scalar
826  // base for the particular indice we're interested in.
827  if (State.isBase() && isa<ExtractElementInst>(I) &&
828  isa<VectorType>(State.getBaseValue()->getType())) {
829  auto *EE = cast<ExtractElementInst>(I);
830  // TODO: In many cases, the new instruction is just EE itself. We should
831  // exploit this, but can't do it here since it would break the invariant
832  // about the BDV not being known to be a base.
833  auto *BaseInst = ExtractElementInst::Create(
834  State.getBaseValue(), EE->getIndexOperand(), "base_ee", EE);
835  BaseInst->setMetadata("is_base_value", MDNode::get(I->getContext(), {}));
836  States[I] = BDVState(BDVState::Base, BaseInst);
837  }
838 
839  // Since we're joining a vector and scalar base, they can never be the
840  // same. As a result, we should always see insert element having reached
841  // the conflict state.
842  assert(!isa<InsertElementInst>(I) || State.isConflict());
843 
844  if (!State.isConflict())
845  continue;
846 
847  /// Create and insert a new instruction which will represent the base of
848  /// the given instruction 'I'.
849  auto MakeBaseInstPlaceholder = [](Instruction *I) -> Instruction* {
850  if (isa<PHINode>(I)) {
851  BasicBlock *BB = I->getParent();
852  int NumPreds = std::distance(pred_begin(BB), pred_end(BB));
853  assert(NumPreds > 0 && "how did we reach here");
854  std::string Name = suffixed_name_or(I, ".base", "base_phi");
855  return PHINode::Create(I->getType(), NumPreds, Name, I);
856  } else if (SelectInst *SI = dyn_cast<SelectInst>(I)) {
857  // The undef will be replaced later
858  UndefValue *Undef = UndefValue::get(SI->getType());
859  std::string Name = suffixed_name_or(I, ".base", "base_select");
860  return SelectInst::Create(SI->getCondition(), Undef, Undef, Name, SI);
861  } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {
862  UndefValue *Undef = UndefValue::get(EE->getVectorOperand()->getType());
863  std::string Name = suffixed_name_or(I, ".base", "base_ee");
864  return ExtractElementInst::Create(Undef, EE->getIndexOperand(), Name,
865  EE);
866  } else if (auto *IE = dyn_cast<InsertElementInst>(I)) {
867  UndefValue *VecUndef = UndefValue::get(IE->getOperand(0)->getType());
868  UndefValue *ScalarUndef = UndefValue::get(IE->getOperand(1)->getType());
869  std::string Name = suffixed_name_or(I, ".base", "base_ie");
870  return InsertElementInst::Create(VecUndef, ScalarUndef,
871  IE->getOperand(2), Name, IE);
872  } else {
873  auto *SV = cast<ShuffleVectorInst>(I);
874  UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType());
875  std::string Name = suffixed_name_or(I, ".base", "base_sv");
876  return new ShuffleVectorInst(VecUndef, VecUndef, SV->getOperand(2),
877  Name, SV);
878  }
879  };
880  Instruction *BaseInst = MakeBaseInstPlaceholder(I);
881  // Add metadata marking this as a base value
882  BaseInst->setMetadata("is_base_value", MDNode::get(I->getContext(), {}));
883  States[I] = BDVState(BDVState::Conflict, BaseInst);
884  }
885 
886  // Returns a instruction which produces the base pointer for a given
887  // instruction. The instruction is assumed to be an input to one of the BDVs
888  // seen in the inference algorithm above. As such, we must either already
889  // know it's base defining value is a base, or have inserted a new
890  // instruction to propagate the base of it's BDV and have entered that newly
891  // introduced instruction into the state table. In either case, we are
892  // assured to be able to determine an instruction which produces it's base
893  // pointer.
894  auto getBaseForInput = [&](Value *Input, Instruction *InsertPt) {
895  Value *BDV = findBaseOrBDV(Input, Cache);
896  Value *Base = nullptr;
897  if (isKnownBaseResult(BDV)) {
898  Base = BDV;
899  } else {
900  // Either conflict or base.
901  assert(States.count(BDV));
902  Base = States[BDV].getBaseValue();
903  }
904  assert(Base && "Can't be null");
905  // The cast is needed since base traversal may strip away bitcasts
906  if (Base->getType() != Input->getType() && InsertPt)
907  Base = new BitCastInst(Base, Input->getType(), "cast", InsertPt);
908  return Base;
909  };
910 
911  // Fixup all the inputs of the new PHIs. Visit order needs to be
912  // deterministic and predictable because we're naming newly created
913  // instructions.
914  for (auto Pair : States) {
915  Instruction *BDV = cast<Instruction>(Pair.first);
916  BDVState State = Pair.second;
917 
918  assert(!isKnownBaseResult(BDV) && "why did it get added?");
919  assert(!State.isUnknown() && "Optimistic algorithm didn't complete!");
920  if (!State.isConflict())
921  continue;
922 
923  if (PHINode *BasePHI = dyn_cast<PHINode>(State.getBaseValue())) {
924  PHINode *PN = cast<PHINode>(BDV);
925  unsigned NumPHIValues = PN->getNumIncomingValues();
926  for (unsigned i = 0; i < NumPHIValues; i++) {
927  Value *InVal = PN->getIncomingValue(i);
928  BasicBlock *InBB = PN->getIncomingBlock(i);
929 
930  // If we've already seen InBB, add the same incoming value
931  // we added for it earlier. The IR verifier requires phi
932  // nodes with multiple entries from the same basic block
933  // to have the same incoming value for each of those
934  // entries. If we don't do this check here and basephi
935  // has a different type than base, we'll end up adding two
936  // bitcasts (and hence two distinct values) as incoming
937  // values for the same basic block.
938 
939  int BlockIndex = BasePHI->getBasicBlockIndex(InBB);
940  if (BlockIndex != -1) {
941  Value *OldBase = BasePHI->getIncomingValue(BlockIndex);
942  BasePHI->addIncoming(OldBase, InBB);
943 
944 #ifndef NDEBUG
945  Value *Base = getBaseForInput(InVal, nullptr);
946  // In essence this assert states: the only way two values
947  // incoming from the same basic block may be different is by
948  // being different bitcasts of the same value. A cleanup
949  // that remains TODO is changing findBaseOrBDV to return an
950  // llvm::Value of the correct type (and still remain pure).
951  // This will remove the need to add bitcasts.
952  assert(Base->stripPointerCasts() == OldBase->stripPointerCasts() &&
953  "Sanity -- findBaseOrBDV should be pure!");
954 #endif
955  continue;
956  }
957 
958  // Find the instruction which produces the base for each input. We may
959  // need to insert a bitcast in the incoming block.
960  // TODO: Need to split critical edges if insertion is needed
961  Value *Base = getBaseForInput(InVal, InBB->getTerminator());
962  BasePHI->addIncoming(Base, InBB);
963  }
964  assert(BasePHI->getNumIncomingValues() == NumPHIValues);
965  } else if (SelectInst *BaseSI =
966  dyn_cast<SelectInst>(State.getBaseValue())) {
967  SelectInst *SI = cast<SelectInst>(BDV);
968 
969  // Find the instruction which produces the base for each input.
970  // We may need to insert a bitcast.
971  BaseSI->setTrueValue(getBaseForInput(SI->getTrueValue(), BaseSI));
972  BaseSI->setFalseValue(getBaseForInput(SI->getFalseValue(), BaseSI));
973  } else if (auto *BaseEE =
974  dyn_cast<ExtractElementInst>(State.getBaseValue())) {
975  Value *InVal = cast<ExtractElementInst>(BDV)->getVectorOperand();
976  // Find the instruction which produces the base for each input. We may
977  // need to insert a bitcast.
978  BaseEE->setOperand(0, getBaseForInput(InVal, BaseEE));
979  } else if (auto *BaseIE = dyn_cast<InsertElementInst>(State.getBaseValue())){
980  auto *BdvIE = cast<InsertElementInst>(BDV);
981  auto UpdateOperand = [&](int OperandIdx) {
982  Value *InVal = BdvIE->getOperand(OperandIdx);
983  Value *Base = getBaseForInput(InVal, BaseIE);
984  BaseIE->setOperand(OperandIdx, Base);
985  };
986  UpdateOperand(0); // vector operand
987  UpdateOperand(1); // scalar operand
988  } else {
989  auto *BaseSV = cast<ShuffleVectorInst>(State.getBaseValue());
990  auto *BdvSV = cast<ShuffleVectorInst>(BDV);
991  auto UpdateOperand = [&](int OperandIdx) {
992  Value *InVal = BdvSV->getOperand(OperandIdx);
993  Value *Base = getBaseForInput(InVal, BaseSV);
994  BaseSV->setOperand(OperandIdx, Base);
995  };
996  UpdateOperand(0); // vector operand
997  UpdateOperand(1); // vector operand
998  }
999  }
1000 
1001  // Cache all of our results so we can cheaply reuse them
1002  // NOTE: This is actually two caches: one of the base defining value
1003  // relation and one of the base pointer relation! FIXME
1004  for (auto Pair : States) {
1005  auto *BDV = Pair.first;
1006  Value *Base = Pair.second.getBaseValue();
1007  assert(BDV && Base);
1008  assert(!isKnownBaseResult(BDV) && "why did it get added?");
1009 
1010  DEBUG(dbgs() << "Updating base value cache"
1011  << " for: " << BDV->getName() << " from: "
1012  << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none")
1013  << " to: " << Base->getName() << "\n");
1014 
1015  if (Cache.count(BDV)) {
1016  assert(isKnownBaseResult(Base) &&
1017  "must be something we 'know' is a base pointer");
1018  // Once we transition from the BDV relation being store in the Cache to
1019  // the base relation being stored, it must be stable
1020  assert((!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) &&
1021  "base relation should be stable");
1022  }
1023  Cache[BDV] = Base;
1024  }
1025  assert(Cache.count(Def));
1026  return Cache[Def];
1027 }
1028 
1029 // For a set of live pointers (base and/or derived), identify the base
1030 // pointer of the object which they are derived from. This routine will
1031 // mutate the IR graph as needed to make the 'base' pointer live at the
1032 // definition site of 'derived'. This ensures that any use of 'derived' can
1033 // also use 'base'. This may involve the insertion of a number of
1034 // additional PHI nodes.
1035 //
1036 // preconditions: live is a set of pointer type Values
1037 //
1038 // side effects: may insert PHI nodes into the existing CFG, will preserve
1039 // CFG, will not remove or mutate any existing nodes
1040 //
1041 // post condition: PointerToBase contains one (derived, base) pair for every
1042 // pointer in live. Note that derived can be equal to base if the original
1043 // pointer was a base pointer.
1044 static void
1046  MapVector<Value *, Value *> &PointerToBase,
1047  DominatorTree *DT, DefiningValueMapTy &DVCache) {
1048  for (Value *ptr : live) {
1049  Value *base = findBasePointer(ptr, DVCache);
1050  assert(base && "failed to find base pointer");
1051  PointerToBase[ptr] = base;
1052  assert((!isa<Instruction>(base) || !isa<Instruction>(ptr) ||
1053  DT->dominates(cast<Instruction>(base)->getParent(),
1054  cast<Instruction>(ptr)->getParent())) &&
1055  "The base we found better dominate the derived pointer");
1056  }
1057 }
1058 
1059 /// Find the required based pointers (and adjust the live set) for the given
1060 /// parse point.
1062  CallSite CS,
1063  PartiallyConstructedSafepointRecord &result) {
1064  MapVector<Value *, Value *> PointerToBase;
1065  findBasePointers(result.LiveSet, PointerToBase, &DT, DVCache);
1066 
1067  if (PrintBasePointers) {
1068  errs() << "Base Pairs (w/o Relocation):\n";
1069  for (auto &Pair : PointerToBase) {
1070  errs() << " derived ";
1071  Pair.first->printAsOperand(errs(), false);
1072  errs() << " base ";
1073  Pair.second->printAsOperand(errs(), false);
1074  errs() << "\n";;
1075  }
1076  }
1077 
1078  result.PointerToBase = PointerToBase;
1079 }
1080 
1081 /// Given an updated version of the dataflow liveness results, update the
1082 /// liveset and base pointer maps for the call site CS.
1083 static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
1084  CallSite CS,
1085  PartiallyConstructedSafepointRecord &result);
1086 
1088  Function &F, DominatorTree &DT, ArrayRef<CallSite> toUpdate,
1090  // TODO-PERF: reuse the original liveness, then simply run the dataflow
1091  // again. The old values are still live and will help it stabilize quickly.
1092  GCPtrLivenessData RevisedLivenessData;
1093  computeLiveInValues(DT, F, RevisedLivenessData);
1094  for (size_t i = 0; i < records.size(); i++) {
1095  struct PartiallyConstructedSafepointRecord &info = records[i];
1096  recomputeLiveInValues(RevisedLivenessData, toUpdate[i], info);
1097  }
1098 }
1099 
1100 // When inserting gc.relocate and gc.result calls, we need to ensure there are
1101 // no uses of the original value / return value between the gc.statepoint and
1102 // the gc.relocate / gc.result call. One case which can arise is a phi node
1103 // starting one of the successor blocks. We also need to be able to insert the
1104 // gc.relocates only on the path which goes through the statepoint. We might
1105 // need to split an edge to make this possible.
1106 static BasicBlock *
1108  DominatorTree &DT) {
1109  BasicBlock *Ret = BB;
1110  if (!BB->getUniquePredecessor())
1111  Ret = SplitBlockPredecessors(BB, InvokeParent, "", &DT);
1112 
1113  // Now that 'Ret' has unique predecessor we can safely remove all phi nodes
1114  // from it
1116  assert(!isa<PHINode>(Ret->begin()) &&
1117  "All PHI nodes should have been removed!");
1118 
1119  // At this point, we can safely insert a gc.relocate or gc.result as the first
1120  // instruction in Ret if needed.
1121  return Ret;
1122 }
1123 
1124 // Create new attribute set containing only attributes which can be transferred
1125 // from original call to the safepoint.
1127  AttributeSet Ret;
1128 
1129  for (unsigned Slot = 0; Slot < AS.getNumSlots(); Slot++) {
1130  unsigned Index = AS.getSlotIndex(Slot);
1131 
1132  if (Index == AttributeSet::ReturnIndex ||
1133  Index == AttributeSet::FunctionIndex) {
1134 
1135  for (Attribute Attr : make_range(AS.begin(Slot), AS.end(Slot))) {
1136 
1137  // Do not allow certain attributes - just skip them
1138  // Safepoint can not be read only or read none.
1139  if (Attr.hasAttribute(Attribute::ReadNone) ||
1140  Attr.hasAttribute(Attribute::ReadOnly))
1141  continue;
1142 
1143  // These attributes control the generation of the gc.statepoint call /
1144  // invoke itself; and once the gc.statepoint is in place, they're of no
1145  // use.
1146  if (isStatepointDirectiveAttr(Attr))
1147  continue;
1148 
1149  Ret = Ret.addAttributes(
1150  AS.getContext(), Index,
1151  AttributeSet::get(AS.getContext(), Index, AttrBuilder(Attr)));
1152  }
1153  }
1154 
1155  // Just skip parameter attributes for now
1156  }
1157 
1158  return Ret;
1159 }
1160 
1161 /// Helper function to place all gc relocates necessary for the given
1162 /// statepoint.
1163 /// Inputs:
1164 /// liveVariables - list of variables to be relocated.
1165 /// liveStart - index of the first live variable.
1166 /// basePtrs - base pointers.
1167 /// statepointToken - statepoint instruction to which relocates should be
1168 /// bound.
1169 /// Builder - Llvm IR builder to be used to construct new calls.
1171  const int LiveStart,
1172  ArrayRef<Value *> BasePtrs,
1173  Instruction *StatepointToken,
1174  IRBuilder<> Builder) {
1175  if (LiveVariables.empty())
1176  return;
1177 
1178  auto FindIndex = [](ArrayRef<Value *> LiveVec, Value *Val) {
1179  auto ValIt = find(LiveVec, Val);
1180  assert(ValIt != LiveVec.end() && "Val not found in LiveVec!");
1181  size_t Index = std::distance(LiveVec.begin(), ValIt);
1182  assert(Index < LiveVec.size() && "Bug in std::find?");
1183  return Index;
1184  };
1185  Module *M = StatepointToken->getModule();
1186 
1187  // All gc_relocate are generated as i8 addrspace(1)* (or a vector type whose
1188  // element type is i8 addrspace(1)*). We originally generated unique
1189  // declarations for each pointer type, but this proved problematic because
1190  // the intrinsic mangling code is incomplete and fragile. Since we're moving
1191  // towards a single unified pointer type anyways, we can just cast everything
1192  // to an i8* of the right address space. A bitcast is added later to convert
1193  // gc_relocate to the actual value's type.
1194  auto getGCRelocateDecl = [&] (Type *Ty) {
1196  auto AS = Ty->getScalarType()->getPointerAddressSpace();
1197  Type *NewTy = Type::getInt8PtrTy(M->getContext(), AS);
1198  if (auto *VT = dyn_cast<VectorType>(Ty))
1199  NewTy = VectorType::get(NewTy, VT->getNumElements());
1200  return Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate,
1201  {NewTy});
1202  };
1203 
1204  // Lazily populated map from input types to the canonicalized form mentioned
1205  // in the comment above. This should probably be cached somewhere more
1206  // broadly.
1207  DenseMap<Type*, Value*> TypeToDeclMap;
1208 
1209  for (unsigned i = 0; i < LiveVariables.size(); i++) {
1210  // Generate the gc.relocate call and save the result
1211  Value *BaseIdx =
1212  Builder.getInt32(LiveStart + FindIndex(LiveVariables, BasePtrs[i]));
1213  Value *LiveIdx = Builder.getInt32(LiveStart + i);
1214 
1215  Type *Ty = LiveVariables[i]->getType();
1216  if (!TypeToDeclMap.count(Ty))
1217  TypeToDeclMap[Ty] = getGCRelocateDecl(Ty);
1218  Value *GCRelocateDecl = TypeToDeclMap[Ty];
1219 
1220  // only specify a debug name if we can give a useful one
1221  CallInst *Reloc = Builder.CreateCall(
1222  GCRelocateDecl, {StatepointToken, BaseIdx, LiveIdx},
1223  suffixed_name_or(LiveVariables[i], ".relocated", ""));
1224  // Trick CodeGen into thinking there are lots of free registers at this
1225  // fake call.
1227  }
1228 }
1229 
1230 namespace {
1231 
1232 /// This struct is used to defer RAUWs and `eraseFromParent` s. Using this
1233 /// avoids having to worry about keeping around dangling pointers to Values.
1234 class DeferredReplacement {
1237  bool IsDeoptimize = false;
1238 
1239  DeferredReplacement() {}
1240 
1241 public:
1242  static DeferredReplacement createRAUW(Instruction *Old, Instruction *New) {
1243  assert(Old != New && Old && New &&
1244  "Cannot RAUW equal values or to / from null!");
1245 
1246  DeferredReplacement D;
1247  D.Old = Old;
1248  D.New = New;
1249  return D;
1250  }
1251 
1252  static DeferredReplacement createDelete(Instruction *ToErase) {
1253  DeferredReplacement D;
1254  D.Old = ToErase;
1255  return D;
1256  }
1257 
1258  static DeferredReplacement createDeoptimizeReplacement(Instruction *Old) {
1259 #ifndef NDEBUG
1260  auto *F = cast<CallInst>(Old)->getCalledFunction();
1261  assert(F && F->getIntrinsicID() == Intrinsic::experimental_deoptimize &&
1262  "Only way to construct a deoptimize deferred replacement");
1263 #endif
1264  DeferredReplacement D;
1265  D.Old = Old;
1266  D.IsDeoptimize = true;
1267  return D;
1268  }
1269 
1270  /// Does the task represented by this instance.
1271  void doReplacement() {
1272  Instruction *OldI = Old;
1273  Instruction *NewI = New;
1274 
1275  assert(OldI != NewI && "Disallowed at construction?!");
1276  assert((!IsDeoptimize || !New) &&
1277  "Deoptimize instrinsics are not replaced!");
1278 
1279  Old = nullptr;
1280  New = nullptr;
1281 
1282  if (NewI)
1283  OldI->replaceAllUsesWith(NewI);
1284 
1285  if (IsDeoptimize) {
1286  // Note: we've inserted instructions, so the call to llvm.deoptimize may
1287  // not necessarilly be followed by the matching return.
1288  auto *RI = cast<ReturnInst>(OldI->getParent()->getTerminator());
1289  new UnreachableInst(RI->getContext(), RI);
1290  RI->eraseFromParent();
1291  }
1292 
1293  OldI->eraseFromParent();
1294  }
1295 };
1296 }
1297 
1299  const char *DeoptLowering = "deopt-lowering";
1300  if (CS.hasFnAttr(DeoptLowering)) {
1301  // FIXME: CallSite has a *really* confusing interface around attributes
1302  // with values.
1303  const AttributeSet &CSAS = CS.getAttributes();
1305  DeoptLowering))
1307  DeoptLowering).getValueAsString();
1308  Function *F = CS.getCalledFunction();
1309  assert(F && F->hasFnAttribute(DeoptLowering));
1310  return F->getFnAttribute(DeoptLowering).getValueAsString();
1311  }
1312  return "live-through";
1313 }
1314 
1315 
1316 static void
1317 makeStatepointExplicitImpl(const CallSite CS, /* to replace */
1318  const SmallVectorImpl<Value *> &BasePtrs,
1320  PartiallyConstructedSafepointRecord &Result,
1321  std::vector<DeferredReplacement> &Replacements) {
1322  assert(BasePtrs.size() == LiveVariables.size());
1323 
1324  // Then go ahead and use the builder do actually do the inserts. We insert
1325  // immediately before the previous instruction under the assumption that all
1326  // arguments will be available here. We can't insert afterwards since we may
1327  // be replacing a terminator.
1328  Instruction *InsertBefore = CS.getInstruction();
1329  IRBuilder<> Builder(InsertBefore);
1330 
1331  ArrayRef<Value *> GCArgs(LiveVariables);
1332  uint64_t StatepointID = StatepointDirectives::DefaultStatepointID;
1333  uint32_t NumPatchBytes = 0;
1335 
1336  ArrayRef<Use> CallArgs(CS.arg_begin(), CS.arg_end());
1337  ArrayRef<Use> DeoptArgs = GetDeoptBundleOperands(CS);
1338  ArrayRef<Use> TransitionArgs;
1339  if (auto TransitionBundle =
1342  TransitionArgs = TransitionBundle->Inputs;
1343  }
1344 
1345  // Instead of lowering calls to @llvm.experimental.deoptimize as normal calls
1346  // with a return value, we lower then as never returning calls to
1347  // __llvm_deoptimize that are followed by unreachable to get better codegen.
1348  bool IsDeoptimize = false;
1349 
1352  if (SD.NumPatchBytes)
1353  NumPatchBytes = *SD.NumPatchBytes;
1354  if (SD.StatepointID)
1355  StatepointID = *SD.StatepointID;
1356 
1357  // Pass through the requested lowering if any. The default is live-through.
1358  StringRef DeoptLowering = getDeoptLowering(CS);
1359  if (DeoptLowering.equals("live-in"))
1361  else {
1362  assert(DeoptLowering.equals("live-through") && "Unsupported value!");
1363  }
1364 
1365  Value *CallTarget = CS.getCalledValue();
1366  if (Function *F = dyn_cast<Function>(CallTarget)) {
1367  if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize) {
1368  // Calls to llvm.experimental.deoptimize are lowered to calls to the
1369  // __llvm_deoptimize symbol. We want to resolve this now, since the
1370  // verifier does not allow taking the address of an intrinsic function.
1371 
1372  SmallVector<Type *, 8> DomainTy;
1373  for (Value *Arg : CallArgs)
1374  DomainTy.push_back(Arg->getType());
1375  auto *FTy = FunctionType::get(Type::getVoidTy(F->getContext()), DomainTy,
1376  /* isVarArg = */ false);
1377 
1378  // Note: CallTarget can be a bitcast instruction of a symbol if there are
1379  // calls to @llvm.experimental.deoptimize with different argument types in
1380  // the same module. This is fine -- we assume the frontend knew what it
1381  // was doing when generating this kind of IR.
1382  CallTarget =
1383  F->getParent()->getOrInsertFunction("__llvm_deoptimize", FTy);
1384 
1385  IsDeoptimize = true;
1386  }
1387  }
1388 
1389  // Create the statepoint given all the arguments
1390  Instruction *Token = nullptr;
1391  AttributeSet ReturnAttrs;
1392  if (CS.isCall()) {
1393  CallInst *ToReplace = cast<CallInst>(CS.getInstruction());
1395  StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs,
1396  TransitionArgs, DeoptArgs, GCArgs, "safepoint_token");
1397 
1398  Call->setTailCallKind(ToReplace->getTailCallKind());
1399  Call->setCallingConv(ToReplace->getCallingConv());
1400 
1401  // Currently we will fail on parameter attributes and on certain
1402  // function attributes.
1403  AttributeSet NewAttrs = legalizeCallAttributes(ToReplace->getAttributes());
1404  // In case if we can handle this set of attributes - set up function attrs
1405  // directly on statepoint and return attrs later for gc_result intrinsic.
1406  Call->setAttributes(NewAttrs.getFnAttributes());
1407  ReturnAttrs = NewAttrs.getRetAttributes();
1408 
1409  Token = Call;
1410 
1411  // Put the following gc_result and gc_relocate calls immediately after the
1412  // the old call (which we're about to delete)
1413  assert(ToReplace->getNextNode() && "Not a terminator, must have next!");
1414  Builder.SetInsertPoint(ToReplace->getNextNode());
1415  Builder.SetCurrentDebugLocation(ToReplace->getNextNode()->getDebugLoc());
1416  } else {
1417  InvokeInst *ToReplace = cast<InvokeInst>(CS.getInstruction());
1418 
1419  // Insert the new invoke into the old block. We'll remove the old one in a
1420  // moment at which point this will become the new terminator for the
1421  // original block.
1422  InvokeInst *Invoke = Builder.CreateGCStatepointInvoke(
1423  StatepointID, NumPatchBytes, CallTarget, ToReplace->getNormalDest(),
1424  ToReplace->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs,
1425  GCArgs, "statepoint_token");
1426 
1427  Invoke->setCallingConv(ToReplace->getCallingConv());
1428 
1429  // Currently we will fail on parameter attributes and on certain
1430  // function attributes.
1431  AttributeSet NewAttrs = legalizeCallAttributes(ToReplace->getAttributes());
1432  // In case if we can handle this set of attributes - set up function attrs
1433  // directly on statepoint and return attrs later for gc_result intrinsic.
1434  Invoke->setAttributes(NewAttrs.getFnAttributes());
1435  ReturnAttrs = NewAttrs.getRetAttributes();
1436 
1437  Token = Invoke;
1438 
1439  // Generate gc relocates in exceptional path
1440  BasicBlock *UnwindBlock = ToReplace->getUnwindDest();
1441  assert(!isa<PHINode>(UnwindBlock->begin()) &&
1442  UnwindBlock->getUniquePredecessor() &&
1443  "can't safely insert in this block!");
1444 
1445  Builder.SetInsertPoint(&*UnwindBlock->getFirstInsertionPt());
1446  Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
1447 
1448  // Attach exceptional gc relocates to the landingpad.
1449  Instruction *ExceptionalToken = UnwindBlock->getLandingPadInst();
1450  Result.UnwindToken = ExceptionalToken;
1451 
1452  const unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx();
1453  CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, ExceptionalToken,
1454  Builder);
1455 
1456  // Generate gc relocates and returns for normal block
1457  BasicBlock *NormalDest = ToReplace->getNormalDest();
1458  assert(!isa<PHINode>(NormalDest->begin()) &&
1459  NormalDest->getUniquePredecessor() &&
1460  "can't safely insert in this block!");
1461 
1462  Builder.SetInsertPoint(&*NormalDest->getFirstInsertionPt());
1463 
1464  // gc relocates will be generated later as if it were regular call
1465  // statepoint
1466  }
1467  assert(Token && "Should be set in one of the above branches!");
1468 
1469  if (IsDeoptimize) {
1470  // If we're wrapping an @llvm.experimental.deoptimize in a statepoint, we
1471  // transform the tail-call like structure to a call to a void function
1472  // followed by unreachable to get better codegen.
1473  Replacements.push_back(
1474  DeferredReplacement::createDeoptimizeReplacement(CS.getInstruction()));
1475  } else {
1476  Token->setName("statepoint_token");
1477  if (!CS.getType()->isVoidTy() && !CS.getInstruction()->use_empty()) {
1478  StringRef Name =
1479  CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : "";
1480  CallInst *GCResult = Builder.CreateGCResult(Token, CS.getType(), Name);
1481  GCResult->setAttributes(CS.getAttributes().getRetAttributes());
1482 
1483  // We cannot RAUW or delete CS.getInstruction() because it could be in the
1484  // live set of some other safepoint, in which case that safepoint's
1485  // PartiallyConstructedSafepointRecord will hold a raw pointer to this
1486  // llvm::Instruction. Instead, we defer the replacement and deletion to
1487  // after the live sets have been made explicit in the IR, and we no longer
1488  // have raw pointers to worry about.
1489  Replacements.emplace_back(
1490  DeferredReplacement::createRAUW(CS.getInstruction(), GCResult));
1491  } else {
1492  Replacements.emplace_back(
1493  DeferredReplacement::createDelete(CS.getInstruction()));
1494  }
1495  }
1496 
1497  Result.StatepointToken = Token;
1498 
1499  // Second, create a gc.relocate for every live variable
1500  const unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx();
1501  CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, Token, Builder);
1502 }
1503 
1504 // Replace an existing gc.statepoint with a new one and a set of gc.relocates
1505 // which make the relocations happening at this safepoint explicit.
1506 //
1507 // WARNING: Does not do any fixup to adjust users of the original live
1508 // values. That's the callers responsibility.
1509 static void
1511  PartiallyConstructedSafepointRecord &Result,
1512  std::vector<DeferredReplacement> &Replacements) {
1513  const auto &LiveSet = Result.LiveSet;
1514  const auto &PointerToBase = Result.PointerToBase;
1515 
1516  // Convert to vector for efficient cross referencing.
1517  SmallVector<Value *, 64> BaseVec, LiveVec;
1518  LiveVec.reserve(LiveSet.size());
1519  BaseVec.reserve(LiveSet.size());
1520  for (Value *L : LiveSet) {
1521  LiveVec.push_back(L);
1522  assert(PointerToBase.count(L));
1523  Value *Base = PointerToBase.find(L)->second;
1524  BaseVec.push_back(Base);
1525  }
1526  assert(LiveVec.size() == BaseVec.size());
1527 
1528  // Do the actual rewriting and delete the old statepoint
1529  makeStatepointExplicitImpl(CS, BaseVec, LiveVec, Result, Replacements);
1530 }
1531 
1532 // Helper function for the relocationViaAlloca.
1533 //
1534 // It receives iterator to the statepoint gc relocates and emits a store to the
1535 // assigned location (via allocaMap) for the each one of them. It adds the
1536 // visited values into the visitedLiveValues set, which we will later use them
1537 // for sanity checking.
1538 static void
1540  DenseMap<Value *, Value *> &AllocaMap,
1541  DenseSet<Value *> &VisitedLiveValues) {
1542 
1543  for (User *U : GCRelocs) {
1544  GCRelocateInst *Relocate = dyn_cast<GCRelocateInst>(U);
1545  if (!Relocate)
1546  continue;
1547 
1548  Value *OriginalValue = Relocate->getDerivedPtr();
1549  assert(AllocaMap.count(OriginalValue));
1550  Value *Alloca = AllocaMap[OriginalValue];
1551 
1552  // Emit store into the related alloca
1553  // All gc_relocates are i8 addrspace(1)* typed, and it must be bitcasted to
1554  // the correct type according to alloca.
1555  assert(Relocate->getNextNode() &&
1556  "Should always have one since it's not a terminator");
1557  IRBuilder<> Builder(Relocate->getNextNode());
1558  Value *CastedRelocatedValue =
1559  Builder.CreateBitCast(Relocate,
1560  cast<AllocaInst>(Alloca)->getAllocatedType(),
1561  suffixed_name_or(Relocate, ".casted", ""));
1562 
1563  StoreInst *Store = new StoreInst(CastedRelocatedValue, Alloca);
1564  Store->insertAfter(cast<Instruction>(CastedRelocatedValue));
1565 
1566 #ifndef NDEBUG
1567  VisitedLiveValues.insert(OriginalValue);
1568 #endif
1569  }
1570 }
1571 
1572 // Helper function for the "relocationViaAlloca". Similar to the
1573 // "insertRelocationStores" but works for rematerialized values.
1575  const RematerializedValueMapTy &RematerializedValues,
1576  DenseMap<Value *, Value *> &AllocaMap,
1577  DenseSet<Value *> &VisitedLiveValues) {
1578 
1579  for (auto RematerializedValuePair: RematerializedValues) {
1580  Instruction *RematerializedValue = RematerializedValuePair.first;
1581  Value *OriginalValue = RematerializedValuePair.second;
1582 
1583  assert(AllocaMap.count(OriginalValue) &&
1584  "Can not find alloca for rematerialized value");
1585  Value *Alloca = AllocaMap[OriginalValue];
1586 
1587  StoreInst *Store = new StoreInst(RematerializedValue, Alloca);
1588  Store->insertAfter(RematerializedValue);
1589 
1590 #ifndef NDEBUG
1591  VisitedLiveValues.insert(OriginalValue);
1592 #endif
1593  }
1594 }
1595 
1596 /// Do all the relocation update via allocas and mem2reg
1600 #ifndef NDEBUG
1601  // record initial number of (static) allocas; we'll check we have the same
1602  // number when we get done.
1603  int InitialAllocaNum = 0;
1604  for (Instruction &I : F.getEntryBlock())
1605  if (isa<AllocaInst>(I))
1606  InitialAllocaNum++;
1607 #endif
1608 
1609  // TODO-PERF: change data structures, reserve
1610  DenseMap<Value *, Value *> AllocaMap;
1611  SmallVector<AllocaInst *, 200> PromotableAllocas;
1612  // Used later to chack that we have enough allocas to store all values
1613  std::size_t NumRematerializedValues = 0;
1614  PromotableAllocas.reserve(Live.size());
1615 
1616  // Emit alloca for "LiveValue" and record it in "allocaMap" and
1617  // "PromotableAllocas"
1618  auto emitAllocaFor = [&](Value *LiveValue) {
1619  AllocaInst *Alloca = new AllocaInst(LiveValue->getType(), "",
1621  AllocaMap[LiveValue] = Alloca;
1622  PromotableAllocas.push_back(Alloca);
1623  };
1624 
1625  // Emit alloca for each live gc pointer
1626  for (Value *V : Live)
1627  emitAllocaFor(V);
1628 
1629  // Emit allocas for rematerialized values
1630  for (const auto &Info : Records)
1631  for (auto RematerializedValuePair : Info.RematerializedValues) {
1632  Value *OriginalValue = RematerializedValuePair.second;
1633  if (AllocaMap.count(OriginalValue) != 0)
1634  continue;
1635 
1636  emitAllocaFor(OriginalValue);
1637  ++NumRematerializedValues;
1638  }
1639 
1640  // The next two loops are part of the same conceptual operation. We need to
1641  // insert a store to the alloca after the original def and at each
1642  // redefinition. We need to insert a load before each use. These are split
1643  // into distinct loops for performance reasons.
1644 
1645  // Update gc pointer after each statepoint: either store a relocated value or
1646  // null (if no relocated value was found for this gc pointer and it is not a
1647  // gc_result). This must happen before we update the statepoint with load of
1648  // alloca otherwise we lose the link between statepoint and old def.
1649  for (const auto &Info : Records) {
1650  Value *Statepoint = Info.StatepointToken;
1651 
1652  // This will be used for consistency check
1653  DenseSet<Value *> VisitedLiveValues;
1654 
1655  // Insert stores for normal statepoint gc relocates
1656  insertRelocationStores(Statepoint->users(), AllocaMap, VisitedLiveValues);
1657 
1658  // In case if it was invoke statepoint
1659  // we will insert stores for exceptional path gc relocates.
1660  if (isa<InvokeInst>(Statepoint)) {
1661  insertRelocationStores(Info.UnwindToken->users(), AllocaMap,
1662  VisitedLiveValues);
1663  }
1664 
1665  // Do similar thing with rematerialized values
1666  insertRematerializationStores(Info.RematerializedValues, AllocaMap,
1667  VisitedLiveValues);
1668 
1669  if (ClobberNonLive) {
1670  // As a debugging aid, pretend that an unrelocated pointer becomes null at
1671  // the gc.statepoint. This will turn some subtle GC problems into
1672  // slightly easier to debug SEGVs. Note that on large IR files with
1673  // lots of gc.statepoints this is extremely costly both memory and time
1674  // wise.
1676  for (auto Pair : AllocaMap) {
1677  Value *Def = Pair.first;
1678  AllocaInst *Alloca = cast<AllocaInst>(Pair.second);
1679 
1680  // This value was relocated
1681  if (VisitedLiveValues.count(Def)) {
1682  continue;
1683  }
1684  ToClobber.push_back(Alloca);
1685  }
1686 
1687  auto InsertClobbersAt = [&](Instruction *IP) {
1688  for (auto *AI : ToClobber) {
1689  auto PT = cast<PointerType>(AI->getAllocatedType());
1691  StoreInst *Store = new StoreInst(CPN, AI);
1692  Store->insertBefore(IP);
1693  }
1694  };
1695 
1696  // Insert the clobbering stores. These may get intermixed with the
1697  // gc.results and gc.relocates, but that's fine.
1698  if (auto II = dyn_cast<InvokeInst>(Statepoint)) {
1699  InsertClobbersAt(&*II->getNormalDest()->getFirstInsertionPt());
1700  InsertClobbersAt(&*II->getUnwindDest()->getFirstInsertionPt());
1701  } else {
1702  InsertClobbersAt(cast<Instruction>(Statepoint)->getNextNode());
1703  }
1704  }
1705  }
1706 
1707  // Update use with load allocas and add store for gc_relocated.
1708  for (auto Pair : AllocaMap) {
1709  Value *Def = Pair.first;
1710  Value *Alloca = Pair.second;
1711 
1712  // We pre-record the uses of allocas so that we dont have to worry about
1713  // later update that changes the user information..
1714 
1716  // PERF: trade a linear scan for repeated reallocation
1717  Uses.reserve(std::distance(Def->user_begin(), Def->user_end()));
1718  for (User *U : Def->users()) {
1719  if (!isa<ConstantExpr>(U)) {
1720  // If the def has a ConstantExpr use, then the def is either a
1721  // ConstantExpr use itself or null. In either case
1722  // (recursively in the first, directly in the second), the oop
1723  // it is ultimately dependent on is null and this particular
1724  // use does not need to be fixed up.
1725  Uses.push_back(cast<Instruction>(U));
1726  }
1727  }
1728 
1729  std::sort(Uses.begin(), Uses.end());
1730  auto Last = std::unique(Uses.begin(), Uses.end());
1731  Uses.erase(Last, Uses.end());
1732 
1733  for (Instruction *Use : Uses) {
1734  if (isa<PHINode>(Use)) {
1735  PHINode *Phi = cast<PHINode>(Use);
1736  for (unsigned i = 0; i < Phi->getNumIncomingValues(); i++) {
1737  if (Def == Phi->getIncomingValue(i)) {
1738  LoadInst *Load = new LoadInst(
1739  Alloca, "", Phi->getIncomingBlock(i)->getTerminator());
1740  Phi->setIncomingValue(i, Load);
1741  }
1742  }
1743  } else {
1744  LoadInst *Load = new LoadInst(Alloca, "", Use);
1745  Use->replaceUsesOfWith(Def, Load);
1746  }
1747  }
1748 
1749  // Emit store for the initial gc value. Store must be inserted after load,
1750  // otherwise store will be in alloca's use list and an extra load will be
1751  // inserted before it.
1752  StoreInst *Store = new StoreInst(Def, Alloca);
1753  if (Instruction *Inst = dyn_cast<Instruction>(Def)) {
1754  if (InvokeInst *Invoke = dyn_cast<InvokeInst>(Inst)) {
1755  // InvokeInst is a TerminatorInst so the store need to be inserted
1756  // into its normal destination block.
1757  BasicBlock *NormalDest = Invoke->getNormalDest();
1758  Store->insertBefore(NormalDest->getFirstNonPHI());
1759  } else {
1760  assert(!Inst->isTerminator() &&
1761  "The only TerminatorInst that can produce a value is "
1762  "InvokeInst which is handled above.");
1763  Store->insertAfter(Inst);
1764  }
1765  } else {
1766  assert(isa<Argument>(Def));
1767  Store->insertAfter(cast<Instruction>(Alloca));
1768  }
1769  }
1770 
1771  assert(PromotableAllocas.size() == Live.size() + NumRematerializedValues &&
1772  "we must have the same allocas with lives");
1773  if (!PromotableAllocas.empty()) {
1774  // Apply mem2reg to promote alloca to SSA
1775  PromoteMemToReg(PromotableAllocas, DT);
1776  }
1777 
1778 #ifndef NDEBUG
1779  for (auto &I : F.getEntryBlock())
1780  if (isa<AllocaInst>(I))
1781  InitialAllocaNum--;
1782  assert(InitialAllocaNum == 0 && "We must not introduce any extra allocas");
1783 #endif
1784 }
1785 
1786 /// Implement a unique function which doesn't require we sort the input
1787 /// vector. Doing so has the effect of changing the output of a couple of
1788 /// tests in ways which make them less useful in testing fused safepoints.
1789 template <typename T> static void unique_unsorted(SmallVectorImpl<T> &Vec) {
1790  SmallSet<T, 8> Seen;
1791  Vec.erase(remove_if(Vec, [&](const T &V) { return !Seen.insert(V).second; }),
1792  Vec.end());
1793 }
1794 
1795 /// Insert holders so that each Value is obviously live through the entire
1796 /// lifetime of the call.
1797 static void insertUseHolderAfter(CallSite &CS, const ArrayRef<Value *> Values,
1798  SmallVectorImpl<CallInst *> &Holders) {
1799  if (Values.empty())
1800  // No values to hold live, might as well not insert the empty holder
1801  return;
1802 
1803  Module *M = CS.getInstruction()->getModule();
1804  // Use a dummy vararg function to actually hold the values live
1805  Function *Func = cast<Function>(M->getOrInsertFunction(
1806  "__tmp_use", FunctionType::get(Type::getVoidTy(M->getContext()), true)));
1807  if (CS.isCall()) {
1808  // For call safepoints insert dummy calls right after safepoint
1809  Holders.push_back(CallInst::Create(Func, Values, "",
1810  &*++CS.getInstruction()->getIterator()));
1811  return;
1812  }
1813  // For invoke safepooints insert dummy calls both in normal and
1814  // exceptional destination blocks
1815  auto *II = cast<InvokeInst>(CS.getInstruction());
1816  Holders.push_back(CallInst::Create(
1817  Func, Values, "", &*II->getNormalDest()->getFirstInsertionPt()));
1818  Holders.push_back(CallInst::Create(
1819  Func, Values, "", &*II->getUnwindDest()->getFirstInsertionPt()));
1820 }
1821 
1823  Function &F, DominatorTree &DT, ArrayRef<CallSite> toUpdate,
1825  GCPtrLivenessData OriginalLivenessData;
1826  computeLiveInValues(DT, F, OriginalLivenessData);
1827  for (size_t i = 0; i < records.size(); i++) {
1828  struct PartiallyConstructedSafepointRecord &info = records[i];
1829  analyzeParsePointLiveness(DT, OriginalLivenessData, toUpdate[i], info);
1830  }
1831 }
1832 
1833 // Helper function for the "rematerializeLiveValues". It walks use chain
1834 // starting from the "CurrentValue" until it reaches the root of the chain, i.e.
1835 // the base or a value it cannot process. Only "simple" values are processed
1836 // (currently it is GEP's and casts). The returned root is examined by the
1837 // callers of findRematerializableChainToBasePointer. Fills "ChainToBase" array
1838 // with all visited values.
1840  SmallVectorImpl<Instruction*> &ChainToBase,
1841  Value *CurrentValue) {
1842 
1843  if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(CurrentValue)) {
1844  ChainToBase.push_back(GEP);
1845  return findRematerializableChainToBasePointer(ChainToBase,
1846  GEP->getPointerOperand());
1847  }
1848 
1849  if (CastInst *CI = dyn_cast<CastInst>(CurrentValue)) {
1850  if (!CI->isNoopCast(CI->getModule()->getDataLayout()))
1851  return CI;
1852 
1853  ChainToBase.push_back(CI);
1854  return findRematerializableChainToBasePointer(ChainToBase,
1855  CI->getOperand(0));
1856  }
1857 
1858  // We have reached the root of the chain, which is either equal to the base or
1859  // is the first unsupported value along the use chain.
1860  return CurrentValue;
1861 }
1862 
1863 // Helper function for the "rematerializeLiveValues". Compute cost of the use
1864 // chain we are going to rematerialize.
1865 static unsigned
1867  TargetTransformInfo &TTI) {
1868  unsigned Cost = 0;
1869 
1870  for (Instruction *Instr : Chain) {
1871  if (CastInst *CI = dyn_cast<CastInst>(Instr)) {
1872  assert(CI->isNoopCast(CI->getModule()->getDataLayout()) &&
1873  "non noop cast is found during rematerialization");
1874 
1875  Type *SrcTy = CI->getOperand(0)->getType();
1876  Cost += TTI.getCastInstrCost(CI->getOpcode(), CI->getType(), SrcTy);
1877 
1878  } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Instr)) {
1879  // Cost of the address calculation
1880  Type *ValTy = GEP->getSourceElementType();
1881  Cost += TTI.getAddressComputationCost(ValTy);
1882 
1883  // And cost of the GEP itself
1884  // TODO: Use TTI->getGEPCost here (it exists, but appears to be not
1885  // allowed for the external usage)
1886  if (!GEP->hasAllConstantIndices())
1887  Cost += 2;
1888 
1889  } else {
1890  llvm_unreachable("unsupported instruciton type during rematerialization");
1891  }
1892  }
1893 
1894  return Cost;
1895 }
1896 
1897 static bool AreEquivalentPhiNodes(PHINode &OrigRootPhi, PHINode &AlternateRootPhi) {
1898 
1899  unsigned PhiNum = OrigRootPhi.getNumIncomingValues();
1900  if (PhiNum != AlternateRootPhi.getNumIncomingValues() ||
1901  OrigRootPhi.getParent() != AlternateRootPhi.getParent())
1902  return false;
1903  // Map of incoming values and their corresponding basic blocks of
1904  // OrigRootPhi.
1905  SmallDenseMap<Value *, BasicBlock *, 8> CurrentIncomingValues;
1906  for (unsigned i = 0; i < PhiNum; i++)
1907  CurrentIncomingValues[OrigRootPhi.getIncomingValue(i)] =
1908  OrigRootPhi.getIncomingBlock(i);
1909 
1910  // Both current and base PHIs should have same incoming values and
1911  // the same basic blocks corresponding to the incoming values.
1912  for (unsigned i = 0; i < PhiNum; i++) {
1913  auto CIVI =
1914  CurrentIncomingValues.find(AlternateRootPhi.getIncomingValue(i));
1915  if (CIVI == CurrentIncomingValues.end())
1916  return false;
1917  BasicBlock *CurrentIncomingBB = CIVI->second;
1918  if (CurrentIncomingBB != AlternateRootPhi.getIncomingBlock(i))
1919  return false;
1920  }
1921  return true;
1922 
1923 }
1924 
1925 // From the statepoint live set pick values that are cheaper to recompute then
1926 // to relocate. Remove this values from the live set, rematerialize them after
1927 // statepoint and record them in "Info" structure. Note that similar to
1928 // relocated values we don't do any user adjustments here.
1930  PartiallyConstructedSafepointRecord &Info,
1931  TargetTransformInfo &TTI) {
1932  const unsigned int ChainLengthThreshold = 10;
1933 
1934  // Record values we are going to delete from this statepoint live set.
1935  // We can not di this in following loop due to iterator invalidation.
1936  SmallVector<Value *, 32> LiveValuesToBeDeleted;
1937 
1938  for (Value *LiveValue: Info.LiveSet) {
1939  // For each live pointer find it's defining chain
1940  SmallVector<Instruction *, 3> ChainToBase;
1941  assert(Info.PointerToBase.count(LiveValue));
1942  Value *RootOfChain =
1944  LiveValue);
1945 
1946  // Nothing to do, or chain is too long
1947  if ( ChainToBase.size() == 0 ||
1948  ChainToBase.size() > ChainLengthThreshold)
1949  continue;
1950 
1951  // Handle the scenario where the RootOfChain is not equal to the
1952  // Base Value, but they are essentially the same phi values.
1953  if (RootOfChain != Info.PointerToBase[LiveValue]) {
1954  PHINode *OrigRootPhi = dyn_cast<PHINode>(RootOfChain);
1955  PHINode *AlternateRootPhi = dyn_cast<PHINode>(Info.PointerToBase[LiveValue]);
1956  if (!OrigRootPhi || !AlternateRootPhi)
1957  continue;
1958  // PHI nodes that have the same incoming values, and belonging to the same
1959  // basic blocks are essentially the same SSA value. When the original phi
1960  // has incoming values with different base pointers, the original phi is
1961  // marked as conflict, and an additional `AlternateRootPhi` with the same
1962  // incoming values get generated by the findBasePointer function. We need
1963  // to identify the newly generated AlternateRootPhi (.base version of phi)
1964  // and RootOfChain (the original phi node itself) are the same, so that we
1965  // can rematerialize the gep and casts. This is a workaround for the
1966  // deficieny in the findBasePointer algorithm.
1967  if (!AreEquivalentPhiNodes(*OrigRootPhi, *AlternateRootPhi))
1968  continue;
1969  // Now that the phi nodes are proved to be the same, assert that
1970  // findBasePointer's newly generated AlternateRootPhi is present in the
1971  // liveset of the call.
1972  assert(Info.LiveSet.count(AlternateRootPhi));
1973  }
1974  // Compute cost of this chain
1975  unsigned Cost = chainToBasePointerCost(ChainToBase, TTI);
1976  // TODO: We can also account for cases when we will be able to remove some
1977  // of the rematerialized values by later optimization passes. I.e if
1978  // we rematerialized several intersecting chains. Or if original values
1979  // don't have any uses besides this statepoint.
1980 
1981  // For invokes we need to rematerialize each chain twice - for normal and
1982  // for unwind basic blocks. Model this by multiplying cost by two.
1983  if (CS.isInvoke()) {
1984  Cost *= 2;
1985  }
1986  // If it's too expensive - skip it
1987  if (Cost >= RematerializationThreshold)
1988  continue;
1989 
1990  // Remove value from the live set
1991  LiveValuesToBeDeleted.push_back(LiveValue);
1992 
1993  // Clone instructions and record them inside "Info" structure
1994 
1995  // Walk backwards to visit top-most instructions first
1996  std::reverse(ChainToBase.begin(), ChainToBase.end());
1997 
1998  // Utility function which clones all instructions from "ChainToBase"
1999  // and inserts them before "InsertBefore". Returns rematerialized value
2000  // which should be used after statepoint.
2001  auto rematerializeChain = [&ChainToBase](
2002  Instruction *InsertBefore, Value *RootOfChain, Value *AlternateLiveBase) {
2003  Instruction *LastClonedValue = nullptr;
2004  Instruction *LastValue = nullptr;
2005  for (Instruction *Instr: ChainToBase) {
2006  // Only GEP's and casts are suported as we need to be careful to not
2007  // introduce any new uses of pointers not in the liveset.
2008  // Note that it's fine to introduce new uses of pointers which were
2009  // otherwise not used after this statepoint.
2010  assert(isa<GetElementPtrInst>(Instr) || isa<CastInst>(Instr));
2011 
2012  Instruction *ClonedValue = Instr->clone();
2013  ClonedValue->insertBefore(InsertBefore);
2014  ClonedValue->setName(Instr->getName() + ".remat");
2015 
2016  // If it is not first instruction in the chain then it uses previously
2017  // cloned value. We should update it to use cloned value.
2018  if (LastClonedValue) {
2019  assert(LastValue);
2020  ClonedValue->replaceUsesOfWith(LastValue, LastClonedValue);
2021 #ifndef NDEBUG
2022  for (auto OpValue : ClonedValue->operand_values()) {
2023  // Assert that cloned instruction does not use any instructions from
2024  // this chain other than LastClonedValue
2025  assert(!is_contained(ChainToBase, OpValue) &&
2026  "incorrect use in rematerialization chain");
2027  // Assert that the cloned instruction does not use the RootOfChain
2028  // or the AlternateLiveBase.
2029  assert(OpValue != RootOfChain && OpValue != AlternateLiveBase);
2030  }
2031 #endif
2032  } else {
2033  // For the first instruction, replace the use of unrelocated base i.e.
2034  // RootOfChain/OrigRootPhi, with the corresponding PHI present in the
2035  // live set. They have been proved to be the same PHI nodes. Note
2036  // that the *only* use of the RootOfChain in the ChainToBase list is
2037  // the first Value in the list.
2038  if (RootOfChain != AlternateLiveBase)
2039  ClonedValue->replaceUsesOfWith(RootOfChain, AlternateLiveBase);
2040  }
2041 
2042  LastClonedValue = ClonedValue;
2043  LastValue = Instr;
2044  }
2045  assert(LastClonedValue);
2046  return LastClonedValue;
2047  };
2048 
2049  // Different cases for calls and invokes. For invokes we need to clone
2050  // instructions both on normal and unwind path.
2051  if (CS.isCall()) {
2052  Instruction *InsertBefore = CS.getInstruction()->getNextNode();
2053  assert(InsertBefore);
2054  Instruction *RematerializedValue = rematerializeChain(
2055  InsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
2056  Info.RematerializedValues[RematerializedValue] = LiveValue;
2057  } else {
2058  InvokeInst *Invoke = cast<InvokeInst>(CS.getInstruction());
2059 
2060  Instruction *NormalInsertBefore =
2061  &*Invoke->getNormalDest()->getFirstInsertionPt();
2062  Instruction *UnwindInsertBefore =
2063  &*Invoke->getUnwindDest()->getFirstInsertionPt();
2064 
2065  Instruction *NormalRematerializedValue = rematerializeChain(
2066  NormalInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
2067  Instruction *UnwindRematerializedValue = rematerializeChain(
2068  UnwindInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
2069 
2070  Info.RematerializedValues[NormalRematerializedValue] = LiveValue;
2071  Info.RematerializedValues[UnwindRematerializedValue] = LiveValue;
2072  }
2073  }
2074 
2075  // Remove rematerializaed values from the live set
2076  for (auto LiveValue: LiveValuesToBeDeleted) {
2077  Info.LiveSet.remove(LiveValue);
2078  }
2079 }
2080 
2082  TargetTransformInfo &TTI,
2083  SmallVectorImpl<CallSite> &ToUpdate) {
2084 #ifndef NDEBUG
2085  // sanity check the input
2086  std::set<CallSite> Uniqued;
2087  Uniqued.insert(ToUpdate.begin(), ToUpdate.end());
2088  assert(Uniqued.size() == ToUpdate.size() && "no duplicates please!");
2089 
2090  for (CallSite CS : ToUpdate)
2091  assert(CS.getInstruction()->getFunction() == &F);
2092 #endif
2093 
2094  // When inserting gc.relocates for invokes, we need to be able to insert at
2095  // the top of the successor blocks. See the comment on
2096  // normalForInvokeSafepoint on exactly what is needed. Note that this step
2097  // may restructure the CFG.
2098  for (CallSite CS : ToUpdate) {
2099  if (!CS.isInvoke())
2100  continue;
2101  auto *II = cast<InvokeInst>(CS.getInstruction());
2102  normalizeForInvokeSafepoint(II->getNormalDest(), II->getParent(), DT);
2103  normalizeForInvokeSafepoint(II->getUnwindDest(), II->getParent(), DT);
2104  }
2105 
2106  // A list of dummy calls added to the IR to keep various values obviously
2107  // live in the IR. We'll remove all of these when done.
2109 
2110  // Insert a dummy call with all of the arguments to the vm_state we'll need
2111  // for the actual safepoint insertion. This ensures reference arguments in
2112  // the deopt argument list are considered live through the safepoint (and
2113  // thus makes sure they get relocated.)
2114  for (CallSite CS : ToUpdate) {
2115  SmallVector<Value *, 64> DeoptValues;
2116 
2117  for (Value *Arg : GetDeoptBundleOperands(CS)) {
2118  assert(!isUnhandledGCPointerType(Arg->getType()) &&
2119  "support for FCA unimplemented");
2120  if (isHandledGCPointerType(Arg->getType()))
2121  DeoptValues.push_back(Arg);
2122  }
2123 
2124  insertUseHolderAfter(CS, DeoptValues, Holders);
2125  }
2126 
2128 
2129  // A) Identify all gc pointers which are statically live at the given call
2130  // site.
2131  findLiveReferences(F, DT, ToUpdate, Records);
2132 
2133  // B) Find the base pointers for each live pointer
2134  /* scope for caching */ {
2135  // Cache the 'defining value' relation used in the computation and
2136  // insertion of base phis and selects. This ensures that we don't insert
2137  // large numbers of duplicate base_phis.
2138  DefiningValueMapTy DVCache;
2139 
2140  for (size_t i = 0; i < Records.size(); i++) {
2141  PartiallyConstructedSafepointRecord &info = Records[i];
2142  findBasePointers(DT, DVCache, ToUpdate[i], info);
2143  }
2144  } // end of cache scope
2145 
2146  // The base phi insertion logic (for any safepoint) may have inserted new
2147  // instructions which are now live at some safepoint. The simplest such
2148  // example is:
2149  // loop:
2150  // phi a <-- will be a new base_phi here
2151  // safepoint 1 <-- that needs to be live here
2152  // gep a + 1
2153  // safepoint 2
2154  // br loop
2155  // We insert some dummy calls after each safepoint to definitely hold live
2156  // the base pointers which were identified for that safepoint. We'll then
2157  // ask liveness for _every_ base inserted to see what is now live. Then we
2158  // remove the dummy calls.
2159  Holders.reserve(Holders.size() + Records.size());
2160  for (size_t i = 0; i < Records.size(); i++) {
2161  PartiallyConstructedSafepointRecord &Info = Records[i];
2162 
2164  for (auto Pair : Info.PointerToBase)
2165  Bases.push_back(Pair.second);
2166 
2167  insertUseHolderAfter(ToUpdate[i], Bases, Holders);
2168  }
2169 
2170  // By selecting base pointers, we've effectively inserted new uses. Thus, we
2171  // need to rerun liveness. We may *also* have inserted new defs, but that's
2172  // not the key issue.
2173  recomputeLiveInValues(F, DT, ToUpdate, Records);
2174 
2175  if (PrintBasePointers) {
2176  for (auto &Info : Records) {
2177  errs() << "Base Pairs: (w/Relocation)\n";
2178  for (auto Pair : Info.PointerToBase) {
2179  errs() << " derived ";
2180  Pair.first->printAsOperand(errs(), false);
2181  errs() << " base ";
2182  Pair.second->printAsOperand(errs(), false);
2183  errs() << "\n";
2184  }
2185  }
2186  }
2187 
2188  // It is possible that non-constant live variables have a constant base. For
2189  // example, a GEP with a variable offset from a global. In this case we can
2190  // remove it from the liveset. We already don't add constants to the liveset
2191  // because we assume they won't move at runtime and the GC doesn't need to be
2192  // informed about them. The same reasoning applies if the base is constant.
2193  // Note that the relocation placement code relies on this filtering for
2194  // correctness as it expects the base to be in the liveset, which isn't true
2195  // if the base is constant.
2196  for (auto &Info : Records)
2197  for (auto &BasePair : Info.PointerToBase)
2198  if (isa<Constant>(BasePair.second))
2199  Info.LiveSet.remove(BasePair.first);
2200 
2201  for (CallInst *CI : Holders)
2202  CI->eraseFromParent();
2203 
2204  Holders.clear();
2205 
2206  // In order to reduce live set of statepoint we might choose to rematerialize
2207  // some values instead of relocating them. This is purely an optimization and
2208  // does not influence correctness.
2209  for (size_t i = 0; i < Records.size(); i++)
2210  rematerializeLiveValues(ToUpdate[i], Records[i], TTI);
2211 
2212  // We need this to safely RAUW and delete call or invoke return values that
2213  // may themselves be live over a statepoint. For details, please see usage in
2214  // makeStatepointExplicitImpl.
2215  std::vector<DeferredReplacement> Replacements;
2216 
2217  // Now run through and replace the existing statepoints with new ones with
2218  // the live variables listed. We do not yet update uses of the values being
2219  // relocated. We have references to live variables that need to
2220  // survive to the last iteration of this loop. (By construction, the
2221  // previous statepoint can not be a live variable, thus we can and remove
2222  // the old statepoint calls as we go.)
2223  for (size_t i = 0; i < Records.size(); i++)
2224  makeStatepointExplicit(DT, ToUpdate[i], Records[i], Replacements);
2225 
2226  ToUpdate.clear(); // prevent accident use of invalid CallSites
2227 
2228  for (auto &PR : Replacements)
2229  PR.doReplacement();
2230 
2231  Replacements.clear();
2232 
2233  for (auto &Info : Records) {
2234  // These live sets may contain state Value pointers, since we replaced calls
2235  // with operand bundles with calls wrapped in gc.statepoint, and some of
2236  // those calls may have been def'ing live gc pointers. Clear these out to
2237  // avoid accidentally using them.
2238  //
2239  // TODO: We should create a separate data structure that does not contain
2240  // these live sets, and migrate to using that data structure from this point
2241  // onward.
2242  Info.LiveSet.clear();
2243  Info.PointerToBase.clear();
2244  }
2245 
2246  // Do all the fixups of the original live variables to their relocated selves
2248  for (size_t i = 0; i < Records.size(); i++) {
2249  PartiallyConstructedSafepointRecord &Info = Records[i];
2250 
2251  // We can't simply save the live set from the original insertion. One of
2252  // the live values might be the result of a call which needs a safepoint.
2253  // That Value* no longer exists and we need to use the new gc_result.
2254  // Thankfully, the live set is embedded in the statepoint (and updated), so
2255  // we just grab that.
2256  Statepoint Statepoint(Info.StatepointToken);
2257  Live.insert(Live.end(), Statepoint.gc_args_begin(),
2258  Statepoint.gc_args_end());
2259 #ifndef NDEBUG
2260  // Do some basic sanity checks on our liveness results before performing
2261  // relocation. Relocation can and will turn mistakes in liveness results
2262  // into non-sensical code which is must harder to debug.
2263  // TODO: It would be nice to test consistency as well
2264  assert(DT.isReachableFromEntry(Info.StatepointToken->getParent()) &&
2265  "statepoint must be reachable or liveness is meaningless");
2266  for (Value *V : Statepoint.gc_args()) {
2267  if (!isa<Instruction>(V))
2268  // Non-instruction values trivial dominate all possible uses
2269  continue;
2270  auto *LiveInst = cast<Instruction>(V);
2271  assert(DT.isReachableFromEntry(LiveInst->getParent()) &&
2272  "unreachable values should never be live");
2273  assert(DT.dominates(LiveInst, Info.StatepointToken) &&
2274  "basic SSA liveness expectation violated by liveness analysis");
2275  }
2276 #endif
2277  }
2278  unique_unsorted(Live);
2279 
2280 #ifndef NDEBUG
2281  // sanity check
2282  for (auto *Ptr : Live)
2283  assert(isHandledGCPointerType(Ptr->getType()) &&
2284  "must be a gc pointer type");
2285 #endif
2286 
2287  relocationViaAlloca(F, DT, Live, Records);
2288  return !Records.empty();
2289 }
2290 
2291 // Handles both return values and arguments for Functions and CallSites.
2292 template <typename AttrHolder>
2293 static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH,
2294  unsigned Index) {
2295  AttrBuilder R;
2296  if (AH.getDereferenceableBytes(Index))
2297  R.addAttribute(Attribute::get(Ctx, Attribute::Dereferenceable,
2298  AH.getDereferenceableBytes(Index)));
2299  if (AH.getDereferenceableOrNullBytes(Index))
2300  R.addAttribute(Attribute::get(Ctx, Attribute::DereferenceableOrNull,
2301  AH.getDereferenceableOrNullBytes(Index)));
2302  if (AH.doesNotAlias(Index))
2304 
2305  if (!R.empty())
2306  AH.setAttributes(AH.getAttributes().removeAttributes(
2307  Ctx, Index, AttributeSet::get(Ctx, Index, R)));
2308 }
2309 
2310 void
2311 RewriteStatepointsForGC::stripNonValidAttributesFromPrototype(Function &F) {
2312  LLVMContext &Ctx = F.getContext();
2313 
2314  for (Argument &A : F.args())
2315  if (isa<PointerType>(A.getType()))
2316  RemoveNonValidAttrAtIndex(Ctx, F, A.getArgNo() + 1);
2317 
2318  if (isa<PointerType>(F.getReturnType()))
2320 }
2321 
2322 void RewriteStatepointsForGC::stripNonValidAttributesFromBody(Function &F) {
2323  if (F.empty())
2324  return;
2325 
2326  LLVMContext &Ctx = F.getContext();
2327  MDBuilder Builder(Ctx);
2328 
2329  for (Instruction &I : instructions(F)) {
2330  if (const MDNode *MD = I.getMetadata(LLVMContext::MD_tbaa)) {
2331  assert(MD->getNumOperands() < 5 && "unrecognized metadata shape!");
2332  bool IsImmutableTBAA =
2333  MD->getNumOperands() == 4 &&
2334  mdconst::extract<ConstantInt>(MD->getOperand(3))->getValue() == 1;
2335 
2336  if (!IsImmutableTBAA)
2337  continue; // no work to do, MD_tbaa is already marked mutable
2338 
2339  MDNode *Base = cast<MDNode>(MD->getOperand(0));
2340  MDNode *Access = cast<MDNode>(MD->getOperand(1));
2341  uint64_t Offset =
2342  mdconst::extract<ConstantInt>(MD->getOperand(2))->getZExtValue();
2343 
2344  MDNode *MutableTBAA =
2345  Builder.createTBAAStructTagNode(Base, Access, Offset);
2346  I.setMetadata(LLVMContext::MD_tbaa, MutableTBAA);
2347  }
2348 
2349  if (CallSite CS = CallSite(&I)) {
2350  for (int i = 0, e = CS.arg_size(); i != e; i++)
2351  if (isa<PointerType>(CS.getArgument(i)->getType()))
2352  RemoveNonValidAttrAtIndex(Ctx, CS, i + 1);
2353  if (isa<PointerType>(CS.getType()))
2355  }
2356  }
2357 }
2358 
2359 /// Returns true if this function should be rewritten by this pass. The main
2360 /// point of this function is as an extension point for custom logic.
2362  // TODO: This should check the GCStrategy
2363  if (F.hasGC()) {
2364  const auto &FunctionGCName = F.getGC();
2365  const StringRef StatepointExampleName("statepoint-example");
2366  const StringRef CoreCLRName("coreclr");
2367  return (StatepointExampleName == FunctionGCName) ||
2368  (CoreCLRName == FunctionGCName);
2369  } else
2370  return false;
2371 }
2372 
2373 void RewriteStatepointsForGC::stripNonValidAttributes(Module &M) {
2374 #ifndef NDEBUG
2375  assert(any_of(M, shouldRewriteStatepointsIn) && "precondition!");
2376 #endif
2377 
2378  for (Function &F : M)
2379  stripNonValidAttributesFromPrototype(F);
2380 
2381  for (Function &F : M)
2382  stripNonValidAttributesFromBody(F);
2383 }
2384 
2385 bool RewriteStatepointsForGC::runOnFunction(Function &F) {
2386  // Nothing to do for declarations.
2387  if (F.isDeclaration() || F.empty())
2388  return false;
2389 
2390  // Policy choice says not to rewrite - the most common reason is that we're
2391  // compiling code without a GCStrategy.
2393  return false;
2394 
2395  DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(F).getDomTree();
2396  TargetTransformInfo &TTI =
2397  getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2398 
2399  auto NeedsRewrite = [](Instruction &I) {
2401  return !callsGCLeafFunction(CS) && !isStatepoint(CS);
2402  return false;
2403  };
2404 
2405  // Gather all the statepoints which need rewritten. Be careful to only
2406  // consider those in reachable code since we need to ask dominance queries
2407  // when rewriting. We'll delete the unreachable ones in a moment.
2408  SmallVector<CallSite, 64> ParsePointNeeded;
2409  bool HasUnreachableStatepoint = false;
2410  for (Instruction &I : instructions(F)) {
2411  // TODO: only the ones with the flag set!
2412  if (NeedsRewrite(I)) {
2413  if (DT.isReachableFromEntry(I.getParent()))
2414  ParsePointNeeded.push_back(CallSite(&I));
2415  else
2416  HasUnreachableStatepoint = true;
2417  }
2418  }
2419 
2420  bool MadeChange = false;
2421 
2422  // Delete any unreachable statepoints so that we don't have unrewritten
2423  // statepoints surviving this pass. This makes testing easier and the
2424  // resulting IR less confusing to human readers. Rather than be fancy, we
2425  // just reuse a utility function which removes the unreachable blocks.
2426  if (HasUnreachableStatepoint)
2427  MadeChange |= removeUnreachableBlocks(F);
2428 
2429  // Return early if no work to do.
2430  if (ParsePointNeeded.empty())
2431  return MadeChange;
2432 
2433  // As a prepass, go ahead and aggressively destroy single entry phi nodes.
2434  // These are created by LCSSA. They have the effect of increasing the size
2435  // of liveness sets for no good reason. It may be harder to do this post
2436  // insertion since relocations and base phis can confuse things.
2437  for (BasicBlock &BB : F)
2438  if (BB.getUniquePredecessor()) {
2439  MadeChange = true;
2441  }
2442 
2443  // Before we start introducing relocations, we want to tweak the IR a bit to
2444  // avoid unfortunate code generation effects. The main example is that we
2445  // want to try to make sure the comparison feeding a branch is after any
2446  // safepoints. Otherwise, we end up with a comparison of pre-relocation
2447  // values feeding a branch after relocation. This is semantically correct,
2448  // but results in extra register pressure since both the pre-relocation and
2449  // post-relocation copies must be available in registers. For code without
2450  // relocations this is handled elsewhere, but teaching the scheduler to
2451  // reverse the transform we're about to do would be slightly complex.
2452  // Note: This may extend the live range of the inputs to the icmp and thus
2453  // increase the liveset of any statepoint we move over. This is profitable
2454  // as long as all statepoints are in rare blocks. If we had in-register
2455  // lowering for live values this would be a much safer transform.
2456  auto getConditionInst = [](TerminatorInst *TI) -> Instruction* {
2457  if (auto *BI = dyn_cast<BranchInst>(TI))
2458  if (BI->isConditional())
2459  return dyn_cast<Instruction>(BI->getCondition());
2460  // TODO: Extend this to handle switches
2461  return nullptr;
2462  };
2463  for (BasicBlock &BB : F) {
2464  TerminatorInst *TI = BB.getTerminator();
2465  if (auto *Cond = getConditionInst(TI))
2466  // TODO: Handle more than just ICmps here. We should be able to move
2467  // most instructions without side effects or memory access.
2468  if (isa<ICmpInst>(Cond) && Cond->hasOneUse()) {
2469  MadeChange = true;
2470  Cond->moveBefore(TI);
2471  }
2472  }
2473 
2474  MadeChange |= insertParsePoints(F, DT, TTI, ParsePointNeeded);
2475  return MadeChange;
2476 }
2477 
2478 // liveness computation via standard dataflow
2479 // -------------------------------------------------------------------
2480 
2481 // TODO: Consider using bitvectors for liveness, the set of potentially
2482 // interesting values should be small and easy to pre-compute.
2483 
2484 /// Compute the live-in set for the location rbegin starting from
2485 /// the live-out set of the basic block
2488  SetVector<Value *> &LiveTmp) {
2489  for (auto &I : make_range(Begin, End)) {
2490  // KILL/Def - Remove this definition from LiveIn
2491  LiveTmp.remove(&I);
2492 
2493  // Don't consider *uses* in PHI nodes, we handle their contribution to
2494  // predecessor blocks when we seed the LiveOut sets
2495  if (isa<PHINode>(I))
2496  continue;
2497 
2498  // USE - Add to the LiveIn set for this instruction
2499  for (Value *V : I.operands()) {
2500  assert(!isUnhandledGCPointerType(V->getType()) &&
2501  "support for FCA unimplemented");
2502  if (isHandledGCPointerType(V->getType()) && !isa<Constant>(V)) {
2503  // The choice to exclude all things constant here is slightly subtle.
2504  // There are two independent reasons:
2505  // - We assume that things which are constant (from LLVM's definition)
2506  // do not move at runtime. For example, the address of a global
2507  // variable is fixed, even though it's contents may not be.
2508  // - Second, we can't disallow arbitrary inttoptr constants even
2509  // if the language frontend does. Optimization passes are free to
2510  // locally exploit facts without respect to global reachability. This
2511  // can create sections of code which are dynamically unreachable and
2512  // contain just about anything. (see constants.ll in tests)
2513  LiveTmp.insert(V);
2514  }
2515  }
2516  }
2517 }
2518 
2520  for (BasicBlock *Succ : successors(BB)) {
2521  for (auto &I : *Succ) {
2522  PHINode *PN = dyn_cast<PHINode>(&I);
2523  if (!PN)
2524  break;
2525 
2526  Value *V = PN->getIncomingValueForBlock(BB);
2528  "support for FCA unimplemented");
2529  if (isHandledGCPointerType(V->getType()) && !isa<Constant>(V))
2530  LiveTmp.insert(V);
2531  }
2532  }
2533 }
2534 
2536  SetVector<Value *> KillSet;
2537  for (Instruction &I : *BB)
2538  if (isHandledGCPointerType(I.getType()))
2539  KillSet.insert(&I);
2540  return KillSet;
2541 }
2542 
2543 #ifndef NDEBUG
2544 /// Check that the items in 'Live' dominate 'TI'. This is used as a basic
2545 /// sanity check for the liveness computation.
2547  TerminatorInst *TI, bool TermOkay = false) {
2548  for (Value *V : Live) {
2549  if (auto *I = dyn_cast<Instruction>(V)) {
2550  // The terminator can be a member of the LiveOut set. LLVM's definition
2551  // of instruction dominance states that V does not dominate itself. As
2552  // such, we need to special case this to allow it.
2553  if (TermOkay && TI == I)
2554  continue;
2555  assert(DT.dominates(I, TI) &&
2556  "basic SSA liveness expectation violated by liveness analysis");
2557  }
2558  }
2559 }
2560 
2561 /// Check that all the liveness sets used during the computation of liveness
2562 /// obey basic SSA properties. This is useful for finding cases where we miss
2563 /// a def.
2564 static void checkBasicSSA(DominatorTree &DT, GCPtrLivenessData &Data,
2565  BasicBlock &BB) {
2566  checkBasicSSA(DT, Data.LiveSet[&BB], BB.getTerminator());
2567  checkBasicSSA(DT, Data.LiveOut[&BB], BB.getTerminator(), true);
2568  checkBasicSSA(DT, Data.LiveIn[&BB], BB.getTerminator());
2569 }
2570 #endif
2571 
2573  GCPtrLivenessData &Data) {
2575 
2576  // Seed the liveness for each individual block
2577  for (BasicBlock &BB : F) {
2578  Data.KillSet[&BB] = computeKillSet(&BB);
2579  Data.LiveSet[&BB].clear();
2580  computeLiveInValues(BB.rbegin(), BB.rend(), Data.LiveSet[&BB]);
2581 
2582 #ifndef NDEBUG
2583  for (Value *Kill : Data.KillSet[&BB])
2584  assert(!Data.LiveSet[&BB].count(Kill) && "live set contains kill");
2585 #endif
2586 
2587  Data.LiveOut[&BB] = SetVector<Value *>();
2588  computeLiveOutSeed(&BB, Data.LiveOut[&BB]);
2589  Data.LiveIn[&BB] = Data.LiveSet[&BB];
2590  Data.LiveIn[&BB].set_union(Data.LiveOut[&BB]);
2591  Data.LiveIn[&BB].set_subtract(Data.KillSet[&BB]);
2592  if (!Data.LiveIn[&BB].empty())
2593  Worklist.insert(pred_begin(&BB), pred_end(&BB));
2594  }
2595 
2596  // Propagate that liveness until stable
2597  while (!Worklist.empty()) {
2598  BasicBlock *BB = Worklist.pop_back_val();
2599 
2600  // Compute our new liveout set, then exit early if it hasn't changed despite
2601  // the contribution of our successor.
2602  SetVector<Value *> LiveOut = Data.LiveOut[BB];
2603  const auto OldLiveOutSize = LiveOut.size();
2604  for (BasicBlock *Succ : successors(BB)) {
2605  assert(Data.LiveIn.count(Succ));
2606  LiveOut.set_union(Data.LiveIn[Succ]);
2607  }
2608  // assert OutLiveOut is a subset of LiveOut
2609  if (OldLiveOutSize == LiveOut.size()) {
2610  // If the sets are the same size, then we didn't actually add anything
2611  // when unioning our successors LiveIn. Thus, the LiveIn of this block
2612  // hasn't changed.
2613  continue;
2614  }
2615  Data.LiveOut[BB] = LiveOut;
2616 
2617  // Apply the effects of this basic block
2618  SetVector<Value *> LiveTmp = LiveOut;
2619  LiveTmp.set_union(Data.LiveSet[BB]);
2620  LiveTmp.set_subtract(Data.KillSet[BB]);
2621 
2622  assert(Data.LiveIn.count(BB));
2623  const SetVector<Value *> &OldLiveIn = Data.LiveIn[BB];
2624  // assert: OldLiveIn is a subset of LiveTmp
2625  if (OldLiveIn.size() != LiveTmp.size()) {
2626  Data.LiveIn[BB] = LiveTmp;
2627  Worklist.insert(pred_begin(BB), pred_end(BB));
2628  }
2629  } // while (!Worklist.empty())
2630 
2631 #ifndef NDEBUG
2632  // Sanity check our output against SSA properties. This helps catch any
2633  // missing kills during the above iteration.
2634  for (BasicBlock &BB : F)
2635  checkBasicSSA(DT, Data, BB);
2636 #endif
2637 }
2638 
2639 static void findLiveSetAtInst(Instruction *Inst, GCPtrLivenessData &Data,
2640  StatepointLiveSetTy &Out) {
2641 
2642  BasicBlock *BB = Inst->getParent();
2643 
2644  // Note: The copy is intentional and required
2645  assert(Data.LiveOut.count(BB));
2646  SetVector<Value *> LiveOut = Data.LiveOut[BB];
2647 
2648  // We want to handle the statepoint itself oddly. It's
2649  // call result is not live (normal), nor are it's arguments
2650  // (unless they're used again later). This adjustment is
2651  // specifically what we need to relocate
2652  computeLiveInValues(BB->rbegin(), ++Inst->getIterator().getReverse(),
2653  LiveOut);
2654  LiveOut.remove(Inst);
2655  Out.insert(LiveOut.begin(), LiveOut.end());
2656 }
2657 
2658 static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
2659  CallSite CS,
2660  PartiallyConstructedSafepointRecord &Info) {
2661  Instruction *Inst = CS.getInstruction();
2662  StatepointLiveSetTy Updated;
2663  findLiveSetAtInst(Inst, RevisedLivenessData, Updated);
2664 
2665 #ifndef NDEBUG
2666  DenseSet<Value *> Bases;
2667  for (auto KVPair : Info.PointerToBase)
2668  Bases.insert(KVPair.second);
2669 #endif
2670 
2671  // We may have base pointers which are now live that weren't before. We need
2672  // to update the PointerToBase structure to reflect this.
2673  for (auto V : Updated)
2674  if (Info.PointerToBase.insert({V, V}).second) {
2675  assert(Bases.count(V) && "Can't find base for unexpected live value!");
2676  continue;
2677  }
2678 
2679 #ifndef NDEBUG
2680  for (auto V : Updated)
2681  assert(Info.PointerToBase.count(V) &&
2682  "Must be able to find base for live value!");
2683 #endif
2684 
2685  // Remove any stale base mappings - this can happen since our liveness is
2686  // more precise then the one inherent in the base pointer analysis.
2687  DenseSet<Value *> ToErase;
2688  for (auto KVPair : Info.PointerToBase)
2689  if (!Updated.count(KVPair.first))
2690  ToErase.insert(KVPair.first);
2691 
2692  for (auto *V : ToErase)
2693  Info.PointerToBase.erase(V);
2694 
2695 #ifndef NDEBUG
2696  for (auto KVPair : Info.PointerToBase)
2697  assert(Updated.count(KVPair.first) && "record for non-live value");
2698 #endif
2699 
2700  Info.LiveSet = Updated;
2701 }
MachineLoop * L
AttributeSet getAttributes() const
Return the parameter attributes for this call.
IterTy arg_end() const
Definition: CallSite.h:532
const NoneType None
Definition: None.h:23
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:76
static void findLiveReferences(Function &F, DominatorTree &DT, ArrayRef< CallSite > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records)
static void unique_unsorted(SmallVectorImpl< T > &Vec)
Implement a unique function which doesn't require we sort the input vector.
static bool isHandledGCPointerType(Type *T)
BasicBlock * getUniquePredecessor()
Return the predecessor of this block if it has a unique predecessor block.
Definition: BasicBlock.cpp:239
static cl::opt< bool, true > ClobberNonLiveOverride("rs4gc-clobber-non-live", cl::location(ClobberNonLive), cl::Hidden)
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:226
LLVM Argument representation.
Definition: Argument.h:34
uint64_t Token
static Value * findBaseDefiningValueCached(Value *I, DefiningValueMapTy &Cache)
Returns the base defining value for this value.
static bool AreEquivalentPhiNodes(PHINode &OrigRootPhi, PHINode &AlternateRootPhi)
static BDVState meetBDVStateImpl(const BDVState &LHS, const BDVState &RHS)
static bool insertParsePoints(Function &F, DominatorTree &DT, TargetTransformInfo &TTI, SmallVectorImpl< CallSite > &ToUpdate)
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:274
bool hasName() const
Definition: Value.h:236
bool hasValue() const
Definition: Optional.h:125
size_t i
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
Definition: Compiler.h:450
Instruction * StatepointToken
The new gc.statepoint instruction itself.
AttributeSet getAttributes() const
Return the parameter attributes for this invoke.
Instruction * UnwindToken
Instruction to which exceptional gc relocates are attached Makes it easier to iterate through them du...
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:52
auto remove_if(R &&Range, UnaryPredicate P) -> decltype(std::begin(Range))
Provide wrappers to std::remove_if which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:776
static ConstantAggregateZero * get(Type *Ty)
Definition: Constants.cpp:1254
iterator end() const
Definition: ArrayRef.h:130
static void rematerializeLiveValues(CallSite CS, PartiallyConstructedSafepointRecord &Info, TargetTransformInfo &TTI)
static void computeLiveOutSeed(BasicBlock *BB, SetVector< Value * > &LiveTmp)
static StringRef getDeoptLowering(CallSite CS)
ilist_iterator< OptionsT,!IsReverse, IsConst > getReverse() const
Get a reverse iterator to the same node.
This class represents a function call, abstracting a target machine's calling convention.
MapVector< BasicBlock *, SetVector< Value * > > KillSet
Values defined in this block.
unsigned gcArgsStartIdx() const
Definition: Statepoint.h:252
InvokeInst * CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > InvokeArgs, ArrayRef< Value * > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
brief Create an invoke to the experimental.gc.statepoint intrinsic to start a new statepoint sequence...
Definition: IRBuilder.cpp:427
This instruction constructs a fixed permutation of two input vectors.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
MapVector< BasicBlock *, SetVector< Value * > > LiveSet
Values used in this block (and thus live); does not included values killed within this block...
iterator begin(unsigned Slot) const
MapVector< BasicBlock *, SetVector< Value * > > LiveIn
Values live into this basic block (i.e.
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition: Type.h:216
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.cpp:238
This class implements a map that also provides access to all stored values in a deterministic order...
Definition: MapVector.h:32
Metadata node.
Definition: Metadata.h:830
The two locations do not alias at all.
Definition: AliasAnalysis.h:79
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:234
An instruction for reading from memory.
Definition: Instructions.h:164
reverse_iterator rbegin()
Definition: BasicBlock.h:233
static BDVState meetBDVState(BDVState LHS, BDVState RHS)
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
Hexagon Common GEP
static BaseDefiningValueResult findBaseDefiningValueOfVector(Value *I)
Return a base defining value for the 'Index' element of the given vector instruction 'I'...
void reserve(size_type N)
Definition: SmallVector.h:377
bool hasAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return true if the attribute exists at the given index.
Definition: Attributes.cpp:994
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:78
static void computeLiveInValues(DominatorTree &DT, Function &F, GCPtrLivenessData &Data)
Compute the live-in set for every basic block in the function.
AttributeSet getRetAttributes() const
The attributes for the ret value are returned.
Definition: Attributes.cpp:976
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:228
void setCallingConv(CallingConv::ID CC)
static void relocationViaAlloca(Function &F, DominatorTree &DT, ArrayRef< Value * > Live, ArrayRef< PartiallyConstructedSafepointRecord > Records)
Do all the relocation update via allocas and mem2reg.
lazy value info
Value * getDerivedPtr() const
Definition: Statepoint.h:394
AnalysisUsage & addRequired()
static InsertElementInst * Create(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
rewrite statepoints for Make relocations at false
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:53
static unsigned chainToBasePointerCost(SmallVectorImpl< Instruction * > &Chain, TargetTransformInfo &TTI)
bool hasGC() const
hasGC/getGC/setGC/clearGC - The name of the garbage collection algorithm to use during code generatio...
Definition: Function.h:250
This class represents the LLVM 'select' instruction.
struct fuzzer::@269 Flags
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:578
'undef' values are things that do not have specified contents.
Definition: Constants.h:1258
static bool isKnownBaseResult(Value *V)
Given the result of a call to findBaseDefiningValue, or findBaseOrBDV, is it known to be a base point...
Class to represent struct types.
Definition: DerivedTypes.h:199
static void CreateGCRelocates(ArrayRef< Value * > LiveVariables, const int LiveStart, ArrayRef< Value * > BasePtrs, Instruction *StatepointToken, IRBuilder<> Builder)
Helper function to place all gc relocates necessary for the given statepoint.
static Value * findBasePointer(Value *I, DefiningValueMapTy &Cache)
For a given value or instruction, figure out what base ptr its derived from.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
ValTy * getCalledValue() const
getCalledValue - Return the pointer to function that is being called.
Definition: CallSite.h:102
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:180
static bool isGCPointerType(Type *T)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:588
bool isCall() const
isCall - true if a CallInst is enclosed.
Definition: CallSite.h:87
static void makeStatepointExplicit(DominatorTree &DT, CallSite CS, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements)
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool equals(StringRef RHS) const
equals - Check for string equality, this is more efficient than compare() when the relative ordering ...
Definition: StringRef.h:166
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:257
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following: ...
bool remove(const value_type &X)
Remove an item from the set vector.
Definition: SetVector.h:152
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:60
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:241
static cl::opt< bool > PrintLiveSet("spp-print-liveset", cl::Hidden, cl::init(false))
void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AliasSetTracker *AST=nullptr, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
#define F(x, y, z)
Definition: MD5.cpp:51
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:136
static void insertUseHolderAfter(CallSite &CS, const ArrayRef< Value * > Values, SmallVectorImpl< CallInst * > &Holders)
Insert holders so that each Value is obviously live through the entire lifetime of the call...
const T & getValue() const LLVM_LVALUE_FUNCTION
Definition: Optional.h:121
Class to represent array types.
Definition: DerivedTypes.h:345
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Definition: Dominators.cpp:269
This class represents a no-op cast from one type to another.
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:73
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:291
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:949
An instruction for storing to memory.
Definition: Instructions.h:300
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Definition: IRBuilder.h:151
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:401
static Value * findRematerializableChainToBasePointer(SmallVectorImpl< Instruction * > &ChainToBase, Value *CurrentValue)
StatepointLiveSetTy LiveSet
The set of values known to be live across this safepoint.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:96
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:141
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Definition: IRBuilder.h:127
BasicBlock * getNormalDest() const
void setAttributes(AttributeSet Attrs)
Set the parameter attributes for this call.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
void replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
Definition: User.cpp:24
static SetVector< Value * > computeKillSet(BasicBlock *BB)
bool isInvoke() const
isInvoke - true if a InvokeInst is enclosed.
Definition: CallSite.h:91
CallSiteTy::arg_iterator gc_args_begin() const
Definition: Statepoint.h:245
an instruction for type-safe pointer arithmetic to access elements of arrays and structs ...
Definition: Instructions.h:830
unsigned getNumSlots() const
Return the number of slots used in this attribute list.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:395
static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH, unsigned Index)
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Definition: CallSite.h:512
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
Subclasses of this class are all able to terminate a basic block.
Definition: InstrTypes.h:52
Wrapper pass for TargetTransformInfo.
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1323
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:283
static ArrayRef< Use > GetDeoptBundleOperands(ImmutableCallSite CS)
Constant * getOrInsertFunction(StringRef Name, FunctionType *T, AttributeSet AttributeList)
Look up the specified function in the module symbol table.
Definition: Module.cpp:123
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
Definition: Instruction.cpp:82
MapVector< Value *, Value * > DefiningValueMapTy
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:48
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:219
This function has undefined behavior.
This is an important base class in LLVM.
Definition: Constant.h:42
bool set_union(const STy &S)
Compute This := This u S, return whether 'This' changed.
Definition: SetVector.h:240
static void analyzeParsePointLiveness(DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallSite CS, PartiallyConstructedSafepointRecord &Result)
ArrayRef< Use > Inputs
Definition: InstrTypes.h:1207
CallSiteTy::arg_iterator gc_args_end() const
Definition: Statepoint.h:248
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
Definition: SmallVector.h:115
static cl::opt< unsigned > RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden, cl::init(6))
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:36
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:168
LandingPadInst * getLandingPadInst()
Return the landingpad instruction associated with the landing pad.
Definition: BasicBlock.cpp:441
iterator_range< arg_iterator > gc_args() const
range adapter for gc arguments
Definition: Statepoint.h:257
CallInst * CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, ArrayRef< Value * > CallArgs, ArrayRef< Value * > DeoptArgs, ArrayRef< Value * > GCArgs, const Twine &Name="")
Create a call to the experimental.gc.statepoint intrinsic to start a new statepoint sequence...
Definition: IRBuilder.cpp:377
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:116
Optional< uint32_t > NumPatchBytes
Definition: Statepoint.h:448
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:259
Represent the analysis usage information of a pass.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:154
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
static void insertRelocationStores(iterator_range< Value::user_iterator > GCRelocs, DenseMap< Value *, Value * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)
bool any_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:743
uint32_t Offset
RematerializedValueMapTy RematerializedValues
Record live values we are rematerialized instead of relocating.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
static const unsigned End
iterator begin() const
Definition: ArrayRef.h:129
for(unsigned i=0, e=MI->getNumOperands();i!=e;++i)
A specialization of it's base class for read-write access to a gc.statepoint.
Definition: Statepoint.h:313
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if this function has the given attribute.
Definition: CallSite.h:349
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:119
self_iterator getIterator()
Definition: ilist_node.h:81
SetVector< Value * > StatepointLiveSetTy
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:80
bool callsGCLeafFunction(ImmutableCallSite CS)
Return true if the CallSite CS calls a gc leaf function.
Definition: Local.cpp:1798
rewrite statepoints for gc
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:136
void setTailCallKind(TailCallKind TCK)
int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) const
static bool containsGCPtrType(Type *Ty)
Returns true if this type contains a gc pointer whether we know how to handle that type or not...
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:225
Type * getType() const
getType - Return the type of the instruction that generated this call site
Definition: CallSite.h:258
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1337
iterator erase(const_iterator CI)
Definition: SmallVector.h:431
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeSet AS)
Parse out statepoint directives from the function attributes present in AS.
Definition: Statepoint.cpp:56
static void findLiveSetAtInst(Instruction *inst, GCPtrLivenessData &Data, StatepointLiveSetTy &out)
Given results from the dataflow liveness computation, find the set of live Values at a particular ins...
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:654
INITIALIZE_PASS_BEGIN(RewriteStatepointsForGC,"rewrite-statepoints-for-gc","Make relocations explicit at statepoints", false, false) INITIALIZE_PASS_END(RewriteStatepointsForGC
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:213
const Value * getTrueValue() const
static Value * findBaseOrBDV(Value *I, DefiningValueMapTy &Cache)
Return a base pointer for this value if known.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1183
static bool shouldRewriteStatepointsIn(Function &F)
Returns true if this function should be rewritten by this pass.
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - Get or set the calling convention of this function call...
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
Definition: Dominators.cpp:218
AttributeSet getAttributes() const
getAttributes/setAttributes - get or set the parameter attributes of the call.
Definition: CallSite.h:325
IterTy arg_begin() const
Definition: CallSite.h:528
static void findBasePointers(const StatepointLiveSetTy &live, MapVector< Value *, Value * > &PointerToBase, DominatorTree *DT, DefiningValueMapTy &DVCache)
LLVMContext & getContext() const
Retrieve the LLVM context.
Definition: Attributes.cpp:964
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: MapVector.h:101
static cl::opt< bool > PrintBasePointers("spp-print-base-pointers", cl::Hidden, cl::init(false))
static cl::opt< bool > AllowStatepointWithNoDeoptInfo("rs4gc-allow-statepoint-with-no-deopt-info", cl::Hidden, cl::init(true))
rewrite statepoints for Make relocations at statepoints
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:292
Iterator for intrusive lists based on ilist_node.
BasicBlock * getUnwindDest() const
MapVector< AssertingVH< Instruction >, AssertingVH< Value > > RematerializedValueMapTy
auto find(R &&Range, const T &Val) -> decltype(std::begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:757
InstrTy * getInstruction() const
Definition: CallSite.h:93
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
bool removeUnreachableBlocks(Function &F, LazyValueInfo *LVI=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:1648
static AttributeSet legalizeCallAttributes(AttributeSet AS)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:58
static CallInst * Create(Value *Func, ArrayRef< Value * > Args, ArrayRef< OperandBundleDef > Bundles=None, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
void initializeRewriteStatepointsForGCPass(PassRegistry &)
iterator end(unsigned Slot) const
TailCallKind getTailCallKind() const
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
bool empty() const
Return true if the builder contains no target-independent attributes.
Definition: Attributes.h:559
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:382
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:307
Value * stripPointerCasts()
Strip off pointer casts, all-zero GEPs, and aliases.
Definition: Value.cpp:490
static BasicBlock * normalizeForInvokeSafepoint(BasicBlock *BB, BasicBlock *InvokeParent, DominatorTree &DT)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
const BasicBlock & getEntryBlock() const
Definition: Function.h:519
void set_subtract(const STy &S)
Compute This := This - B TODO: We should be able to use set_subtract from SetOperations.h, but SetVector interface is inconsistent with DenseSet.
Definition: SetVector.h:255
Value handle that asserts if the Value is deleted.
Definition: ValueHandle.h:182
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:122
static bool ClobberNonLive
static void checkBasicSSA(DominatorTree &DT, SetVector< Value * > &Live, TerminatorInst *TI, bool TermOkay=false)
Check that the items in 'Live' dominate 'TI'.
A range adaptor for a pair of iterators.
Class to represent vector types.
Definition: DerivedTypes.h:369
static std::string suffixed_name_or(Value *V, StringRef Suffix, StringRef DefaultName)
bool isStatepointDirectiveAttr(Attribute Attr)
Return true if the the Attr is an attribute that is a statepoint directive.
Definition: Statepoint.cpp:51
Optional< uint64_t > StatepointID
Definition: Statepoint.h:449
Value * getIncomingValueForBlock(const BasicBlock *BB) const
iterator_range< user_iterator > users()
Definition: Value.h:370
static const uint64_t DefaultStatepointID
Definition: Statepoint.h:451
bool empty() const
Definition: Function.h:541
void FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceResults *MemDep=nullptr)
We know that BB has one predecessor.
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:464
Call sites that get wrapped by a gc.statepoint (currently only in RewriteStatepointsForGC and potenti...
Definition: Statepoint.h:447
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1132
bool operator!=(uint64_t V1, const APInt &V2)
Definition: APInt.h:1724
static void insertRematerializationStores(const RematerializedValueMapTy &RematerializedValues, DenseMap< Value *, Value * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Definition: SmallVector.h:119
static cl::opt< bool > PrintLiveSetSize("spp-print-liveset-size", cl::Hidden, cl::init(false))
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:226
size_type count(const ValueT &V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:81
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:188
unsigned getSlotIndex(unsigned Slot) const
Return the index for the given slot.
static bool isUnhandledGCPointerType(Type *Ty)
ImmutableCallSite - establish a view to a call site for examination.
Definition: CallSite.h:665
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction...
Definition: Instruction.cpp:88
const std::string & getGC() const
Definition: Function.cpp:412
#define I(x, y, z)
Definition: MD5.cpp:54
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
CallInst * CreateGCResult(Instruction *Statepoint, Type *ResultType, const Twine &Name="")
Create a call to the experimental.gc.result intrinsic to extract the result from a call wrapped in a ...
Definition: IRBuilder.cpp:458
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:135
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:235
CallInst * CreateCall(Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1579
iterator_range< value_op_iterator > operand_values()
Definition: User.h:237
static void makeStatepointExplicitImpl(const CallSite CS, const SmallVectorImpl< Value * > &BasePtrs, const SmallVectorImpl< Value * > &LiveVariables, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:64
raw_ostream & operator<<(raw_ostream &OS, const APInt &I)
Definition: APInt.h:1726
static Function * getCalledFunction(const Value *V, bool LookThroughBitCast, bool &IsNoBuiltin)
Attribute getAttribute(unsigned Index, Attribute::AttrKind Kind) const
Return the attribute object that exists at the given index.
BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
bool isStatepoint(ImmutableCallSite CS)
Definition: Statepoint.cpp:27
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:178
bool use_empty() const
Definition: Value.h:299
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
user_iterator user_begin()
Definition: Value.h:346
FunTy * getCalledFunction() const
getCalledFunction - Return the function being called if this is a direct call, otherwise return null ...
Definition: CallSite.h:110
Represents calls to the gc.relocate intrinsic.
Definition: Statepoint.h:367
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
Definition: Value.h:71
succ_range successors(BasicBlock *BB)
Definition: IR/CFG.h:143
ModulePass * createRewriteStatepointsForGCPass()
static VectorType * get(Type *ElementType, unsigned NumElements)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:631
static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, CallSite CS, PartiallyConstructedSafepointRecord &result)
Given an updated version of the dataflow liveness results, update the liveset and base pointer maps f...
static const Function * getParent(const Value *V)
void setCallingConv(CallingConv::ID CC)
MapVector< BasicBlock *, SetVector< Value * > > LiveOut
Values live out of this basic block (i.e.
This class implements an extremely fast bulk output stream that can only output to a stream...
Definition: raw_ostream.h:44
Invoke instruction.
#define DEBUG(X)
Definition: Debug.h:100
const Value * getFalseValue() const
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
inst_range instructions(Function *F)
Definition: InstIterator.h:132
CallingConv::ID getCallingConv() const
getCallingConv/setCallingConv - Get or set the calling convention of this function call...
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:217
This pass exposes codegen information to IR-level passes.
bool operator==(uint64_t V1, const APInt &V2)
Definition: APInt.h:1722
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:209
MapVector< Value *, Value * > PointerToBase
Mapping from live pointers to a base-defining-value.
int * Ptr
void setIncomingValue(unsigned i, Value *V)
static ExtractElementInst * Create(Value *Vec, Value *Idx, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
AttributeSet addAttributes(LLVMContext &C, unsigned Index, AttributeSet Attrs) const
Add attributes to the attribute set at the given index.
Definition: Attributes.cpp:796
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
int getAddressComputationCost(Type *Ty, ScalarEvolution *SE=nullptr, const SCEV *Ptr=nullptr) const
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:411
const BasicBlock * getParent() const
Definition: Instruction.h:62
iterator_range< arg_iterator > args()
Definition: Function.h:568
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:44
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:222
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
an instruction to allocate memory on the stack
Definition: Instructions.h:60
static BaseDefiningValueResult findBaseDefiningValue(Value *I)
Helper function for findBasePointer - Will return a value which either a) defines the base pointer fo...
AttributeSet getFnAttributes() const
The function attributes are returned.
Definition: Attributes.cpp:985
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
Definition: STLExtras.h:783
user_iterator user_end()
Definition: Value.h:354