File: | lib/Transforms/Scalar/RewriteStatepointsForGC.cpp |
Warning: | line 1003, column 9 Called C++ object pointer is null |
1 | //===- RewriteStatepointsForGC.cpp - Make GC relocations explicit ---------===// |
2 | // |
3 | // The LLVM Compiler Infrastructure |
4 | // |
5 | // This file is distributed under the University of Illinois Open Source |
6 | // License. See LICENSE.TXT for details. |
7 | // |
8 | //===----------------------------------------------------------------------===// |
9 | // |
10 | // Rewrite call/invoke instructions so as to make potential relocations |
11 | // performed by the garbage collector explicit in the IR. |
12 | // |
13 | //===----------------------------------------------------------------------===// |
14 | |
15 | #include "llvm/ADT/DenseSet.h" |
16 | #include "llvm/ADT/MapVector.h" |
17 | #include "llvm/ADT/SetOperations.h" |
18 | #include "llvm/ADT/SetVector.h" |
19 | #include "llvm/ADT/Statistic.h" |
20 | #include "llvm/ADT/StringRef.h" |
21 | #include "llvm/Analysis/CFG.h" |
22 | #include "llvm/Analysis/TargetTransformInfo.h" |
23 | #include "llvm/IR/BasicBlock.h" |
24 | #include "llvm/IR/CallSite.h" |
25 | #include "llvm/IR/Dominators.h" |
26 | #include "llvm/IR/Function.h" |
27 | #include "llvm/IR/IRBuilder.h" |
28 | #include "llvm/IR/InstIterator.h" |
29 | #include "llvm/IR/Instructions.h" |
30 | #include "llvm/IR/IntrinsicInst.h" |
31 | #include "llvm/IR/Intrinsics.h" |
32 | #include "llvm/IR/MDBuilder.h" |
33 | #include "llvm/IR/Module.h" |
34 | #include "llvm/IR/Statepoint.h" |
35 | #include "llvm/IR/Value.h" |
36 | #include "llvm/IR/Verifier.h" |
37 | #include "llvm/Pass.h" |
38 | #include "llvm/Support/CommandLine.h" |
39 | #include "llvm/Support/Debug.h" |
40 | #include "llvm/Transforms/Scalar.h" |
41 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
42 | #include "llvm/Transforms/Utils/Cloning.h" |
43 | #include "llvm/Transforms/Utils/Local.h" |
44 | #include "llvm/Transforms/Utils/PromoteMemToReg.h" |
45 | |
46 | #define DEBUG_TYPE"rewrite-statepoints-for-gc" "rewrite-statepoints-for-gc" |
47 | |
48 | using namespace llvm; |
49 | |
50 | // Print the liveset found at the insert location |
51 | static cl::opt<bool> PrintLiveSet("spp-print-liveset", cl::Hidden, |
52 | cl::init(false)); |
53 | static cl::opt<bool> PrintLiveSetSize("spp-print-liveset-size", cl::Hidden, |
54 | cl::init(false)); |
55 | // Print out the base pointers for debugging |
56 | static cl::opt<bool> PrintBasePointers("spp-print-base-pointers", cl::Hidden, |
57 | cl::init(false)); |
58 | |
59 | // Cost threshold measuring when it is profitable to rematerialize value instead |
60 | // of relocating it |
61 | static cl::opt<unsigned> |
62 | RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden, |
63 | cl::init(6)); |
64 | |
65 | #ifdef EXPENSIVE_CHECKS |
66 | static bool ClobberNonLive = true; |
67 | #else |
68 | static bool ClobberNonLive = false; |
69 | #endif |
70 | static cl::opt<bool, true> ClobberNonLiveOverride("rs4gc-clobber-non-live", |
71 | cl::location(ClobberNonLive), |
72 | cl::Hidden); |
73 | |
74 | static cl::opt<bool> |
75 | AllowStatepointWithNoDeoptInfo("rs4gc-allow-statepoint-with-no-deopt-info", |
76 | cl::Hidden, cl::init(true)); |
77 | |
78 | namespace { |
79 | struct RewriteStatepointsForGC : public ModulePass { |
80 | static char ID; // Pass identification, replacement for typeid |
81 | |
82 | RewriteStatepointsForGC() : ModulePass(ID) { |
83 | initializeRewriteStatepointsForGCPass(*PassRegistry::getPassRegistry()); |
84 | } |
85 | bool runOnFunction(Function &F); |
86 | bool runOnModule(Module &M) override { |
87 | bool Changed = false; |
88 | for (Function &F : M) |
89 | Changed |= runOnFunction(F); |
90 | |
91 | if (Changed) { |
92 | // stripNonValidAttributesAndMetadata asserts that shouldRewriteStatepointsIn |
93 | // returns true for at least one function in the module. Since at least |
94 | // one function changed, we know that the precondition is satisfied. |
95 | stripNonValidAttributesAndMetadata(M); |
96 | } |
97 | |
98 | return Changed; |
99 | } |
100 | |
101 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
102 | // We add and rewrite a bunch of instructions, but don't really do much |
103 | // else. We could in theory preserve a lot more analyses here. |
104 | AU.addRequired<DominatorTreeWrapperPass>(); |
105 | AU.addRequired<TargetTransformInfoWrapperPass>(); |
106 | } |
107 | |
108 | /// The IR fed into RewriteStatepointsForGC may have had attributes and |
109 | /// metadata implying dereferenceability that are no longer valid/correct after |
110 | /// RewriteStatepointsForGC has run. This is because semantically, after |
111 | /// RewriteStatepointsForGC runs, all calls to gc.statepoint "free" the entire |
112 | /// heap. stripNonValidAttributesAndMetadata (conservatively) restores |
113 | /// correctness by erasing all attributes in the module that externally imply |
114 | /// dereferenceability. Similar reasoning also applies to the noalias |
115 | /// attributes and metadata. gc.statepoint can touch the entire heap including |
116 | /// noalias objects. |
117 | void stripNonValidAttributesAndMetadata(Module &M); |
118 | |
119 | // Helpers for stripNonValidAttributesAndMetadata |
120 | void stripNonValidAttributesAndMetadataFromBody(Function &F); |
121 | void stripNonValidAttributesFromPrototype(Function &F); |
122 | // Certain metadata on instructions are invalid after running RS4GC. |
123 | // Optimizations that run after RS4GC can incorrectly use this metadata to |
124 | // optimize functions. We drop such metadata on the instruction. |
125 | void stripInvalidMetadataFromInstruction(Instruction &I); |
126 | }; |
127 | } // namespace |
128 | |
129 | char RewriteStatepointsForGC::ID = 0; |
130 | |
131 | ModulePass *llvm::createRewriteStatepointsForGCPass() { |
132 | return new RewriteStatepointsForGC(); |
133 | } |
134 | |
135 | INITIALIZE_PASS_BEGIN(RewriteStatepointsForGC, "rewrite-statepoints-for-gc",static void *initializeRewriteStatepointsForGCPassOnce(PassRegistry &Registry) { |
136 | "Make relocations explicit at statepoints", false, false)static void *initializeRewriteStatepointsForGCPassOnce(PassRegistry &Registry) { |
137 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); |
138 | INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry); |
139 | INITIALIZE_PASS_END(RewriteStatepointsForGC, "rewrite-statepoints-for-gc",PassInfo *PI = new PassInfo( "Make relocations explicit at statepoints" , "rewrite-statepoints-for-gc", &RewriteStatepointsForGC:: ID, PassInfo::NormalCtor_t(callDefaultCtor<RewriteStatepointsForGC >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeRewriteStatepointsForGCPassFlag ; void llvm::initializeRewriteStatepointsForGCPass(PassRegistry &Registry) { llvm::call_once(InitializeRewriteStatepointsForGCPassFlag , initializeRewriteStatepointsForGCPassOnce, std::ref(Registry )); } |
140 | "Make relocations explicit at statepoints", false, false)PassInfo *PI = new PassInfo( "Make relocations explicit at statepoints" , "rewrite-statepoints-for-gc", &RewriteStatepointsForGC:: ID, PassInfo::NormalCtor_t(callDefaultCtor<RewriteStatepointsForGC >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeRewriteStatepointsForGCPassFlag ; void llvm::initializeRewriteStatepointsForGCPass(PassRegistry &Registry) { llvm::call_once(InitializeRewriteStatepointsForGCPassFlag , initializeRewriteStatepointsForGCPassOnce, std::ref(Registry )); } |
141 | |
142 | namespace { |
143 | struct GCPtrLivenessData { |
144 | /// Values defined in this block. |
145 | MapVector<BasicBlock *, SetVector<Value *>> KillSet; |
146 | /// Values used in this block (and thus live); does not included values |
147 | /// killed within this block. |
148 | MapVector<BasicBlock *, SetVector<Value *>> LiveSet; |
149 | |
150 | /// Values live into this basic block (i.e. used by any |
151 | /// instruction in this basic block or ones reachable from here) |
152 | MapVector<BasicBlock *, SetVector<Value *>> LiveIn; |
153 | |
154 | /// Values live out of this basic block (i.e. live into |
155 | /// any successor block) |
156 | MapVector<BasicBlock *, SetVector<Value *>> LiveOut; |
157 | }; |
158 | |
159 | // The type of the internal cache used inside the findBasePointers family |
160 | // of functions. From the callers perspective, this is an opaque type and |
161 | // should not be inspected. |
162 | // |
163 | // In the actual implementation this caches two relations: |
164 | // - The base relation itself (i.e. this pointer is based on that one) |
165 | // - The base defining value relation (i.e. before base_phi insertion) |
166 | // Generally, after the execution of a full findBasePointer call, only the |
167 | // base relation will remain. Internally, we add a mixture of the two |
168 | // types, then update all the second type to the first type |
169 | typedef MapVector<Value *, Value *> DefiningValueMapTy; |
170 | typedef SetVector<Value *> StatepointLiveSetTy; |
171 | typedef MapVector<AssertingVH<Instruction>, AssertingVH<Value>> |
172 | RematerializedValueMapTy; |
173 | |
174 | struct PartiallyConstructedSafepointRecord { |
175 | /// The set of values known to be live across this safepoint |
176 | StatepointLiveSetTy LiveSet; |
177 | |
178 | /// Mapping from live pointers to a base-defining-value |
179 | MapVector<Value *, Value *> PointerToBase; |
180 | |
181 | /// The *new* gc.statepoint instruction itself. This produces the token |
182 | /// that normal path gc.relocates and the gc.result are tied to. |
183 | Instruction *StatepointToken; |
184 | |
185 | /// Instruction to which exceptional gc relocates are attached |
186 | /// Makes it easier to iterate through them during relocationViaAlloca. |
187 | Instruction *UnwindToken; |
188 | |
189 | /// Record live values we are rematerialized instead of relocating. |
190 | /// They are not included into 'LiveSet' field. |
191 | /// Maps rematerialized copy to it's original value. |
192 | RematerializedValueMapTy RematerializedValues; |
193 | }; |
194 | } |
195 | |
196 | static ArrayRef<Use> GetDeoptBundleOperands(ImmutableCallSite CS) { |
197 | Optional<OperandBundleUse> DeoptBundle = |
198 | CS.getOperandBundle(LLVMContext::OB_deopt); |
199 | |
200 | if (!DeoptBundle.hasValue()) { |
201 | assert(AllowStatepointWithNoDeoptInfo &&((AllowStatepointWithNoDeoptInfo && "Found non-leaf call without deopt info!" ) ? static_cast<void> (0) : __assert_fail ("AllowStatepointWithNoDeoptInfo && \"Found non-leaf call without deopt info!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 202, __PRETTY_FUNCTION__)) |
202 | "Found non-leaf call without deopt info!")((AllowStatepointWithNoDeoptInfo && "Found non-leaf call without deopt info!" ) ? static_cast<void> (0) : __assert_fail ("AllowStatepointWithNoDeoptInfo && \"Found non-leaf call without deopt info!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 202, __PRETTY_FUNCTION__)); |
203 | return None; |
204 | } |
205 | |
206 | return DeoptBundle.getValue().Inputs; |
207 | } |
208 | |
209 | /// Compute the live-in set for every basic block in the function |
210 | static void computeLiveInValues(DominatorTree &DT, Function &F, |
211 | GCPtrLivenessData &Data); |
212 | |
213 | /// Given results from the dataflow liveness computation, find the set of live |
214 | /// Values at a particular instruction. |
215 | static void findLiveSetAtInst(Instruction *inst, GCPtrLivenessData &Data, |
216 | StatepointLiveSetTy &out); |
217 | |
218 | // TODO: Once we can get to the GCStrategy, this becomes |
219 | // Optional<bool> isGCManagedPointer(const Type *Ty) const override { |
220 | |
221 | static bool isGCPointerType(Type *T) { |
222 | if (auto *PT = dyn_cast<PointerType>(T)) |
223 | // For the sake of this example GC, we arbitrarily pick addrspace(1) as our |
224 | // GC managed heap. We know that a pointer into this heap needs to be |
225 | // updated and that no other pointer does. |
226 | return PT->getAddressSpace() == 1; |
227 | return false; |
228 | } |
229 | |
230 | // Return true if this type is one which a) is a gc pointer or contains a GC |
231 | // pointer and b) is of a type this code expects to encounter as a live value. |
232 | // (The insertion code will assert that a type which matches (a) and not (b) |
233 | // is not encountered.) |
234 | static bool isHandledGCPointerType(Type *T) { |
235 | // We fully support gc pointers |
236 | if (isGCPointerType(T)) |
237 | return true; |
238 | // We partially support vectors of gc pointers. The code will assert if it |
239 | // can't handle something. |
240 | if (auto VT = dyn_cast<VectorType>(T)) |
241 | if (isGCPointerType(VT->getElementType())) |
242 | return true; |
243 | return false; |
244 | } |
245 | |
246 | #ifndef NDEBUG |
247 | /// Returns true if this type contains a gc pointer whether we know how to |
248 | /// handle that type or not. |
249 | static bool containsGCPtrType(Type *Ty) { |
250 | if (isGCPointerType(Ty)) |
251 | return true; |
252 | if (VectorType *VT = dyn_cast<VectorType>(Ty)) |
253 | return isGCPointerType(VT->getScalarType()); |
254 | if (ArrayType *AT = dyn_cast<ArrayType>(Ty)) |
255 | return containsGCPtrType(AT->getElementType()); |
256 | if (StructType *ST = dyn_cast<StructType>(Ty)) |
257 | return any_of(ST->subtypes(), containsGCPtrType); |
258 | return false; |
259 | } |
260 | |
261 | // Returns true if this is a type which a) is a gc pointer or contains a GC |
262 | // pointer and b) is of a type which the code doesn't expect (i.e. first class |
263 | // aggregates). Used to trip assertions. |
264 | static bool isUnhandledGCPointerType(Type *Ty) { |
265 | return containsGCPtrType(Ty) && !isHandledGCPointerType(Ty); |
266 | } |
267 | #endif |
268 | |
269 | // Return the name of the value suffixed with the provided value, or if the |
270 | // value didn't have a name, the default value specified. |
271 | static std::string suffixed_name_or(Value *V, StringRef Suffix, |
272 | StringRef DefaultName) { |
273 | return V->hasName() ? (V->getName() + Suffix).str() : DefaultName.str(); |
274 | } |
275 | |
276 | // Conservatively identifies any definitions which might be live at the |
277 | // given instruction. The analysis is performed immediately before the |
278 | // given instruction. Values defined by that instruction are not considered |
279 | // live. Values used by that instruction are considered live. |
280 | static void |
281 | analyzeParsePointLiveness(DominatorTree &DT, |
282 | GCPtrLivenessData &OriginalLivenessData, CallSite CS, |
283 | PartiallyConstructedSafepointRecord &Result) { |
284 | Instruction *Inst = CS.getInstruction(); |
285 | |
286 | StatepointLiveSetTy LiveSet; |
287 | findLiveSetAtInst(Inst, OriginalLivenessData, LiveSet); |
288 | |
289 | if (PrintLiveSet) { |
290 | dbgs() << "Live Variables:\n"; |
291 | for (Value *V : LiveSet) |
292 | dbgs() << " " << V->getName() << " " << *V << "\n"; |
293 | } |
294 | if (PrintLiveSetSize) { |
295 | dbgs() << "Safepoint For: " << CS.getCalledValue()->getName() << "\n"; |
296 | dbgs() << "Number live values: " << LiveSet.size() << "\n"; |
297 | } |
298 | Result.LiveSet = LiveSet; |
299 | } |
300 | |
301 | static bool isKnownBaseResult(Value *V); |
302 | namespace { |
303 | /// A single base defining value - An immediate base defining value for an |
304 | /// instruction 'Def' is an input to 'Def' whose base is also a base of 'Def'. |
305 | /// For instructions which have multiple pointer [vector] inputs or that |
306 | /// transition between vector and scalar types, there is no immediate base |
307 | /// defining value. The 'base defining value' for 'Def' is the transitive |
308 | /// closure of this relation stopping at the first instruction which has no |
309 | /// immediate base defining value. The b.d.v. might itself be a base pointer, |
310 | /// but it can also be an arbitrary derived pointer. |
311 | struct BaseDefiningValueResult { |
312 | /// Contains the value which is the base defining value. |
313 | Value * const BDV; |
314 | /// True if the base defining value is also known to be an actual base |
315 | /// pointer. |
316 | const bool IsKnownBase; |
317 | BaseDefiningValueResult(Value *BDV, bool IsKnownBase) |
318 | : BDV(BDV), IsKnownBase(IsKnownBase) { |
319 | #ifndef NDEBUG |
320 | // Check consistency between new and old means of checking whether a BDV is |
321 | // a base. |
322 | bool MustBeBase = isKnownBaseResult(BDV); |
323 | assert(!MustBeBase || MustBeBase == IsKnownBase)((!MustBeBase || MustBeBase == IsKnownBase) ? static_cast< void> (0) : __assert_fail ("!MustBeBase || MustBeBase == IsKnownBase" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 323, __PRETTY_FUNCTION__)); |
324 | #endif |
325 | } |
326 | }; |
327 | } |
328 | |
329 | static BaseDefiningValueResult findBaseDefiningValue(Value *I); |
330 | |
331 | /// Return a base defining value for the 'Index' element of the given vector |
332 | /// instruction 'I'. If Index is null, returns a BDV for the entire vector |
333 | /// 'I'. As an optimization, this method will try to determine when the |
334 | /// element is known to already be a base pointer. If this can be established, |
335 | /// the second value in the returned pair will be true. Note that either a |
336 | /// vector or a pointer typed value can be returned. For the former, the |
337 | /// vector returned is a BDV (and possibly a base) of the entire vector 'I'. |
338 | /// If the later, the return pointer is a BDV (or possibly a base) for the |
339 | /// particular element in 'I'. |
340 | static BaseDefiningValueResult |
341 | findBaseDefiningValueOfVector(Value *I) { |
342 | // Each case parallels findBaseDefiningValue below, see that code for |
343 | // detailed motivation. |
344 | |
345 | if (isa<Argument>(I)) |
346 | // An incoming argument to the function is a base pointer |
347 | return BaseDefiningValueResult(I, true); |
348 | |
349 | if (isa<Constant>(I)) |
350 | // Base of constant vector consists only of constant null pointers. |
351 | // For reasoning see similar case inside 'findBaseDefiningValue' function. |
352 | return BaseDefiningValueResult(ConstantAggregateZero::get(I->getType()), |
353 | true); |
354 | |
355 | if (isa<LoadInst>(I)) |
356 | return BaseDefiningValueResult(I, true); |
357 | |
358 | if (isa<InsertElementInst>(I)) |
359 | // We don't know whether this vector contains entirely base pointers or |
360 | // not. To be conservatively correct, we treat it as a BDV and will |
361 | // duplicate code as needed to construct a parallel vector of bases. |
362 | return BaseDefiningValueResult(I, false); |
363 | |
364 | if (isa<ShuffleVectorInst>(I)) |
365 | // We don't know whether this vector contains entirely base pointers or |
366 | // not. To be conservatively correct, we treat it as a BDV and will |
367 | // duplicate code as needed to construct a parallel vector of bases. |
368 | // TODO: There a number of local optimizations which could be applied here |
369 | // for particular sufflevector patterns. |
370 | return BaseDefiningValueResult(I, false); |
371 | |
372 | // The behavior of getelementptr instructions is the same for vector and |
373 | // non-vector data types. |
374 | if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) |
375 | return findBaseDefiningValue(GEP->getPointerOperand()); |
376 | |
377 | // A PHI or Select is a base defining value. The outer findBasePointer |
378 | // algorithm is responsible for constructing a base value for this BDV. |
379 | assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&(((isa<SelectInst>(I) || isa<PHINode>(I)) && "unknown vector instruction - no base found for vector element" ) ? static_cast<void> (0) : __assert_fail ("(isa<SelectInst>(I) || isa<PHINode>(I)) && \"unknown vector instruction - no base found for vector element\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 380, __PRETTY_FUNCTION__)) |
380 | "unknown vector instruction - no base found for vector element")(((isa<SelectInst>(I) || isa<PHINode>(I)) && "unknown vector instruction - no base found for vector element" ) ? static_cast<void> (0) : __assert_fail ("(isa<SelectInst>(I) || isa<PHINode>(I)) && \"unknown vector instruction - no base found for vector element\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 380, __PRETTY_FUNCTION__)); |
381 | return BaseDefiningValueResult(I, false); |
382 | } |
383 | |
384 | /// Helper function for findBasePointer - Will return a value which either a) |
385 | /// defines the base pointer for the input, b) blocks the simple search |
386 | /// (i.e. a PHI or Select of two derived pointers), or c) involves a change |
387 | /// from pointer to vector type or back. |
388 | static BaseDefiningValueResult findBaseDefiningValue(Value *I) { |
389 | assert(I->getType()->isPtrOrPtrVectorTy() &&((I->getType()->isPtrOrPtrVectorTy() && "Illegal to ask for the base pointer of a non-pointer type" ) ? static_cast<void> (0) : __assert_fail ("I->getType()->isPtrOrPtrVectorTy() && \"Illegal to ask for the base pointer of a non-pointer type\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 390, __PRETTY_FUNCTION__)) |
390 | "Illegal to ask for the base pointer of a non-pointer type")((I->getType()->isPtrOrPtrVectorTy() && "Illegal to ask for the base pointer of a non-pointer type" ) ? static_cast<void> (0) : __assert_fail ("I->getType()->isPtrOrPtrVectorTy() && \"Illegal to ask for the base pointer of a non-pointer type\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 390, __PRETTY_FUNCTION__)); |
391 | |
392 | if (I->getType()->isVectorTy()) |
393 | return findBaseDefiningValueOfVector(I); |
394 | |
395 | if (isa<Argument>(I)) |
396 | // An incoming argument to the function is a base pointer |
397 | // We should have never reached here if this argument isn't an gc value |
398 | return BaseDefiningValueResult(I, true); |
399 | |
400 | if (isa<Constant>(I)) { |
401 | // We assume that objects with a constant base (e.g. a global) can't move |
402 | // and don't need to be reported to the collector because they are always |
403 | // live. Besides global references, all kinds of constants (e.g. undef, |
404 | // constant expressions, null pointers) can be introduced by the inliner or |
405 | // the optimizer, especially on dynamically dead paths. |
406 | // Here we treat all of them as having single null base. By doing this we |
407 | // trying to avoid problems reporting various conflicts in a form of |
408 | // "phi (const1, const2)" or "phi (const, regular gc ptr)". |
409 | // See constant.ll file for relevant test cases. |
410 | |
411 | return BaseDefiningValueResult( |
412 | ConstantPointerNull::get(cast<PointerType>(I->getType())), true); |
413 | } |
414 | |
415 | if (CastInst *CI = dyn_cast<CastInst>(I)) { |
416 | Value *Def = CI->stripPointerCasts(); |
417 | // If stripping pointer casts changes the address space there is an |
418 | // addrspacecast in between. |
419 | assert(cast<PointerType>(Def->getType())->getAddressSpace() ==((cast<PointerType>(Def->getType())->getAddressSpace () == cast<PointerType>(CI->getType())->getAddressSpace () && "unsupported addrspacecast") ? static_cast<void > (0) : __assert_fail ("cast<PointerType>(Def->getType())->getAddressSpace() == cast<PointerType>(CI->getType())->getAddressSpace() && \"unsupported addrspacecast\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 421, __PRETTY_FUNCTION__)) |
420 | cast<PointerType>(CI->getType())->getAddressSpace() &&((cast<PointerType>(Def->getType())->getAddressSpace () == cast<PointerType>(CI->getType())->getAddressSpace () && "unsupported addrspacecast") ? static_cast<void > (0) : __assert_fail ("cast<PointerType>(Def->getType())->getAddressSpace() == cast<PointerType>(CI->getType())->getAddressSpace() && \"unsupported addrspacecast\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 421, __PRETTY_FUNCTION__)) |
421 | "unsupported addrspacecast")((cast<PointerType>(Def->getType())->getAddressSpace () == cast<PointerType>(CI->getType())->getAddressSpace () && "unsupported addrspacecast") ? static_cast<void > (0) : __assert_fail ("cast<PointerType>(Def->getType())->getAddressSpace() == cast<PointerType>(CI->getType())->getAddressSpace() && \"unsupported addrspacecast\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 421, __PRETTY_FUNCTION__)); |
422 | // If we find a cast instruction here, it means we've found a cast which is |
423 | // not simply a pointer cast (i.e. an inttoptr). We don't know how to |
424 | // handle int->ptr conversion. |
425 | assert(!isa<CastInst>(Def) && "shouldn't find another cast here")((!isa<CastInst>(Def) && "shouldn't find another cast here" ) ? static_cast<void> (0) : __assert_fail ("!isa<CastInst>(Def) && \"shouldn't find another cast here\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 425, __PRETTY_FUNCTION__)); |
426 | return findBaseDefiningValue(Def); |
427 | } |
428 | |
429 | if (isa<LoadInst>(I)) |
430 | // The value loaded is an gc base itself |
431 | return BaseDefiningValueResult(I, true); |
432 | |
433 | |
434 | if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) |
435 | // The base of this GEP is the base |
436 | return findBaseDefiningValue(GEP->getPointerOperand()); |
437 | |
438 | if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { |
439 | switch (II->getIntrinsicID()) { |
440 | default: |
441 | // fall through to general call handling |
442 | break; |
443 | case Intrinsic::experimental_gc_statepoint: |
444 | llvm_unreachable("statepoints don't produce pointers")::llvm::llvm_unreachable_internal("statepoints don't produce pointers" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 444); |
445 | case Intrinsic::experimental_gc_relocate: { |
446 | // Rerunning safepoint insertion after safepoints are already |
447 | // inserted is not supported. It could probably be made to work, |
448 | // but why are you doing this? There's no good reason. |
449 | llvm_unreachable("repeat safepoint insertion is not supported")::llvm::llvm_unreachable_internal("repeat safepoint insertion is not supported" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 449); |
450 | } |
451 | case Intrinsic::gcroot: |
452 | // Currently, this mechanism hasn't been extended to work with gcroot. |
453 | // There's no reason it couldn't be, but I haven't thought about the |
454 | // implications much. |
455 | llvm_unreachable(::llvm::llvm_unreachable_internal("interaction with the gcroot mechanism is not supported" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 456) |
456 | "interaction with the gcroot mechanism is not supported")::llvm::llvm_unreachable_internal("interaction with the gcroot mechanism is not supported" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 456); |
457 | } |
458 | } |
459 | // We assume that functions in the source language only return base |
460 | // pointers. This should probably be generalized via attributes to support |
461 | // both source language and internal functions. |
462 | if (isa<CallInst>(I) || isa<InvokeInst>(I)) |
463 | return BaseDefiningValueResult(I, true); |
464 | |
465 | // TODO: I have absolutely no idea how to implement this part yet. It's not |
466 | // necessarily hard, I just haven't really looked at it yet. |
467 | assert(!isa<LandingPadInst>(I) && "Landing Pad is unimplemented")((!isa<LandingPadInst>(I) && "Landing Pad is unimplemented" ) ? static_cast<void> (0) : __assert_fail ("!isa<LandingPadInst>(I) && \"Landing Pad is unimplemented\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 467, __PRETTY_FUNCTION__)); |
468 | |
469 | if (isa<AtomicCmpXchgInst>(I)) |
470 | // A CAS is effectively a atomic store and load combined under a |
471 | // predicate. From the perspective of base pointers, we just treat it |
472 | // like a load. |
473 | return BaseDefiningValueResult(I, true); |
474 | |
475 | assert(!isa<AtomicRMWInst>(I) && "Xchg handled above, all others are "((!isa<AtomicRMWInst>(I) && "Xchg handled above, all others are " "binary ops which don't apply to pointers") ? static_cast< void> (0) : __assert_fail ("!isa<AtomicRMWInst>(I) && \"Xchg handled above, all others are \" \"binary ops which don't apply to pointers\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 476, __PRETTY_FUNCTION__)) |
476 | "binary ops which don't apply to pointers")((!isa<AtomicRMWInst>(I) && "Xchg handled above, all others are " "binary ops which don't apply to pointers") ? static_cast< void> (0) : __assert_fail ("!isa<AtomicRMWInst>(I) && \"Xchg handled above, all others are \" \"binary ops which don't apply to pointers\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 476, __PRETTY_FUNCTION__)); |
477 | |
478 | // The aggregate ops. Aggregates can either be in the heap or on the |
479 | // stack, but in either case, this is simply a field load. As a result, |
480 | // this is a defining definition of the base just like a load is. |
481 | if (isa<ExtractValueInst>(I)) |
482 | return BaseDefiningValueResult(I, true); |
483 | |
484 | // We should never see an insert vector since that would require we be |
485 | // tracing back a struct value not a pointer value. |
486 | assert(!isa<InsertValueInst>(I) &&((!isa<InsertValueInst>(I) && "Base pointer for a struct is meaningless" ) ? static_cast<void> (0) : __assert_fail ("!isa<InsertValueInst>(I) && \"Base pointer for a struct is meaningless\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 487, __PRETTY_FUNCTION__)) |
487 | "Base pointer for a struct is meaningless")((!isa<InsertValueInst>(I) && "Base pointer for a struct is meaningless" ) ? static_cast<void> (0) : __assert_fail ("!isa<InsertValueInst>(I) && \"Base pointer for a struct is meaningless\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 487, __PRETTY_FUNCTION__)); |
488 | |
489 | // An extractelement produces a base result exactly when it's input does. |
490 | // We may need to insert a parallel instruction to extract the appropriate |
491 | // element out of the base vector corresponding to the input. Given this, |
492 | // it's analogous to the phi and select case even though it's not a merge. |
493 | if (isa<ExtractElementInst>(I)) |
494 | // Note: There a lot of obvious peephole cases here. This are deliberately |
495 | // handled after the main base pointer inference algorithm to make writing |
496 | // test cases to exercise that code easier. |
497 | return BaseDefiningValueResult(I, false); |
498 | |
499 | // The last two cases here don't return a base pointer. Instead, they |
500 | // return a value which dynamically selects from among several base |
501 | // derived pointers (each with it's own base potentially). It's the job of |
502 | // the caller to resolve these. |
503 | assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&(((isa<SelectInst>(I) || isa<PHINode>(I)) && "missing instruction case in findBaseDefiningValing") ? static_cast <void> (0) : __assert_fail ("(isa<SelectInst>(I) || isa<PHINode>(I)) && \"missing instruction case in findBaseDefiningValing\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 504, __PRETTY_FUNCTION__)) |
504 | "missing instruction case in findBaseDefiningValing")(((isa<SelectInst>(I) || isa<PHINode>(I)) && "missing instruction case in findBaseDefiningValing") ? static_cast <void> (0) : __assert_fail ("(isa<SelectInst>(I) || isa<PHINode>(I)) && \"missing instruction case in findBaseDefiningValing\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 504, __PRETTY_FUNCTION__)); |
505 | return BaseDefiningValueResult(I, false); |
506 | } |
507 | |
508 | /// Returns the base defining value for this value. |
509 | static Value *findBaseDefiningValueCached(Value *I, DefiningValueMapTy &Cache) { |
510 | Value *&Cached = Cache[I]; |
511 | if (!Cached) { |
512 | Cached = findBaseDefiningValue(I).BDV; |
513 | DEBUG(dbgs() << "fBDV-cached: " << I->getName() << " -> "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << "fBDV-cached: " << I->getName() << " -> " << Cached-> getName() << "\n"; } } while (false) |
514 | << Cached->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << "fBDV-cached: " << I->getName() << " -> " << Cached-> getName() << "\n"; } } while (false); |
515 | } |
516 | assert(Cache[I] != nullptr)((Cache[I] != nullptr) ? static_cast<void> (0) : __assert_fail ("Cache[I] != nullptr", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 516, __PRETTY_FUNCTION__)); |
517 | return Cached; |
518 | } |
519 | |
520 | /// Return a base pointer for this value if known. Otherwise, return it's |
521 | /// base defining value. |
522 | static Value *findBaseOrBDV(Value *I, DefiningValueMapTy &Cache) { |
523 | Value *Def = findBaseDefiningValueCached(I, Cache); |
524 | auto Found = Cache.find(Def); |
525 | if (Found != Cache.end()) { |
526 | // Either a base-of relation, or a self reference. Caller must check. |
527 | return Found->second; |
528 | } |
529 | // Only a BDV available |
530 | return Def; |
531 | } |
532 | |
533 | /// Given the result of a call to findBaseDefiningValue, or findBaseOrBDV, |
534 | /// is it known to be a base pointer? Or do we need to continue searching. |
535 | static bool isKnownBaseResult(Value *V) { |
536 | if (!isa<PHINode>(V) && !isa<SelectInst>(V) && |
537 | !isa<ExtractElementInst>(V) && !isa<InsertElementInst>(V) && |
538 | !isa<ShuffleVectorInst>(V)) { |
539 | // no recursion possible |
540 | return true; |
541 | } |
542 | if (isa<Instruction>(V) && |
543 | cast<Instruction>(V)->getMetadata("is_base_value")) { |
544 | // This is a previously inserted base phi or select. We know |
545 | // that this is a base value. |
546 | return true; |
547 | } |
548 | |
549 | // We need to keep searching |
550 | return false; |
551 | } |
552 | |
553 | namespace { |
554 | /// Models the state of a single base defining value in the findBasePointer |
555 | /// algorithm for determining where a new instruction is needed to propagate |
556 | /// the base of this BDV. |
557 | class BDVState { |
558 | public: |
559 | enum Status { Unknown, Base, Conflict }; |
560 | |
561 | BDVState() : Status(Unknown), BaseValue(nullptr) {} |
562 | |
563 | explicit BDVState(Status Status, Value *BaseValue = nullptr) |
564 | : Status(Status), BaseValue(BaseValue) { |
565 | assert(Status != Base || BaseValue)((Status != Base || BaseValue) ? static_cast<void> (0) : __assert_fail ("Status != Base || BaseValue", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 565, __PRETTY_FUNCTION__)); |
566 | } |
567 | |
568 | explicit BDVState(Value *BaseValue) : Status(Base), BaseValue(BaseValue) {} |
569 | |
570 | Status getStatus() const { return Status; } |
571 | Value *getBaseValue() const { return BaseValue; } |
572 | |
573 | bool isBase() const { return getStatus() == Base; } |
574 | bool isUnknown() const { return getStatus() == Unknown; } |
575 | bool isConflict() const { return getStatus() == Conflict; } |
576 | |
577 | bool operator==(const BDVState &Other) const { |
578 | return BaseValue == Other.BaseValue && Status == Other.Status; |
579 | } |
580 | |
581 | bool operator!=(const BDVState &other) const { return !(*this == other); } |
582 | |
583 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) |
584 | void dump() const { |
585 | print(dbgs()); |
586 | dbgs() << '\n'; |
587 | } |
588 | |
589 | void print(raw_ostream &OS) const { |
590 | switch (getStatus()) { |
591 | case Unknown: |
592 | OS << "U"; |
593 | break; |
594 | case Base: |
595 | OS << "B"; |
596 | break; |
597 | case Conflict: |
598 | OS << "C"; |
599 | break; |
600 | }; |
601 | OS << " (" << getBaseValue() << " - " |
602 | << (getBaseValue() ? getBaseValue()->getName() : "nullptr") << "): "; |
603 | } |
604 | |
605 | private: |
606 | Status Status; |
607 | AssertingVH<Value> BaseValue; // Non-null only if Status == Base. |
608 | }; |
609 | } |
610 | |
611 | #ifndef NDEBUG |
612 | static raw_ostream &operator<<(raw_ostream &OS, const BDVState &State) { |
613 | State.print(OS); |
614 | return OS; |
615 | } |
616 | #endif |
617 | |
618 | static BDVState meetBDVStateImpl(const BDVState &LHS, const BDVState &RHS) { |
619 | switch (LHS.getStatus()) { |
620 | case BDVState::Unknown: |
621 | return RHS; |
622 | |
623 | case BDVState::Base: |
624 | assert(LHS.getBaseValue() && "can't be null")((LHS.getBaseValue() && "can't be null") ? static_cast <void> (0) : __assert_fail ("LHS.getBaseValue() && \"can't be null\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 624, __PRETTY_FUNCTION__)); |
625 | if (RHS.isUnknown()) |
626 | return LHS; |
627 | |
628 | if (RHS.isBase()) { |
629 | if (LHS.getBaseValue() == RHS.getBaseValue()) { |
630 | assert(LHS == RHS && "equality broken!")((LHS == RHS && "equality broken!") ? static_cast< void> (0) : __assert_fail ("LHS == RHS && \"equality broken!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 630, __PRETTY_FUNCTION__)); |
631 | return LHS; |
632 | } |
633 | return BDVState(BDVState::Conflict); |
634 | } |
635 | assert(RHS.isConflict() && "only three states!")((RHS.isConflict() && "only three states!") ? static_cast <void> (0) : __assert_fail ("RHS.isConflict() && \"only three states!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 635, __PRETTY_FUNCTION__)); |
636 | return BDVState(BDVState::Conflict); |
637 | |
638 | case BDVState::Conflict: |
639 | return LHS; |
640 | } |
641 | llvm_unreachable("only three states!")::llvm::llvm_unreachable_internal("only three states!", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 641); |
642 | } |
643 | |
644 | // Values of type BDVState form a lattice, and this function implements the meet |
645 | // operation. |
646 | static BDVState meetBDVState(const BDVState &LHS, const BDVState &RHS) { |
647 | BDVState Result = meetBDVStateImpl(LHS, RHS); |
648 | assert(Result == meetBDVStateImpl(RHS, LHS) &&((Result == meetBDVStateImpl(RHS, LHS) && "Math is wrong: meet does not commute!" ) ? static_cast<void> (0) : __assert_fail ("Result == meetBDVStateImpl(RHS, LHS) && \"Math is wrong: meet does not commute!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 649, __PRETTY_FUNCTION__)) |
649 | "Math is wrong: meet does not commute!")((Result == meetBDVStateImpl(RHS, LHS) && "Math is wrong: meet does not commute!" ) ? static_cast<void> (0) : __assert_fail ("Result == meetBDVStateImpl(RHS, LHS) && \"Math is wrong: meet does not commute!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 649, __PRETTY_FUNCTION__)); |
650 | return Result; |
651 | } |
652 | |
653 | /// For a given value or instruction, figure out what base ptr its derived from. |
654 | /// For gc objects, this is simply itself. On success, returns a value which is |
655 | /// the base pointer. (This is reliable and can be used for relocation.) On |
656 | /// failure, returns nullptr. |
657 | static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) { |
658 | Value *Def = findBaseOrBDV(I, Cache); |
659 | |
660 | if (isKnownBaseResult(Def)) |
661 | return Def; |
662 | |
663 | // Here's the rough algorithm: |
664 | // - For every SSA value, construct a mapping to either an actual base |
665 | // pointer or a PHI which obscures the base pointer. |
666 | // - Construct a mapping from PHI to unknown TOP state. Use an |
667 | // optimistic algorithm to propagate base pointer information. Lattice |
668 | // looks like: |
669 | // UNKNOWN |
670 | // b1 b2 b3 b4 |
671 | // CONFLICT |
672 | // When algorithm terminates, all PHIs will either have a single concrete |
673 | // base or be in a conflict state. |
674 | // - For every conflict, insert a dummy PHI node without arguments. Add |
675 | // these to the base[Instruction] = BasePtr mapping. For every |
676 | // non-conflict, add the actual base. |
677 | // - For every conflict, add arguments for the base[a] of each input |
678 | // arguments. |
679 | // |
680 | // Note: A simpler form of this would be to add the conflict form of all |
681 | // PHIs without running the optimistic algorithm. This would be |
682 | // analogous to pessimistic data flow and would likely lead to an |
683 | // overall worse solution. |
684 | |
685 | #ifndef NDEBUG |
686 | auto isExpectedBDVType = [](Value *BDV) { |
687 | return isa<PHINode>(BDV) || isa<SelectInst>(BDV) || |
688 | isa<ExtractElementInst>(BDV) || isa<InsertElementInst>(BDV) || |
689 | isa<ShuffleVectorInst>(BDV); |
690 | }; |
691 | #endif |
692 | |
693 | // Once populated, will contain a mapping from each potentially non-base BDV |
694 | // to a lattice value (described above) which corresponds to that BDV. |
695 | // We use the order of insertion (DFS over the def/use graph) to provide a |
696 | // stable deterministic ordering for visiting DenseMaps (which are unordered) |
697 | // below. This is important for deterministic compilation. |
698 | MapVector<Value *, BDVState> States; |
699 | |
700 | // Recursively fill in all base defining values reachable from the initial |
701 | // one for which we don't already know a definite base value for |
702 | /* scope */ { |
703 | SmallVector<Value*, 16> Worklist; |
704 | Worklist.push_back(Def); |
705 | States.insert({Def, BDVState()}); |
706 | while (!Worklist.empty()) { |
707 | Value *Current = Worklist.pop_back_val(); |
708 | assert(!isKnownBaseResult(Current) && "why did it get added?")((!isKnownBaseResult(Current) && "why did it get added?" ) ? static_cast<void> (0) : __assert_fail ("!isKnownBaseResult(Current) && \"why did it get added?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 708, __PRETTY_FUNCTION__)); |
709 | |
710 | auto visitIncomingValue = [&](Value *InVal) { |
711 | Value *Base = findBaseOrBDV(InVal, Cache); |
712 | if (isKnownBaseResult(Base)) |
713 | // Known bases won't need new instructions introduced and can be |
714 | // ignored safely |
715 | return; |
716 | assert(isExpectedBDVType(Base) && "the only non-base values "((isExpectedBDVType(Base) && "the only non-base values " "we see should be base defining values") ? static_cast<void > (0) : __assert_fail ("isExpectedBDVType(Base) && \"the only non-base values \" \"we see should be base defining values\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 717, __PRETTY_FUNCTION__)) |
717 | "we see should be base defining values")((isExpectedBDVType(Base) && "the only non-base values " "we see should be base defining values") ? static_cast<void > (0) : __assert_fail ("isExpectedBDVType(Base) && \"the only non-base values \" \"we see should be base defining values\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 717, __PRETTY_FUNCTION__)); |
718 | if (States.insert(std::make_pair(Base, BDVState())).second) |
719 | Worklist.push_back(Base); |
720 | }; |
721 | if (PHINode *PN = dyn_cast<PHINode>(Current)) { |
722 | for (Value *InVal : PN->incoming_values()) |
723 | visitIncomingValue(InVal); |
724 | } else if (SelectInst *SI = dyn_cast<SelectInst>(Current)) { |
725 | visitIncomingValue(SI->getTrueValue()); |
726 | visitIncomingValue(SI->getFalseValue()); |
727 | } else if (auto *EE = dyn_cast<ExtractElementInst>(Current)) { |
728 | visitIncomingValue(EE->getVectorOperand()); |
729 | } else if (auto *IE = dyn_cast<InsertElementInst>(Current)) { |
730 | visitIncomingValue(IE->getOperand(0)); // vector operand |
731 | visitIncomingValue(IE->getOperand(1)); // scalar operand |
732 | } else if (auto *SV = dyn_cast<ShuffleVectorInst>(Current)) { |
733 | visitIncomingValue(SV->getOperand(0)); |
734 | visitIncomingValue(SV->getOperand(1)); |
735 | } |
736 | else { |
737 | llvm_unreachable("Unimplemented instruction case")::llvm::llvm_unreachable_internal("Unimplemented instruction case" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 737); |
738 | } |
739 | } |
740 | } |
741 | |
742 | #ifndef NDEBUG |
743 | DEBUG(dbgs() << "States after initialization:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << "States after initialization:\n" ; } } while (false); |
744 | for (auto Pair : States) { |
745 | DEBUG(dbgs() << " " << Pair.second << " for " << *Pair.first << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << " " << Pair.second << " for " << *Pair.first << "\n" ; } } while (false); |
746 | } |
747 | #endif |
748 | |
749 | // Return a phi state for a base defining value. We'll generate a new |
750 | // base state for known bases and expect to find a cached state otherwise. |
751 | auto getStateForBDV = [&](Value *baseValue) { |
752 | if (isKnownBaseResult(baseValue)) |
753 | return BDVState(baseValue); |
754 | auto I = States.find(baseValue); |
755 | assert(I != States.end() && "lookup failed!")((I != States.end() && "lookup failed!") ? static_cast <void> (0) : __assert_fail ("I != States.end() && \"lookup failed!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 755, __PRETTY_FUNCTION__)); |
756 | return I->second; |
757 | }; |
758 | |
759 | bool Progress = true; |
760 | while (Progress) { |
761 | #ifndef NDEBUG |
762 | const size_t OldSize = States.size(); |
763 | #endif |
764 | Progress = false; |
765 | // We're only changing values in this loop, thus safe to keep iterators. |
766 | // Since this is computing a fixed point, the order of visit does not |
767 | // effect the result. TODO: We could use a worklist here and make this run |
768 | // much faster. |
769 | for (auto Pair : States) { |
770 | Value *BDV = Pair.first; |
771 | assert(!isKnownBaseResult(BDV) && "why did it get added?")((!isKnownBaseResult(BDV) && "why did it get added?") ? static_cast<void> (0) : __assert_fail ("!isKnownBaseResult(BDV) && \"why did it get added?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 771, __PRETTY_FUNCTION__)); |
772 | |
773 | // Given an input value for the current instruction, return a BDVState |
774 | // instance which represents the BDV of that value. |
775 | auto getStateForInput = [&](Value *V) mutable { |
776 | Value *BDV = findBaseOrBDV(V, Cache); |
777 | return getStateForBDV(BDV); |
778 | }; |
779 | |
780 | BDVState NewState; |
781 | if (SelectInst *SI = dyn_cast<SelectInst>(BDV)) { |
782 | NewState = meetBDVState(NewState, getStateForInput(SI->getTrueValue())); |
783 | NewState = |
784 | meetBDVState(NewState, getStateForInput(SI->getFalseValue())); |
785 | } else if (PHINode *PN = dyn_cast<PHINode>(BDV)) { |
786 | for (Value *Val : PN->incoming_values()) |
787 | NewState = meetBDVState(NewState, getStateForInput(Val)); |
788 | } else if (auto *EE = dyn_cast<ExtractElementInst>(BDV)) { |
789 | // The 'meet' for an extractelement is slightly trivial, but it's still |
790 | // useful in that it drives us to conflict if our input is. |
791 | NewState = |
792 | meetBDVState(NewState, getStateForInput(EE->getVectorOperand())); |
793 | } else if (auto *IE = dyn_cast<InsertElementInst>(BDV)){ |
794 | // Given there's a inherent type mismatch between the operands, will |
795 | // *always* produce Conflict. |
796 | NewState = meetBDVState(NewState, getStateForInput(IE->getOperand(0))); |
797 | NewState = meetBDVState(NewState, getStateForInput(IE->getOperand(1))); |
798 | } else { |
799 | // The only instance this does not return a Conflict is when both the |
800 | // vector operands are the same vector. |
801 | auto *SV = cast<ShuffleVectorInst>(BDV); |
802 | NewState = meetBDVState(NewState, getStateForInput(SV->getOperand(0))); |
803 | NewState = meetBDVState(NewState, getStateForInput(SV->getOperand(1))); |
804 | } |
805 | |
806 | BDVState OldState = States[BDV]; |
807 | if (OldState != NewState) { |
808 | Progress = true; |
809 | States[BDV] = NewState; |
810 | } |
811 | } |
812 | |
813 | assert(OldSize == States.size() &&((OldSize == States.size() && "fixed point shouldn't be adding any new nodes to state" ) ? static_cast<void> (0) : __assert_fail ("OldSize == States.size() && \"fixed point shouldn't be adding any new nodes to state\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 814, __PRETTY_FUNCTION__)) |
814 | "fixed point shouldn't be adding any new nodes to state")((OldSize == States.size() && "fixed point shouldn't be adding any new nodes to state" ) ? static_cast<void> (0) : __assert_fail ("OldSize == States.size() && \"fixed point shouldn't be adding any new nodes to state\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 814, __PRETTY_FUNCTION__)); |
815 | } |
816 | |
817 | #ifndef NDEBUG |
818 | DEBUG(dbgs() << "States after meet iteration:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << "States after meet iteration:\n" ; } } while (false); |
819 | for (auto Pair : States) { |
820 | DEBUG(dbgs() << " " << Pair.second << " for " << *Pair.first << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << " " << Pair.second << " for " << *Pair.first << "\n" ; } } while (false); |
821 | } |
822 | #endif |
823 | |
824 | // Insert Phis for all conflicts |
825 | // TODO: adjust naming patterns to avoid this order of iteration dependency |
826 | for (auto Pair : States) { |
827 | Instruction *I = cast<Instruction>(Pair.first); |
828 | BDVState State = Pair.second; |
829 | assert(!isKnownBaseResult(I) && "why did it get added?")((!isKnownBaseResult(I) && "why did it get added?") ? static_cast<void> (0) : __assert_fail ("!isKnownBaseResult(I) && \"why did it get added?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 829, __PRETTY_FUNCTION__)); |
830 | assert(!State.isUnknown() && "Optimistic algorithm didn't complete!")((!State.isUnknown() && "Optimistic algorithm didn't complete!" ) ? static_cast<void> (0) : __assert_fail ("!State.isUnknown() && \"Optimistic algorithm didn't complete!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 830, __PRETTY_FUNCTION__)); |
831 | |
832 | // extractelement instructions are a bit special in that we may need to |
833 | // insert an extract even when we know an exact base for the instruction. |
834 | // The problem is that we need to convert from a vector base to a scalar |
835 | // base for the particular indice we're interested in. |
836 | if (State.isBase() && isa<ExtractElementInst>(I) && |
837 | isa<VectorType>(State.getBaseValue()->getType())) { |
838 | auto *EE = cast<ExtractElementInst>(I); |
839 | // TODO: In many cases, the new instruction is just EE itself. We should |
840 | // exploit this, but can't do it here since it would break the invariant |
841 | // about the BDV not being known to be a base. |
842 | auto *BaseInst = ExtractElementInst::Create( |
843 | State.getBaseValue(), EE->getIndexOperand(), "base_ee", EE); |
844 | BaseInst->setMetadata("is_base_value", MDNode::get(I->getContext(), {})); |
845 | States[I] = BDVState(BDVState::Base, BaseInst); |
846 | } |
847 | |
848 | // Since we're joining a vector and scalar base, they can never be the |
849 | // same. As a result, we should always see insert element having reached |
850 | // the conflict state. |
851 | assert(!isa<InsertElementInst>(I) || State.isConflict())((!isa<InsertElementInst>(I) || State.isConflict()) ? static_cast <void> (0) : __assert_fail ("!isa<InsertElementInst>(I) || State.isConflict()" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 851, __PRETTY_FUNCTION__)); |
852 | |
853 | if (!State.isConflict()) |
854 | continue; |
855 | |
856 | /// Create and insert a new instruction which will represent the base of |
857 | /// the given instruction 'I'. |
858 | auto MakeBaseInstPlaceholder = [](Instruction *I) -> Instruction* { |
859 | if (isa<PHINode>(I)) { |
860 | BasicBlock *BB = I->getParent(); |
861 | int NumPreds = std::distance(pred_begin(BB), pred_end(BB)); |
862 | assert(NumPreds > 0 && "how did we reach here")((NumPreds > 0 && "how did we reach here") ? static_cast <void> (0) : __assert_fail ("NumPreds > 0 && \"how did we reach here\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 862, __PRETTY_FUNCTION__)); |
863 | std::string Name = suffixed_name_or(I, ".base", "base_phi"); |
864 | return PHINode::Create(I->getType(), NumPreds, Name, I); |
865 | } else if (SelectInst *SI = dyn_cast<SelectInst>(I)) { |
866 | // The undef will be replaced later |
867 | UndefValue *Undef = UndefValue::get(SI->getType()); |
868 | std::string Name = suffixed_name_or(I, ".base", "base_select"); |
869 | return SelectInst::Create(SI->getCondition(), Undef, Undef, Name, SI); |
870 | } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) { |
871 | UndefValue *Undef = UndefValue::get(EE->getVectorOperand()->getType()); |
872 | std::string Name = suffixed_name_or(I, ".base", "base_ee"); |
873 | return ExtractElementInst::Create(Undef, EE->getIndexOperand(), Name, |
874 | EE); |
875 | } else if (auto *IE = dyn_cast<InsertElementInst>(I)) { |
876 | UndefValue *VecUndef = UndefValue::get(IE->getOperand(0)->getType()); |
877 | UndefValue *ScalarUndef = UndefValue::get(IE->getOperand(1)->getType()); |
878 | std::string Name = suffixed_name_or(I, ".base", "base_ie"); |
879 | return InsertElementInst::Create(VecUndef, ScalarUndef, |
880 | IE->getOperand(2), Name, IE); |
881 | } else { |
882 | auto *SV = cast<ShuffleVectorInst>(I); |
883 | UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType()); |
884 | std::string Name = suffixed_name_or(I, ".base", "base_sv"); |
885 | return new ShuffleVectorInst(VecUndef, VecUndef, SV->getOperand(2), |
886 | Name, SV); |
887 | } |
888 | }; |
889 | Instruction *BaseInst = MakeBaseInstPlaceholder(I); |
890 | // Add metadata marking this as a base value |
891 | BaseInst->setMetadata("is_base_value", MDNode::get(I->getContext(), {})); |
892 | States[I] = BDVState(BDVState::Conflict, BaseInst); |
893 | } |
894 | |
895 | // Returns a instruction which produces the base pointer for a given |
896 | // instruction. The instruction is assumed to be an input to one of the BDVs |
897 | // seen in the inference algorithm above. As such, we must either already |
898 | // know it's base defining value is a base, or have inserted a new |
899 | // instruction to propagate the base of it's BDV and have entered that newly |
900 | // introduced instruction into the state table. In either case, we are |
901 | // assured to be able to determine an instruction which produces it's base |
902 | // pointer. |
903 | auto getBaseForInput = [&](Value *Input, Instruction *InsertPt) { |
904 | Value *BDV = findBaseOrBDV(Input, Cache); |
905 | Value *Base = nullptr; |
906 | if (isKnownBaseResult(BDV)) { |
907 | Base = BDV; |
908 | } else { |
909 | // Either conflict or base. |
910 | assert(States.count(BDV))((States.count(BDV)) ? static_cast<void> (0) : __assert_fail ("States.count(BDV)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 910, __PRETTY_FUNCTION__)); |
911 | Base = States[BDV].getBaseValue(); |
912 | } |
913 | assert(Base && "Can't be null")((Base && "Can't be null") ? static_cast<void> ( 0) : __assert_fail ("Base && \"Can't be null\"", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 913, __PRETTY_FUNCTION__)); |
914 | // The cast is needed since base traversal may strip away bitcasts |
915 | if (Base->getType() != Input->getType() && InsertPt) |
916 | Base = new BitCastInst(Base, Input->getType(), "cast", InsertPt); |
917 | return Base; |
918 | }; |
919 | |
920 | // Fixup all the inputs of the new PHIs. Visit order needs to be |
921 | // deterministic and predictable because we're naming newly created |
922 | // instructions. |
923 | for (auto Pair : States) { |
924 | Instruction *BDV = cast<Instruction>(Pair.first); |
925 | BDVState State = Pair.second; |
926 | |
927 | assert(!isKnownBaseResult(BDV) && "why did it get added?")((!isKnownBaseResult(BDV) && "why did it get added?") ? static_cast<void> (0) : __assert_fail ("!isKnownBaseResult(BDV) && \"why did it get added?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 927, __PRETTY_FUNCTION__)); |
928 | assert(!State.isUnknown() && "Optimistic algorithm didn't complete!")((!State.isUnknown() && "Optimistic algorithm didn't complete!" ) ? static_cast<void> (0) : __assert_fail ("!State.isUnknown() && \"Optimistic algorithm didn't complete!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 928, __PRETTY_FUNCTION__)); |
929 | if (!State.isConflict()) |
930 | continue; |
931 | |
932 | if (PHINode *BasePHI = dyn_cast<PHINode>(State.getBaseValue())) { |
933 | PHINode *PN = cast<PHINode>(BDV); |
934 | unsigned NumPHIValues = PN->getNumIncomingValues(); |
935 | for (unsigned i = 0; i < NumPHIValues; i++) { |
936 | Value *InVal = PN->getIncomingValue(i); |
937 | BasicBlock *InBB = PN->getIncomingBlock(i); |
938 | |
939 | // If we've already seen InBB, add the same incoming value |
940 | // we added for it earlier. The IR verifier requires phi |
941 | // nodes with multiple entries from the same basic block |
942 | // to have the same incoming value for each of those |
943 | // entries. If we don't do this check here and basephi |
944 | // has a different type than base, we'll end up adding two |
945 | // bitcasts (and hence two distinct values) as incoming |
946 | // values for the same basic block. |
947 | |
948 | int BlockIndex = BasePHI->getBasicBlockIndex(InBB); |
949 | if (BlockIndex != -1) { |
950 | Value *OldBase = BasePHI->getIncomingValue(BlockIndex); |
951 | BasePHI->addIncoming(OldBase, InBB); |
952 | |
953 | #ifndef NDEBUG |
954 | Value *Base = getBaseForInput(InVal, nullptr); |
955 | // In essence this assert states: the only way two values |
956 | // incoming from the same basic block may be different is by |
957 | // being different bitcasts of the same value. A cleanup |
958 | // that remains TODO is changing findBaseOrBDV to return an |
959 | // llvm::Value of the correct type (and still remain pure). |
960 | // This will remove the need to add bitcasts. |
961 | assert(Base->stripPointerCasts() == OldBase->stripPointerCasts() &&((Base->stripPointerCasts() == OldBase->stripPointerCasts () && "Sanity -- findBaseOrBDV should be pure!") ? static_cast <void> (0) : __assert_fail ("Base->stripPointerCasts() == OldBase->stripPointerCasts() && \"Sanity -- findBaseOrBDV should be pure!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 962, __PRETTY_FUNCTION__)) |
962 | "Sanity -- findBaseOrBDV should be pure!")((Base->stripPointerCasts() == OldBase->stripPointerCasts () && "Sanity -- findBaseOrBDV should be pure!") ? static_cast <void> (0) : __assert_fail ("Base->stripPointerCasts() == OldBase->stripPointerCasts() && \"Sanity -- findBaseOrBDV should be pure!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 962, __PRETTY_FUNCTION__)); |
963 | #endif |
964 | continue; |
965 | } |
966 | |
967 | // Find the instruction which produces the base for each input. We may |
968 | // need to insert a bitcast in the incoming block. |
969 | // TODO: Need to split critical edges if insertion is needed |
970 | Value *Base = getBaseForInput(InVal, InBB->getTerminator()); |
971 | BasePHI->addIncoming(Base, InBB); |
972 | } |
973 | assert(BasePHI->getNumIncomingValues() == NumPHIValues)((BasePHI->getNumIncomingValues() == NumPHIValues) ? static_cast <void> (0) : __assert_fail ("BasePHI->getNumIncomingValues() == NumPHIValues" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 973, __PRETTY_FUNCTION__)); |
974 | } else if (SelectInst *BaseSI = |
975 | dyn_cast<SelectInst>(State.getBaseValue())) { |
976 | SelectInst *SI = cast<SelectInst>(BDV); |
977 | |
978 | // Find the instruction which produces the base for each input. |
979 | // We may need to insert a bitcast. |
980 | BaseSI->setTrueValue(getBaseForInput(SI->getTrueValue(), BaseSI)); |
981 | BaseSI->setFalseValue(getBaseForInput(SI->getFalseValue(), BaseSI)); |
982 | } else if (auto *BaseEE = |
983 | dyn_cast<ExtractElementInst>(State.getBaseValue())) { |
984 | Value *InVal = cast<ExtractElementInst>(BDV)->getVectorOperand(); |
985 | // Find the instruction which produces the base for each input. We may |
986 | // need to insert a bitcast. |
987 | BaseEE->setOperand(0, getBaseForInput(InVal, BaseEE)); |
988 | } else if (auto *BaseIE = dyn_cast<InsertElementInst>(State.getBaseValue())){ |
989 | auto *BdvIE = cast<InsertElementInst>(BDV); |
990 | auto UpdateOperand = [&](int OperandIdx) { |
991 | Value *InVal = BdvIE->getOperand(OperandIdx); |
992 | Value *Base = getBaseForInput(InVal, BaseIE); |
993 | BaseIE->setOperand(OperandIdx, Base); |
994 | }; |
995 | UpdateOperand(0); // vector operand |
996 | UpdateOperand(1); // scalar operand |
997 | } else { |
998 | auto *BaseSV = cast<ShuffleVectorInst>(State.getBaseValue()); |
999 | auto *BdvSV = cast<ShuffleVectorInst>(BDV); |
1000 | auto UpdateOperand = [&](int OperandIdx) { |
1001 | Value *InVal = BdvSV->getOperand(OperandIdx); |
1002 | Value *Base = getBaseForInput(InVal, BaseSV); |
1003 | BaseSV->setOperand(OperandIdx, Base); |
Called C++ object pointer is null | |
1004 | }; |
1005 | UpdateOperand(0); // vector operand |
1006 | UpdateOperand(1); // vector operand |
1007 | } |
1008 | } |
1009 | |
1010 | // Cache all of our results so we can cheaply reuse them |
1011 | // NOTE: This is actually two caches: one of the base defining value |
1012 | // relation and one of the base pointer relation! FIXME |
1013 | for (auto Pair : States) { |
1014 | auto *BDV = Pair.first; |
1015 | Value *Base = Pair.second.getBaseValue(); |
1016 | assert(BDV && Base)((BDV && Base) ? static_cast<void> (0) : __assert_fail ("BDV && Base", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1016, __PRETTY_FUNCTION__)); |
1017 | assert(!isKnownBaseResult(BDV) && "why did it get added?")((!isKnownBaseResult(BDV) && "why did it get added?") ? static_cast<void> (0) : __assert_fail ("!isKnownBaseResult(BDV) && \"why did it get added?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1017, __PRETTY_FUNCTION__)); |
1018 | |
1019 | DEBUG(dbgs() << "Updating base value cache"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << "Updating base value cache" << " for: " << BDV->getName() << " from: " << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none") << " to: " << Base->getName() << "\n"; } } while (false) |
1020 | << " for: " << BDV->getName() << " from: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << "Updating base value cache" << " for: " << BDV->getName() << " from: " << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none") << " to: " << Base->getName() << "\n"; } } while (false) |
1021 | << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << "Updating base value cache" << " for: " << BDV->getName() << " from: " << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none") << " to: " << Base->getName() << "\n"; } } while (false) |
1022 | << " to: " << Base->getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("rewrite-statepoints-for-gc")) { dbgs() << "Updating base value cache" << " for: " << BDV->getName() << " from: " << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none") << " to: " << Base->getName() << "\n"; } } while (false); |
1023 | |
1024 | if (Cache.count(BDV)) { |
1025 | assert(isKnownBaseResult(Base) &&((isKnownBaseResult(Base) && "must be something we 'know' is a base pointer" ) ? static_cast<void> (0) : __assert_fail ("isKnownBaseResult(Base) && \"must be something we 'know' is a base pointer\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1026, __PRETTY_FUNCTION__)) |
1026 | "must be something we 'know' is a base pointer")((isKnownBaseResult(Base) && "must be something we 'know' is a base pointer" ) ? static_cast<void> (0) : __assert_fail ("isKnownBaseResult(Base) && \"must be something we 'know' is a base pointer\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1026, __PRETTY_FUNCTION__)); |
1027 | // Once we transition from the BDV relation being store in the Cache to |
1028 | // the base relation being stored, it must be stable |
1029 | assert((!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) &&(((!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) && "base relation should be stable") ? static_cast<void> ( 0) : __assert_fail ("(!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) && \"base relation should be stable\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1030, __PRETTY_FUNCTION__)) |
1030 | "base relation should be stable")(((!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) && "base relation should be stable") ? static_cast<void> ( 0) : __assert_fail ("(!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) && \"base relation should be stable\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1030, __PRETTY_FUNCTION__)); |
1031 | } |
1032 | Cache[BDV] = Base; |
1033 | } |
1034 | assert(Cache.count(Def))((Cache.count(Def)) ? static_cast<void> (0) : __assert_fail ("Cache.count(Def)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1034, __PRETTY_FUNCTION__)); |
1035 | return Cache[Def]; |
1036 | } |
1037 | |
1038 | // For a set of live pointers (base and/or derived), identify the base |
1039 | // pointer of the object which they are derived from. This routine will |
1040 | // mutate the IR graph as needed to make the 'base' pointer live at the |
1041 | // definition site of 'derived'. This ensures that any use of 'derived' can |
1042 | // also use 'base'. This may involve the insertion of a number of |
1043 | // additional PHI nodes. |
1044 | // |
1045 | // preconditions: live is a set of pointer type Values |
1046 | // |
1047 | // side effects: may insert PHI nodes into the existing CFG, will preserve |
1048 | // CFG, will not remove or mutate any existing nodes |
1049 | // |
1050 | // post condition: PointerToBase contains one (derived, base) pair for every |
1051 | // pointer in live. Note that derived can be equal to base if the original |
1052 | // pointer was a base pointer. |
1053 | static void |
1054 | findBasePointers(const StatepointLiveSetTy &live, |
1055 | MapVector<Value *, Value *> &PointerToBase, |
1056 | DominatorTree *DT, DefiningValueMapTy &DVCache) { |
1057 | for (Value *ptr : live) { |
1058 | Value *base = findBasePointer(ptr, DVCache); |
1059 | assert(base && "failed to find base pointer")((base && "failed to find base pointer") ? static_cast <void> (0) : __assert_fail ("base && \"failed to find base pointer\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1059, __PRETTY_FUNCTION__)); |
1060 | PointerToBase[ptr] = base; |
1061 | assert((!isa<Instruction>(base) || !isa<Instruction>(ptr) ||(((!isa<Instruction>(base) || !isa<Instruction>(ptr ) || DT->dominates(cast<Instruction>(base)->getParent (), cast<Instruction>(ptr)->getParent())) && "The base we found better dominate the derived pointer") ? static_cast <void> (0) : __assert_fail ("(!isa<Instruction>(base) || !isa<Instruction>(ptr) || DT->dominates(cast<Instruction>(base)->getParent(), cast<Instruction>(ptr)->getParent())) && \"The base we found better dominate the derived pointer\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1064, __PRETTY_FUNCTION__)) |
1062 | DT->dominates(cast<Instruction>(base)->getParent(),(((!isa<Instruction>(base) || !isa<Instruction>(ptr ) || DT->dominates(cast<Instruction>(base)->getParent (), cast<Instruction>(ptr)->getParent())) && "The base we found better dominate the derived pointer") ? static_cast <void> (0) : __assert_fail ("(!isa<Instruction>(base) || !isa<Instruction>(ptr) || DT->dominates(cast<Instruction>(base)->getParent(), cast<Instruction>(ptr)->getParent())) && \"The base we found better dominate the derived pointer\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1064, __PRETTY_FUNCTION__)) |
1063 | cast<Instruction>(ptr)->getParent())) &&(((!isa<Instruction>(base) || !isa<Instruction>(ptr ) || DT->dominates(cast<Instruction>(base)->getParent (), cast<Instruction>(ptr)->getParent())) && "The base we found better dominate the derived pointer") ? static_cast <void> (0) : __assert_fail ("(!isa<Instruction>(base) || !isa<Instruction>(ptr) || DT->dominates(cast<Instruction>(base)->getParent(), cast<Instruction>(ptr)->getParent())) && \"The base we found better dominate the derived pointer\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1064, __PRETTY_FUNCTION__)) |
1064 | "The base we found better dominate the derived pointer")(((!isa<Instruction>(base) || !isa<Instruction>(ptr ) || DT->dominates(cast<Instruction>(base)->getParent (), cast<Instruction>(ptr)->getParent())) && "The base we found better dominate the derived pointer") ? static_cast <void> (0) : __assert_fail ("(!isa<Instruction>(base) || !isa<Instruction>(ptr) || DT->dominates(cast<Instruction>(base)->getParent(), cast<Instruction>(ptr)->getParent())) && \"The base we found better dominate the derived pointer\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1064, __PRETTY_FUNCTION__)); |
1065 | } |
1066 | } |
1067 | |
1068 | /// Find the required based pointers (and adjust the live set) for the given |
1069 | /// parse point. |
1070 | static void findBasePointers(DominatorTree &DT, DefiningValueMapTy &DVCache, |
1071 | CallSite CS, |
1072 | PartiallyConstructedSafepointRecord &result) { |
1073 | MapVector<Value *, Value *> PointerToBase; |
1074 | findBasePointers(result.LiveSet, PointerToBase, &DT, DVCache); |
1075 | |
1076 | if (PrintBasePointers) { |
1077 | errs() << "Base Pairs (w/o Relocation):\n"; |
1078 | for (auto &Pair : PointerToBase) { |
1079 | errs() << " derived "; |
1080 | Pair.first->printAsOperand(errs(), false); |
1081 | errs() << " base "; |
1082 | Pair.second->printAsOperand(errs(), false); |
1083 | errs() << "\n";; |
1084 | } |
1085 | } |
1086 | |
1087 | result.PointerToBase = PointerToBase; |
1088 | } |
1089 | |
1090 | /// Given an updated version of the dataflow liveness results, update the |
1091 | /// liveset and base pointer maps for the call site CS. |
1092 | static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, |
1093 | CallSite CS, |
1094 | PartiallyConstructedSafepointRecord &result); |
1095 | |
1096 | static void recomputeLiveInValues( |
1097 | Function &F, DominatorTree &DT, ArrayRef<CallSite> toUpdate, |
1098 | MutableArrayRef<struct PartiallyConstructedSafepointRecord> records) { |
1099 | // TODO-PERF: reuse the original liveness, then simply run the dataflow |
1100 | // again. The old values are still live and will help it stabilize quickly. |
1101 | GCPtrLivenessData RevisedLivenessData; |
1102 | computeLiveInValues(DT, F, RevisedLivenessData); |
1103 | for (size_t i = 0; i < records.size(); i++) { |
1104 | struct PartiallyConstructedSafepointRecord &info = records[i]; |
1105 | recomputeLiveInValues(RevisedLivenessData, toUpdate[i], info); |
1106 | } |
1107 | } |
1108 | |
1109 | // When inserting gc.relocate and gc.result calls, we need to ensure there are |
1110 | // no uses of the original value / return value between the gc.statepoint and |
1111 | // the gc.relocate / gc.result call. One case which can arise is a phi node |
1112 | // starting one of the successor blocks. We also need to be able to insert the |
1113 | // gc.relocates only on the path which goes through the statepoint. We might |
1114 | // need to split an edge to make this possible. |
1115 | static BasicBlock * |
1116 | normalizeForInvokeSafepoint(BasicBlock *BB, BasicBlock *InvokeParent, |
1117 | DominatorTree &DT) { |
1118 | BasicBlock *Ret = BB; |
1119 | if (!BB->getUniquePredecessor()) |
1120 | Ret = SplitBlockPredecessors(BB, InvokeParent, "", &DT); |
1121 | |
1122 | // Now that 'Ret' has unique predecessor we can safely remove all phi nodes |
1123 | // from it |
1124 | FoldSingleEntryPHINodes(Ret); |
1125 | assert(!isa<PHINode>(Ret->begin()) &&((!isa<PHINode>(Ret->begin()) && "All PHI nodes should have been removed!" ) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(Ret->begin()) && \"All PHI nodes should have been removed!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1126, __PRETTY_FUNCTION__)) |
1126 | "All PHI nodes should have been removed!")((!isa<PHINode>(Ret->begin()) && "All PHI nodes should have been removed!" ) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(Ret->begin()) && \"All PHI nodes should have been removed!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1126, __PRETTY_FUNCTION__)); |
1127 | |
1128 | // At this point, we can safely insert a gc.relocate or gc.result as the first |
1129 | // instruction in Ret if needed. |
1130 | return Ret; |
1131 | } |
1132 | |
1133 | // Create new attribute set containing only attributes which can be transferred |
1134 | // from original call to the safepoint. |
1135 | static AttributeList legalizeCallAttributes(AttributeList AL) { |
1136 | if (AL.isEmpty()) |
1137 | return AL; |
1138 | |
1139 | // Remove the readonly, readnone, and statepoint function attributes. |
1140 | AttrBuilder FnAttrs = AL.getFnAttributes(); |
1141 | FnAttrs.removeAttribute(Attribute::ReadNone); |
1142 | FnAttrs.removeAttribute(Attribute::ReadOnly); |
1143 | for (Attribute A : AL.getFnAttributes()) { |
1144 | if (isStatepointDirectiveAttr(A)) |
1145 | FnAttrs.remove(A); |
1146 | } |
1147 | |
1148 | // Just skip parameter and return attributes for now |
1149 | LLVMContext &Ctx = AL.getContext(); |
1150 | return AttributeList::get(Ctx, AttributeList::FunctionIndex, |
1151 | AttributeSet::get(Ctx, FnAttrs)); |
1152 | } |
1153 | |
1154 | /// Helper function to place all gc relocates necessary for the given |
1155 | /// statepoint. |
1156 | /// Inputs: |
1157 | /// liveVariables - list of variables to be relocated. |
1158 | /// liveStart - index of the first live variable. |
1159 | /// basePtrs - base pointers. |
1160 | /// statepointToken - statepoint instruction to which relocates should be |
1161 | /// bound. |
1162 | /// Builder - Llvm IR builder to be used to construct new calls. |
1163 | static void CreateGCRelocates(ArrayRef<Value *> LiveVariables, |
1164 | const int LiveStart, |
1165 | ArrayRef<Value *> BasePtrs, |
1166 | Instruction *StatepointToken, |
1167 | IRBuilder<> Builder) { |
1168 | if (LiveVariables.empty()) |
1169 | return; |
1170 | |
1171 | auto FindIndex = [](ArrayRef<Value *> LiveVec, Value *Val) { |
1172 | auto ValIt = find(LiveVec, Val); |
1173 | assert(ValIt != LiveVec.end() && "Val not found in LiveVec!")((ValIt != LiveVec.end() && "Val not found in LiveVec!" ) ? static_cast<void> (0) : __assert_fail ("ValIt != LiveVec.end() && \"Val not found in LiveVec!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1173, __PRETTY_FUNCTION__)); |
1174 | size_t Index = std::distance(LiveVec.begin(), ValIt); |
1175 | assert(Index < LiveVec.size() && "Bug in std::find?")((Index < LiveVec.size() && "Bug in std::find?") ? static_cast<void> (0) : __assert_fail ("Index < LiveVec.size() && \"Bug in std::find?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1175, __PRETTY_FUNCTION__)); |
1176 | return Index; |
1177 | }; |
1178 | Module *M = StatepointToken->getModule(); |
1179 | |
1180 | // All gc_relocate are generated as i8 addrspace(1)* (or a vector type whose |
1181 | // element type is i8 addrspace(1)*). We originally generated unique |
1182 | // declarations for each pointer type, but this proved problematic because |
1183 | // the intrinsic mangling code is incomplete and fragile. Since we're moving |
1184 | // towards a single unified pointer type anyways, we can just cast everything |
1185 | // to an i8* of the right address space. A bitcast is added later to convert |
1186 | // gc_relocate to the actual value's type. |
1187 | auto getGCRelocateDecl = [&] (Type *Ty) { |
1188 | assert(isHandledGCPointerType(Ty))((isHandledGCPointerType(Ty)) ? static_cast<void> (0) : __assert_fail ("isHandledGCPointerType(Ty)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1188, __PRETTY_FUNCTION__)); |
1189 | auto AS = Ty->getScalarType()->getPointerAddressSpace(); |
1190 | Type *NewTy = Type::getInt8PtrTy(M->getContext(), AS); |
1191 | if (auto *VT = dyn_cast<VectorType>(Ty)) |
1192 | NewTy = VectorType::get(NewTy, VT->getNumElements()); |
1193 | return Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, |
1194 | {NewTy}); |
1195 | }; |
1196 | |
1197 | // Lazily populated map from input types to the canonicalized form mentioned |
1198 | // in the comment above. This should probably be cached somewhere more |
1199 | // broadly. |
1200 | DenseMap<Type*, Value*> TypeToDeclMap; |
1201 | |
1202 | for (unsigned i = 0; i < LiveVariables.size(); i++) { |
1203 | // Generate the gc.relocate call and save the result |
1204 | Value *BaseIdx = |
1205 | Builder.getInt32(LiveStart + FindIndex(LiveVariables, BasePtrs[i])); |
1206 | Value *LiveIdx = Builder.getInt32(LiveStart + i); |
1207 | |
1208 | Type *Ty = LiveVariables[i]->getType(); |
1209 | if (!TypeToDeclMap.count(Ty)) |
1210 | TypeToDeclMap[Ty] = getGCRelocateDecl(Ty); |
1211 | Value *GCRelocateDecl = TypeToDeclMap[Ty]; |
1212 | |
1213 | // only specify a debug name if we can give a useful one |
1214 | CallInst *Reloc = Builder.CreateCall( |
1215 | GCRelocateDecl, {StatepointToken, BaseIdx, LiveIdx}, |
1216 | suffixed_name_or(LiveVariables[i], ".relocated", "")); |
1217 | // Trick CodeGen into thinking there are lots of free registers at this |
1218 | // fake call. |
1219 | Reloc->setCallingConv(CallingConv::Cold); |
1220 | } |
1221 | } |
1222 | |
1223 | namespace { |
1224 | |
1225 | /// This struct is used to defer RAUWs and `eraseFromParent` s. Using this |
1226 | /// avoids having to worry about keeping around dangling pointers to Values. |
1227 | class DeferredReplacement { |
1228 | AssertingVH<Instruction> Old; |
1229 | AssertingVH<Instruction> New; |
1230 | bool IsDeoptimize = false; |
1231 | |
1232 | DeferredReplacement() {} |
1233 | |
1234 | public: |
1235 | static DeferredReplacement createRAUW(Instruction *Old, Instruction *New) { |
1236 | assert(Old != New && Old && New &&((Old != New && Old && New && "Cannot RAUW equal values or to / from null!" ) ? static_cast<void> (0) : __assert_fail ("Old != New && Old && New && \"Cannot RAUW equal values or to / from null!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1237, __PRETTY_FUNCTION__)) |
1237 | "Cannot RAUW equal values or to / from null!")((Old != New && Old && New && "Cannot RAUW equal values or to / from null!" ) ? static_cast<void> (0) : __assert_fail ("Old != New && Old && New && \"Cannot RAUW equal values or to / from null!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1237, __PRETTY_FUNCTION__)); |
1238 | |
1239 | DeferredReplacement D; |
1240 | D.Old = Old; |
1241 | D.New = New; |
1242 | return D; |
1243 | } |
1244 | |
1245 | static DeferredReplacement createDelete(Instruction *ToErase) { |
1246 | DeferredReplacement D; |
1247 | D.Old = ToErase; |
1248 | return D; |
1249 | } |
1250 | |
1251 | static DeferredReplacement createDeoptimizeReplacement(Instruction *Old) { |
1252 | #ifndef NDEBUG |
1253 | auto *F = cast<CallInst>(Old)->getCalledFunction(); |
1254 | assert(F && F->getIntrinsicID() == Intrinsic::experimental_deoptimize &&((F && F->getIntrinsicID() == Intrinsic::experimental_deoptimize && "Only way to construct a deoptimize deferred replacement" ) ? static_cast<void> (0) : __assert_fail ("F && F->getIntrinsicID() == Intrinsic::experimental_deoptimize && \"Only way to construct a deoptimize deferred replacement\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1255, __PRETTY_FUNCTION__)) |
1255 | "Only way to construct a deoptimize deferred replacement")((F && F->getIntrinsicID() == Intrinsic::experimental_deoptimize && "Only way to construct a deoptimize deferred replacement" ) ? static_cast<void> (0) : __assert_fail ("F && F->getIntrinsicID() == Intrinsic::experimental_deoptimize && \"Only way to construct a deoptimize deferred replacement\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1255, __PRETTY_FUNCTION__)); |
1256 | #endif |
1257 | DeferredReplacement D; |
1258 | D.Old = Old; |
1259 | D.IsDeoptimize = true; |
1260 | return D; |
1261 | } |
1262 | |
1263 | /// Does the task represented by this instance. |
1264 | void doReplacement() { |
1265 | Instruction *OldI = Old; |
1266 | Instruction *NewI = New; |
1267 | |
1268 | assert(OldI != NewI && "Disallowed at construction?!")((OldI != NewI && "Disallowed at construction?!") ? static_cast <void> (0) : __assert_fail ("OldI != NewI && \"Disallowed at construction?!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1268, __PRETTY_FUNCTION__)); |
1269 | assert((!IsDeoptimize || !New) &&(((!IsDeoptimize || !New) && "Deoptimize instrinsics are not replaced!" ) ? static_cast<void> (0) : __assert_fail ("(!IsDeoptimize || !New) && \"Deoptimize instrinsics are not replaced!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1270, __PRETTY_FUNCTION__)) |
1270 | "Deoptimize instrinsics are not replaced!")(((!IsDeoptimize || !New) && "Deoptimize instrinsics are not replaced!" ) ? static_cast<void> (0) : __assert_fail ("(!IsDeoptimize || !New) && \"Deoptimize instrinsics are not replaced!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1270, __PRETTY_FUNCTION__)); |
1271 | |
1272 | Old = nullptr; |
1273 | New = nullptr; |
1274 | |
1275 | if (NewI) |
1276 | OldI->replaceAllUsesWith(NewI); |
1277 | |
1278 | if (IsDeoptimize) { |
1279 | // Note: we've inserted instructions, so the call to llvm.deoptimize may |
1280 | // not necessarilly be followed by the matching return. |
1281 | auto *RI = cast<ReturnInst>(OldI->getParent()->getTerminator()); |
1282 | new UnreachableInst(RI->getContext(), RI); |
1283 | RI->eraseFromParent(); |
1284 | } |
1285 | |
1286 | OldI->eraseFromParent(); |
1287 | } |
1288 | }; |
1289 | } |
1290 | |
1291 | static StringRef getDeoptLowering(CallSite CS) { |
1292 | const char *DeoptLowering = "deopt-lowering"; |
1293 | if (CS.hasFnAttr(DeoptLowering)) { |
1294 | // FIXME: CallSite has a *really* confusing interface around attributes |
1295 | // with values. |
1296 | const AttributeList &CSAS = CS.getAttributes(); |
1297 | if (CSAS.hasAttribute(AttributeList::FunctionIndex, DeoptLowering)) |
1298 | return CSAS.getAttribute(AttributeList::FunctionIndex, DeoptLowering) |
1299 | .getValueAsString(); |
1300 | Function *F = CS.getCalledFunction(); |
1301 | assert(F && F->hasFnAttribute(DeoptLowering))((F && F->hasFnAttribute(DeoptLowering)) ? static_cast <void> (0) : __assert_fail ("F && F->hasFnAttribute(DeoptLowering)" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1301, __PRETTY_FUNCTION__)); |
1302 | return F->getFnAttribute(DeoptLowering).getValueAsString(); |
1303 | } |
1304 | return "live-through"; |
1305 | } |
1306 | |
1307 | |
1308 | static void |
1309 | makeStatepointExplicitImpl(const CallSite CS, /* to replace */ |
1310 | const SmallVectorImpl<Value *> &BasePtrs, |
1311 | const SmallVectorImpl<Value *> &LiveVariables, |
1312 | PartiallyConstructedSafepointRecord &Result, |
1313 | std::vector<DeferredReplacement> &Replacements) { |
1314 | assert(BasePtrs.size() == LiveVariables.size())((BasePtrs.size() == LiveVariables.size()) ? static_cast<void > (0) : __assert_fail ("BasePtrs.size() == LiveVariables.size()" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1314, __PRETTY_FUNCTION__)); |
1315 | |
1316 | // Then go ahead and use the builder do actually do the inserts. We insert |
1317 | // immediately before the previous instruction under the assumption that all |
1318 | // arguments will be available here. We can't insert afterwards since we may |
1319 | // be replacing a terminator. |
1320 | Instruction *InsertBefore = CS.getInstruction(); |
1321 | IRBuilder<> Builder(InsertBefore); |
1322 | |
1323 | ArrayRef<Value *> GCArgs(LiveVariables); |
1324 | uint64_t StatepointID = StatepointDirectives::DefaultStatepointID; |
1325 | uint32_t NumPatchBytes = 0; |
1326 | uint32_t Flags = uint32_t(StatepointFlags::None); |
1327 | |
1328 | ArrayRef<Use> CallArgs(CS.arg_begin(), CS.arg_end()); |
1329 | ArrayRef<Use> DeoptArgs = GetDeoptBundleOperands(CS); |
1330 | ArrayRef<Use> TransitionArgs; |
1331 | if (auto TransitionBundle = |
1332 | CS.getOperandBundle(LLVMContext::OB_gc_transition)) { |
1333 | Flags |= uint32_t(StatepointFlags::GCTransition); |
1334 | TransitionArgs = TransitionBundle->Inputs; |
1335 | } |
1336 | |
1337 | // Instead of lowering calls to @llvm.experimental.deoptimize as normal calls |
1338 | // with a return value, we lower then as never returning calls to |
1339 | // __llvm_deoptimize that are followed by unreachable to get better codegen. |
1340 | bool IsDeoptimize = false; |
1341 | |
1342 | StatepointDirectives SD = |
1343 | parseStatepointDirectivesFromAttrs(CS.getAttributes()); |
1344 | if (SD.NumPatchBytes) |
1345 | NumPatchBytes = *SD.NumPatchBytes; |
1346 | if (SD.StatepointID) |
1347 | StatepointID = *SD.StatepointID; |
1348 | |
1349 | // Pass through the requested lowering if any. The default is live-through. |
1350 | StringRef DeoptLowering = getDeoptLowering(CS); |
1351 | if (DeoptLowering.equals("live-in")) |
1352 | Flags |= uint32_t(StatepointFlags::DeoptLiveIn); |
1353 | else { |
1354 | assert(DeoptLowering.equals("live-through") && "Unsupported value!")((DeoptLowering.equals("live-through") && "Unsupported value!" ) ? static_cast<void> (0) : __assert_fail ("DeoptLowering.equals(\"live-through\") && \"Unsupported value!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1354, __PRETTY_FUNCTION__)); |
1355 | } |
1356 | |
1357 | Value *CallTarget = CS.getCalledValue(); |
1358 | if (Function *F = dyn_cast<Function>(CallTarget)) { |
1359 | if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize) { |
1360 | // Calls to llvm.experimental.deoptimize are lowered to calls to the |
1361 | // __llvm_deoptimize symbol. We want to resolve this now, since the |
1362 | // verifier does not allow taking the address of an intrinsic function. |
1363 | |
1364 | SmallVector<Type *, 8> DomainTy; |
1365 | for (Value *Arg : CallArgs) |
1366 | DomainTy.push_back(Arg->getType()); |
1367 | auto *FTy = FunctionType::get(Type::getVoidTy(F->getContext()), DomainTy, |
1368 | /* isVarArg = */ false); |
1369 | |
1370 | // Note: CallTarget can be a bitcast instruction of a symbol if there are |
1371 | // calls to @llvm.experimental.deoptimize with different argument types in |
1372 | // the same module. This is fine -- we assume the frontend knew what it |
1373 | // was doing when generating this kind of IR. |
1374 | CallTarget = |
1375 | F->getParent()->getOrInsertFunction("__llvm_deoptimize", FTy); |
1376 | |
1377 | IsDeoptimize = true; |
1378 | } |
1379 | } |
1380 | |
1381 | // Create the statepoint given all the arguments |
1382 | Instruction *Token = nullptr; |
1383 | if (CS.isCall()) { |
1384 | CallInst *ToReplace = cast<CallInst>(CS.getInstruction()); |
1385 | CallInst *Call = Builder.CreateGCStatepointCall( |
1386 | StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs, |
1387 | TransitionArgs, DeoptArgs, GCArgs, "safepoint_token"); |
1388 | |
1389 | Call->setTailCallKind(ToReplace->getTailCallKind()); |
1390 | Call->setCallingConv(ToReplace->getCallingConv()); |
1391 | |
1392 | // Currently we will fail on parameter attributes and on certain |
1393 | // function attributes. In case if we can handle this set of attributes - |
1394 | // set up function attrs directly on statepoint and return attrs later for |
1395 | // gc_result intrinsic. |
1396 | Call->setAttributes(legalizeCallAttributes(ToReplace->getAttributes())); |
1397 | |
1398 | Token = Call; |
1399 | |
1400 | // Put the following gc_result and gc_relocate calls immediately after the |
1401 | // the old call (which we're about to delete) |
1402 | assert(ToReplace->getNextNode() && "Not a terminator, must have next!")((ToReplace->getNextNode() && "Not a terminator, must have next!" ) ? static_cast<void> (0) : __assert_fail ("ToReplace->getNextNode() && \"Not a terminator, must have next!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1402, __PRETTY_FUNCTION__)); |
1403 | Builder.SetInsertPoint(ToReplace->getNextNode()); |
1404 | Builder.SetCurrentDebugLocation(ToReplace->getNextNode()->getDebugLoc()); |
1405 | } else { |
1406 | InvokeInst *ToReplace = cast<InvokeInst>(CS.getInstruction()); |
1407 | |
1408 | // Insert the new invoke into the old block. We'll remove the old one in a |
1409 | // moment at which point this will become the new terminator for the |
1410 | // original block. |
1411 | InvokeInst *Invoke = Builder.CreateGCStatepointInvoke( |
1412 | StatepointID, NumPatchBytes, CallTarget, ToReplace->getNormalDest(), |
1413 | ToReplace->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs, |
1414 | GCArgs, "statepoint_token"); |
1415 | |
1416 | Invoke->setCallingConv(ToReplace->getCallingConv()); |
1417 | |
1418 | // Currently we will fail on parameter attributes and on certain |
1419 | // function attributes. In case if we can handle this set of attributes - |
1420 | // set up function attrs directly on statepoint and return attrs later for |
1421 | // gc_result intrinsic. |
1422 | Invoke->setAttributes(legalizeCallAttributes(ToReplace->getAttributes())); |
1423 | |
1424 | Token = Invoke; |
1425 | |
1426 | // Generate gc relocates in exceptional path |
1427 | BasicBlock *UnwindBlock = ToReplace->getUnwindDest(); |
1428 | assert(!isa<PHINode>(UnwindBlock->begin()) &&((!isa<PHINode>(UnwindBlock->begin()) && UnwindBlock ->getUniquePredecessor() && "can't safely insert in this block!" ) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(UnwindBlock->begin()) && UnwindBlock->getUniquePredecessor() && \"can't safely insert in this block!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1430, __PRETTY_FUNCTION__)) |
1429 | UnwindBlock->getUniquePredecessor() &&((!isa<PHINode>(UnwindBlock->begin()) && UnwindBlock ->getUniquePredecessor() && "can't safely insert in this block!" ) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(UnwindBlock->begin()) && UnwindBlock->getUniquePredecessor() && \"can't safely insert in this block!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1430, __PRETTY_FUNCTION__)) |
1430 | "can't safely insert in this block!")((!isa<PHINode>(UnwindBlock->begin()) && UnwindBlock ->getUniquePredecessor() && "can't safely insert in this block!" ) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(UnwindBlock->begin()) && UnwindBlock->getUniquePredecessor() && \"can't safely insert in this block!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1430, __PRETTY_FUNCTION__)); |
1431 | |
1432 | Builder.SetInsertPoint(&*UnwindBlock->getFirstInsertionPt()); |
1433 | Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc()); |
1434 | |
1435 | // Attach exceptional gc relocates to the landingpad. |
1436 | Instruction *ExceptionalToken = UnwindBlock->getLandingPadInst(); |
1437 | Result.UnwindToken = ExceptionalToken; |
1438 | |
1439 | const unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx(); |
1440 | CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, ExceptionalToken, |
1441 | Builder); |
1442 | |
1443 | // Generate gc relocates and returns for normal block |
1444 | BasicBlock *NormalDest = ToReplace->getNormalDest(); |
1445 | assert(!isa<PHINode>(NormalDest->begin()) &&((!isa<PHINode>(NormalDest->begin()) && NormalDest ->getUniquePredecessor() && "can't safely insert in this block!" ) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(NormalDest->begin()) && NormalDest->getUniquePredecessor() && \"can't safely insert in this block!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1447, __PRETTY_FUNCTION__)) |
1446 | NormalDest->getUniquePredecessor() &&((!isa<PHINode>(NormalDest->begin()) && NormalDest ->getUniquePredecessor() && "can't safely insert in this block!" ) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(NormalDest->begin()) && NormalDest->getUniquePredecessor() && \"can't safely insert in this block!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1447, __PRETTY_FUNCTION__)) |
1447 | "can't safely insert in this block!")((!isa<PHINode>(NormalDest->begin()) && NormalDest ->getUniquePredecessor() && "can't safely insert in this block!" ) ? static_cast<void> (0) : __assert_fail ("!isa<PHINode>(NormalDest->begin()) && NormalDest->getUniquePredecessor() && \"can't safely insert in this block!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1447, __PRETTY_FUNCTION__)); |
1448 | |
1449 | Builder.SetInsertPoint(&*NormalDest->getFirstInsertionPt()); |
1450 | |
1451 | // gc relocates will be generated later as if it were regular call |
1452 | // statepoint |
1453 | } |
1454 | assert(Token && "Should be set in one of the above branches!")((Token && "Should be set in one of the above branches!" ) ? static_cast<void> (0) : __assert_fail ("Token && \"Should be set in one of the above branches!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1454, __PRETTY_FUNCTION__)); |
1455 | |
1456 | if (IsDeoptimize) { |
1457 | // If we're wrapping an @llvm.experimental.deoptimize in a statepoint, we |
1458 | // transform the tail-call like structure to a call to a void function |
1459 | // followed by unreachable to get better codegen. |
1460 | Replacements.push_back( |
1461 | DeferredReplacement::createDeoptimizeReplacement(CS.getInstruction())); |
1462 | } else { |
1463 | Token->setName("statepoint_token"); |
1464 | if (!CS.getType()->isVoidTy() && !CS.getInstruction()->use_empty()) { |
1465 | StringRef Name = |
1466 | CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : ""; |
1467 | CallInst *GCResult = Builder.CreateGCResult(Token, CS.getType(), Name); |
1468 | GCResult->setAttributes( |
1469 | AttributeList::get(GCResult->getContext(), AttributeList::ReturnIndex, |
1470 | CS.getAttributes().getRetAttributes())); |
1471 | |
1472 | // We cannot RAUW or delete CS.getInstruction() because it could be in the |
1473 | // live set of some other safepoint, in which case that safepoint's |
1474 | // PartiallyConstructedSafepointRecord will hold a raw pointer to this |
1475 | // llvm::Instruction. Instead, we defer the replacement and deletion to |
1476 | // after the live sets have been made explicit in the IR, and we no longer |
1477 | // have raw pointers to worry about. |
1478 | Replacements.emplace_back( |
1479 | DeferredReplacement::createRAUW(CS.getInstruction(), GCResult)); |
1480 | } else { |
1481 | Replacements.emplace_back( |
1482 | DeferredReplacement::createDelete(CS.getInstruction())); |
1483 | } |
1484 | } |
1485 | |
1486 | Result.StatepointToken = Token; |
1487 | |
1488 | // Second, create a gc.relocate for every live variable |
1489 | const unsigned LiveStartIdx = Statepoint(Token).gcArgsStartIdx(); |
1490 | CreateGCRelocates(LiveVariables, LiveStartIdx, BasePtrs, Token, Builder); |
1491 | } |
1492 | |
1493 | // Replace an existing gc.statepoint with a new one and a set of gc.relocates |
1494 | // which make the relocations happening at this safepoint explicit. |
1495 | // |
1496 | // WARNING: Does not do any fixup to adjust users of the original live |
1497 | // values. That's the callers responsibility. |
1498 | static void |
1499 | makeStatepointExplicit(DominatorTree &DT, CallSite CS, |
1500 | PartiallyConstructedSafepointRecord &Result, |
1501 | std::vector<DeferredReplacement> &Replacements) { |
1502 | const auto &LiveSet = Result.LiveSet; |
1503 | const auto &PointerToBase = Result.PointerToBase; |
1504 | |
1505 | // Convert to vector for efficient cross referencing. |
1506 | SmallVector<Value *, 64> BaseVec, LiveVec; |
1507 | LiveVec.reserve(LiveSet.size()); |
1508 | BaseVec.reserve(LiveSet.size()); |
1509 | for (Value *L : LiveSet) { |
1510 | LiveVec.push_back(L); |
1511 | assert(PointerToBase.count(L))((PointerToBase.count(L)) ? static_cast<void> (0) : __assert_fail ("PointerToBase.count(L)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1511, __PRETTY_FUNCTION__)); |
1512 | Value *Base = PointerToBase.find(L)->second; |
1513 | BaseVec.push_back(Base); |
1514 | } |
1515 | assert(LiveVec.size() == BaseVec.size())((LiveVec.size() == BaseVec.size()) ? static_cast<void> (0) : __assert_fail ("LiveVec.size() == BaseVec.size()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1515, __PRETTY_FUNCTION__)); |
1516 | |
1517 | // Do the actual rewriting and delete the old statepoint |
1518 | makeStatepointExplicitImpl(CS, BaseVec, LiveVec, Result, Replacements); |
1519 | } |
1520 | |
1521 | // Helper function for the relocationViaAlloca. |
1522 | // |
1523 | // It receives iterator to the statepoint gc relocates and emits a store to the |
1524 | // assigned location (via allocaMap) for the each one of them. It adds the |
1525 | // visited values into the visitedLiveValues set, which we will later use them |
1526 | // for sanity checking. |
1527 | static void |
1528 | insertRelocationStores(iterator_range<Value::user_iterator> GCRelocs, |
1529 | DenseMap<Value *, Value *> &AllocaMap, |
1530 | DenseSet<Value *> &VisitedLiveValues) { |
1531 | |
1532 | for (User *U : GCRelocs) { |
1533 | GCRelocateInst *Relocate = dyn_cast<GCRelocateInst>(U); |
1534 | if (!Relocate) |
1535 | continue; |
1536 | |
1537 | Value *OriginalValue = Relocate->getDerivedPtr(); |
1538 | assert(AllocaMap.count(OriginalValue))((AllocaMap.count(OriginalValue)) ? static_cast<void> ( 0) : __assert_fail ("AllocaMap.count(OriginalValue)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1538, __PRETTY_FUNCTION__)); |
1539 | Value *Alloca = AllocaMap[OriginalValue]; |
1540 | |
1541 | // Emit store into the related alloca |
1542 | // All gc_relocates are i8 addrspace(1)* typed, and it must be bitcasted to |
1543 | // the correct type according to alloca. |
1544 | assert(Relocate->getNextNode() &&((Relocate->getNextNode() && "Should always have one since it's not a terminator" ) ? static_cast<void> (0) : __assert_fail ("Relocate->getNextNode() && \"Should always have one since it's not a terminator\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1545, __PRETTY_FUNCTION__)) |
1545 | "Should always have one since it's not a terminator")((Relocate->getNextNode() && "Should always have one since it's not a terminator" ) ? static_cast<void> (0) : __assert_fail ("Relocate->getNextNode() && \"Should always have one since it's not a terminator\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1545, __PRETTY_FUNCTION__)); |
1546 | IRBuilder<> Builder(Relocate->getNextNode()); |
1547 | Value *CastedRelocatedValue = |
1548 | Builder.CreateBitCast(Relocate, |
1549 | cast<AllocaInst>(Alloca)->getAllocatedType(), |
1550 | suffixed_name_or(Relocate, ".casted", "")); |
1551 | |
1552 | StoreInst *Store = new StoreInst(CastedRelocatedValue, Alloca); |
1553 | Store->insertAfter(cast<Instruction>(CastedRelocatedValue)); |
1554 | |
1555 | #ifndef NDEBUG |
1556 | VisitedLiveValues.insert(OriginalValue); |
1557 | #endif |
1558 | } |
1559 | } |
1560 | |
1561 | // Helper function for the "relocationViaAlloca". Similar to the |
1562 | // "insertRelocationStores" but works for rematerialized values. |
1563 | static void insertRematerializationStores( |
1564 | const RematerializedValueMapTy &RematerializedValues, |
1565 | DenseMap<Value *, Value *> &AllocaMap, |
1566 | DenseSet<Value *> &VisitedLiveValues) { |
1567 | |
1568 | for (auto RematerializedValuePair: RematerializedValues) { |
1569 | Instruction *RematerializedValue = RematerializedValuePair.first; |
1570 | Value *OriginalValue = RematerializedValuePair.second; |
1571 | |
1572 | assert(AllocaMap.count(OriginalValue) &&((AllocaMap.count(OriginalValue) && "Can not find alloca for rematerialized value" ) ? static_cast<void> (0) : __assert_fail ("AllocaMap.count(OriginalValue) && \"Can not find alloca for rematerialized value\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1573, __PRETTY_FUNCTION__)) |
1573 | "Can not find alloca for rematerialized value")((AllocaMap.count(OriginalValue) && "Can not find alloca for rematerialized value" ) ? static_cast<void> (0) : __assert_fail ("AllocaMap.count(OriginalValue) && \"Can not find alloca for rematerialized value\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1573, __PRETTY_FUNCTION__)); |
1574 | Value *Alloca = AllocaMap[OriginalValue]; |
1575 | |
1576 | StoreInst *Store = new StoreInst(RematerializedValue, Alloca); |
1577 | Store->insertAfter(RematerializedValue); |
1578 | |
1579 | #ifndef NDEBUG |
1580 | VisitedLiveValues.insert(OriginalValue); |
1581 | #endif |
1582 | } |
1583 | } |
1584 | |
1585 | /// Do all the relocation update via allocas and mem2reg |
1586 | static void relocationViaAlloca( |
1587 | Function &F, DominatorTree &DT, ArrayRef<Value *> Live, |
1588 | ArrayRef<PartiallyConstructedSafepointRecord> Records) { |
1589 | #ifndef NDEBUG |
1590 | // record initial number of (static) allocas; we'll check we have the same |
1591 | // number when we get done. |
1592 | int InitialAllocaNum = 0; |
1593 | for (Instruction &I : F.getEntryBlock()) |
1594 | if (isa<AllocaInst>(I)) |
1595 | InitialAllocaNum++; |
1596 | #endif |
1597 | |
1598 | // TODO-PERF: change data structures, reserve |
1599 | DenseMap<Value *, Value *> AllocaMap; |
1600 | SmallVector<AllocaInst *, 200> PromotableAllocas; |
1601 | // Used later to chack that we have enough allocas to store all values |
1602 | std::size_t NumRematerializedValues = 0; |
1603 | PromotableAllocas.reserve(Live.size()); |
1604 | |
1605 | // Emit alloca for "LiveValue" and record it in "allocaMap" and |
1606 | // "PromotableAllocas" |
1607 | const DataLayout &DL = F.getParent()->getDataLayout(); |
1608 | auto emitAllocaFor = [&](Value *LiveValue) { |
1609 | AllocaInst *Alloca = new AllocaInst(LiveValue->getType(), |
1610 | DL.getAllocaAddrSpace(), "", |
1611 | F.getEntryBlock().getFirstNonPHI()); |
1612 | AllocaMap[LiveValue] = Alloca; |
1613 | PromotableAllocas.push_back(Alloca); |
1614 | }; |
1615 | |
1616 | // Emit alloca for each live gc pointer |
1617 | for (Value *V : Live) |
1618 | emitAllocaFor(V); |
1619 | |
1620 | // Emit allocas for rematerialized values |
1621 | for (const auto &Info : Records) |
1622 | for (auto RematerializedValuePair : Info.RematerializedValues) { |
1623 | Value *OriginalValue = RematerializedValuePair.second; |
1624 | if (AllocaMap.count(OriginalValue) != 0) |
1625 | continue; |
1626 | |
1627 | emitAllocaFor(OriginalValue); |
1628 | ++NumRematerializedValues; |
1629 | } |
1630 | |
1631 | // The next two loops are part of the same conceptual operation. We need to |
1632 | // insert a store to the alloca after the original def and at each |
1633 | // redefinition. We need to insert a load before each use. These are split |
1634 | // into distinct loops for performance reasons. |
1635 | |
1636 | // Update gc pointer after each statepoint: either store a relocated value or |
1637 | // null (if no relocated value was found for this gc pointer and it is not a |
1638 | // gc_result). This must happen before we update the statepoint with load of |
1639 | // alloca otherwise we lose the link between statepoint and old def. |
1640 | for (const auto &Info : Records) { |
1641 | Value *Statepoint = Info.StatepointToken; |
1642 | |
1643 | // This will be used for consistency check |
1644 | DenseSet<Value *> VisitedLiveValues; |
1645 | |
1646 | // Insert stores for normal statepoint gc relocates |
1647 | insertRelocationStores(Statepoint->users(), AllocaMap, VisitedLiveValues); |
1648 | |
1649 | // In case if it was invoke statepoint |
1650 | // we will insert stores for exceptional path gc relocates. |
1651 | if (isa<InvokeInst>(Statepoint)) { |
1652 | insertRelocationStores(Info.UnwindToken->users(), AllocaMap, |
1653 | VisitedLiveValues); |
1654 | } |
1655 | |
1656 | // Do similar thing with rematerialized values |
1657 | insertRematerializationStores(Info.RematerializedValues, AllocaMap, |
1658 | VisitedLiveValues); |
1659 | |
1660 | if (ClobberNonLive) { |
1661 | // As a debugging aid, pretend that an unrelocated pointer becomes null at |
1662 | // the gc.statepoint. This will turn some subtle GC problems into |
1663 | // slightly easier to debug SEGVs. Note that on large IR files with |
1664 | // lots of gc.statepoints this is extremely costly both memory and time |
1665 | // wise. |
1666 | SmallVector<AllocaInst *, 64> ToClobber; |
1667 | for (auto Pair : AllocaMap) { |
1668 | Value *Def = Pair.first; |
1669 | AllocaInst *Alloca = cast<AllocaInst>(Pair.second); |
1670 | |
1671 | // This value was relocated |
1672 | if (VisitedLiveValues.count(Def)) { |
1673 | continue; |
1674 | } |
1675 | ToClobber.push_back(Alloca); |
1676 | } |
1677 | |
1678 | auto InsertClobbersAt = [&](Instruction *IP) { |
1679 | for (auto *AI : ToClobber) { |
1680 | auto PT = cast<PointerType>(AI->getAllocatedType()); |
1681 | Constant *CPN = ConstantPointerNull::get(PT); |
1682 | StoreInst *Store = new StoreInst(CPN, AI); |
1683 | Store->insertBefore(IP); |
1684 | } |
1685 | }; |
1686 | |
1687 | // Insert the clobbering stores. These may get intermixed with the |
1688 | // gc.results and gc.relocates, but that's fine. |
1689 | if (auto II = dyn_cast<InvokeInst>(Statepoint)) { |
1690 | InsertClobbersAt(&*II->getNormalDest()->getFirstInsertionPt()); |
1691 | InsertClobbersAt(&*II->getUnwindDest()->getFirstInsertionPt()); |
1692 | } else { |
1693 | InsertClobbersAt(cast<Instruction>(Statepoint)->getNextNode()); |
1694 | } |
1695 | } |
1696 | } |
1697 | |
1698 | // Update use with load allocas and add store for gc_relocated. |
1699 | for (auto Pair : AllocaMap) { |
1700 | Value *Def = Pair.first; |
1701 | Value *Alloca = Pair.second; |
1702 | |
1703 | // We pre-record the uses of allocas so that we dont have to worry about |
1704 | // later update that changes the user information.. |
1705 | |
1706 | SmallVector<Instruction *, 20> Uses; |
1707 | // PERF: trade a linear scan for repeated reallocation |
1708 | Uses.reserve(std::distance(Def->user_begin(), Def->user_end())); |
1709 | for (User *U : Def->users()) { |
1710 | if (!isa<ConstantExpr>(U)) { |
1711 | // If the def has a ConstantExpr use, then the def is either a |
1712 | // ConstantExpr use itself or null. In either case |
1713 | // (recursively in the first, directly in the second), the oop |
1714 | // it is ultimately dependent on is null and this particular |
1715 | // use does not need to be fixed up. |
1716 | Uses.push_back(cast<Instruction>(U)); |
1717 | } |
1718 | } |
1719 | |
1720 | std::sort(Uses.begin(), Uses.end()); |
1721 | auto Last = std::unique(Uses.begin(), Uses.end()); |
1722 | Uses.erase(Last, Uses.end()); |
1723 | |
1724 | for (Instruction *Use : Uses) { |
1725 | if (isa<PHINode>(Use)) { |
1726 | PHINode *Phi = cast<PHINode>(Use); |
1727 | for (unsigned i = 0; i < Phi->getNumIncomingValues(); i++) { |
1728 | if (Def == Phi->getIncomingValue(i)) { |
1729 | LoadInst *Load = new LoadInst( |
1730 | Alloca, "", Phi->getIncomingBlock(i)->getTerminator()); |
1731 | Phi->setIncomingValue(i, Load); |
1732 | } |
1733 | } |
1734 | } else { |
1735 | LoadInst *Load = new LoadInst(Alloca, "", Use); |
1736 | Use->replaceUsesOfWith(Def, Load); |
1737 | } |
1738 | } |
1739 | |
1740 | // Emit store for the initial gc value. Store must be inserted after load, |
1741 | // otherwise store will be in alloca's use list and an extra load will be |
1742 | // inserted before it. |
1743 | StoreInst *Store = new StoreInst(Def, Alloca); |
1744 | if (Instruction *Inst = dyn_cast<Instruction>(Def)) { |
1745 | if (InvokeInst *Invoke = dyn_cast<InvokeInst>(Inst)) { |
1746 | // InvokeInst is a TerminatorInst so the store need to be inserted |
1747 | // into its normal destination block. |
1748 | BasicBlock *NormalDest = Invoke->getNormalDest(); |
1749 | Store->insertBefore(NormalDest->getFirstNonPHI()); |
1750 | } else { |
1751 | assert(!Inst->isTerminator() &&((!Inst->isTerminator() && "The only TerminatorInst that can produce a value is " "InvokeInst which is handled above.") ? static_cast<void> (0) : __assert_fail ("!Inst->isTerminator() && \"The only TerminatorInst that can produce a value is \" \"InvokeInst which is handled above.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1753, __PRETTY_FUNCTION__)) |
1752 | "The only TerminatorInst that can produce a value is "((!Inst->isTerminator() && "The only TerminatorInst that can produce a value is " "InvokeInst which is handled above.") ? static_cast<void> (0) : __assert_fail ("!Inst->isTerminator() && \"The only TerminatorInst that can produce a value is \" \"InvokeInst which is handled above.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1753, __PRETTY_FUNCTION__)) |
1753 | "InvokeInst which is handled above.")((!Inst->isTerminator() && "The only TerminatorInst that can produce a value is " "InvokeInst which is handled above.") ? static_cast<void> (0) : __assert_fail ("!Inst->isTerminator() && \"The only TerminatorInst that can produce a value is \" \"InvokeInst which is handled above.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1753, __PRETTY_FUNCTION__)); |
1754 | Store->insertAfter(Inst); |
1755 | } |
1756 | } else { |
1757 | assert(isa<Argument>(Def))((isa<Argument>(Def)) ? static_cast<void> (0) : __assert_fail ("isa<Argument>(Def)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1757, __PRETTY_FUNCTION__)); |
1758 | Store->insertAfter(cast<Instruction>(Alloca)); |
1759 | } |
1760 | } |
1761 | |
1762 | assert(PromotableAllocas.size() == Live.size() + NumRematerializedValues &&((PromotableAllocas.size() == Live.size() + NumRematerializedValues && "we must have the same allocas with lives") ? static_cast <void> (0) : __assert_fail ("PromotableAllocas.size() == Live.size() + NumRematerializedValues && \"we must have the same allocas with lives\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1763, __PRETTY_FUNCTION__)) |
1763 | "we must have the same allocas with lives")((PromotableAllocas.size() == Live.size() + NumRematerializedValues && "we must have the same allocas with lives") ? static_cast <void> (0) : __assert_fail ("PromotableAllocas.size() == Live.size() + NumRematerializedValues && \"we must have the same allocas with lives\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1763, __PRETTY_FUNCTION__)); |
1764 | if (!PromotableAllocas.empty()) { |
1765 | // Apply mem2reg to promote alloca to SSA |
1766 | PromoteMemToReg(PromotableAllocas, DT); |
1767 | } |
1768 | |
1769 | #ifndef NDEBUG |
1770 | for (auto &I : F.getEntryBlock()) |
1771 | if (isa<AllocaInst>(I)) |
1772 | InitialAllocaNum--; |
1773 | assert(InitialAllocaNum == 0 && "We must not introduce any extra allocas")((InitialAllocaNum == 0 && "We must not introduce any extra allocas" ) ? static_cast<void> (0) : __assert_fail ("InitialAllocaNum == 0 && \"We must not introduce any extra allocas\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1773, __PRETTY_FUNCTION__)); |
1774 | #endif |
1775 | } |
1776 | |
1777 | /// Implement a unique function which doesn't require we sort the input |
1778 | /// vector. Doing so has the effect of changing the output of a couple of |
1779 | /// tests in ways which make them less useful in testing fused safepoints. |
1780 | template <typename T> static void unique_unsorted(SmallVectorImpl<T> &Vec) { |
1781 | SmallSet<T, 8> Seen; |
1782 | Vec.erase(remove_if(Vec, [&](const T &V) { return !Seen.insert(V).second; }), |
1783 | Vec.end()); |
1784 | } |
1785 | |
1786 | /// Insert holders so that each Value is obviously live through the entire |
1787 | /// lifetime of the call. |
1788 | static void insertUseHolderAfter(CallSite &CS, const ArrayRef<Value *> Values, |
1789 | SmallVectorImpl<CallInst *> &Holders) { |
1790 | if (Values.empty()) |
1791 | // No values to hold live, might as well not insert the empty holder |
1792 | return; |
1793 | |
1794 | Module *M = CS.getInstruction()->getModule(); |
1795 | // Use a dummy vararg function to actually hold the values live |
1796 | Function *Func = cast<Function>(M->getOrInsertFunction( |
1797 | "__tmp_use", FunctionType::get(Type::getVoidTy(M->getContext()), true))); |
1798 | if (CS.isCall()) { |
1799 | // For call safepoints insert dummy calls right after safepoint |
1800 | Holders.push_back(CallInst::Create(Func, Values, "", |
1801 | &*++CS.getInstruction()->getIterator())); |
1802 | return; |
1803 | } |
1804 | // For invoke safepooints insert dummy calls both in normal and |
1805 | // exceptional destination blocks |
1806 | auto *II = cast<InvokeInst>(CS.getInstruction()); |
1807 | Holders.push_back(CallInst::Create( |
1808 | Func, Values, "", &*II->getNormalDest()->getFirstInsertionPt())); |
1809 | Holders.push_back(CallInst::Create( |
1810 | Func, Values, "", &*II->getUnwindDest()->getFirstInsertionPt())); |
1811 | } |
1812 | |
1813 | static void findLiveReferences( |
1814 | Function &F, DominatorTree &DT, ArrayRef<CallSite> toUpdate, |
1815 | MutableArrayRef<struct PartiallyConstructedSafepointRecord> records) { |
1816 | GCPtrLivenessData OriginalLivenessData; |
1817 | computeLiveInValues(DT, F, OriginalLivenessData); |
1818 | for (size_t i = 0; i < records.size(); i++) { |
1819 | struct PartiallyConstructedSafepointRecord &info = records[i]; |
1820 | analyzeParsePointLiveness(DT, OriginalLivenessData, toUpdate[i], info); |
1821 | } |
1822 | } |
1823 | |
1824 | // Helper function for the "rematerializeLiveValues". It walks use chain |
1825 | // starting from the "CurrentValue" until it reaches the root of the chain, i.e. |
1826 | // the base or a value it cannot process. Only "simple" values are processed |
1827 | // (currently it is GEP's and casts). The returned root is examined by the |
1828 | // callers of findRematerializableChainToBasePointer. Fills "ChainToBase" array |
1829 | // with all visited values. |
1830 | static Value* findRematerializableChainToBasePointer( |
1831 | SmallVectorImpl<Instruction*> &ChainToBase, |
1832 | Value *CurrentValue) { |
1833 | |
1834 | if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(CurrentValue)) { |
1835 | ChainToBase.push_back(GEP); |
1836 | return findRematerializableChainToBasePointer(ChainToBase, |
1837 | GEP->getPointerOperand()); |
1838 | } |
1839 | |
1840 | if (CastInst *CI = dyn_cast<CastInst>(CurrentValue)) { |
1841 | if (!CI->isNoopCast(CI->getModule()->getDataLayout())) |
1842 | return CI; |
1843 | |
1844 | ChainToBase.push_back(CI); |
1845 | return findRematerializableChainToBasePointer(ChainToBase, |
1846 | CI->getOperand(0)); |
1847 | } |
1848 | |
1849 | // We have reached the root of the chain, which is either equal to the base or |
1850 | // is the first unsupported value along the use chain. |
1851 | return CurrentValue; |
1852 | } |
1853 | |
1854 | // Helper function for the "rematerializeLiveValues". Compute cost of the use |
1855 | // chain we are going to rematerialize. |
1856 | static unsigned |
1857 | chainToBasePointerCost(SmallVectorImpl<Instruction*> &Chain, |
1858 | TargetTransformInfo &TTI) { |
1859 | unsigned Cost = 0; |
1860 | |
1861 | for (Instruction *Instr : Chain) { |
1862 | if (CastInst *CI = dyn_cast<CastInst>(Instr)) { |
1863 | assert(CI->isNoopCast(CI->getModule()->getDataLayout()) &&((CI->isNoopCast(CI->getModule()->getDataLayout()) && "non noop cast is found during rematerialization") ? static_cast <void> (0) : __assert_fail ("CI->isNoopCast(CI->getModule()->getDataLayout()) && \"non noop cast is found during rematerialization\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1864, __PRETTY_FUNCTION__)) |
1864 | "non noop cast is found during rematerialization")((CI->isNoopCast(CI->getModule()->getDataLayout()) && "non noop cast is found during rematerialization") ? static_cast <void> (0) : __assert_fail ("CI->isNoopCast(CI->getModule()->getDataLayout()) && \"non noop cast is found during rematerialization\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1864, __PRETTY_FUNCTION__)); |
1865 | |
1866 | Type *SrcTy = CI->getOperand(0)->getType(); |
1867 | Cost += TTI.getCastInstrCost(CI->getOpcode(), CI->getType(), SrcTy, CI); |
1868 | |
1869 | } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Instr)) { |
1870 | // Cost of the address calculation |
1871 | Type *ValTy = GEP->getSourceElementType(); |
1872 | Cost += TTI.getAddressComputationCost(ValTy); |
1873 | |
1874 | // And cost of the GEP itself |
1875 | // TODO: Use TTI->getGEPCost here (it exists, but appears to be not |
1876 | // allowed for the external usage) |
1877 | if (!GEP->hasAllConstantIndices()) |
1878 | Cost += 2; |
1879 | |
1880 | } else { |
1881 | llvm_unreachable("unsupported instruciton type during rematerialization")::llvm::llvm_unreachable_internal("unsupported instruciton type during rematerialization" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1881); |
1882 | } |
1883 | } |
1884 | |
1885 | return Cost; |
1886 | } |
1887 | |
1888 | static bool AreEquivalentPhiNodes(PHINode &OrigRootPhi, PHINode &AlternateRootPhi) { |
1889 | |
1890 | unsigned PhiNum = OrigRootPhi.getNumIncomingValues(); |
1891 | if (PhiNum != AlternateRootPhi.getNumIncomingValues() || |
1892 | OrigRootPhi.getParent() != AlternateRootPhi.getParent()) |
1893 | return false; |
1894 | // Map of incoming values and their corresponding basic blocks of |
1895 | // OrigRootPhi. |
1896 | SmallDenseMap<Value *, BasicBlock *, 8> CurrentIncomingValues; |
1897 | for (unsigned i = 0; i < PhiNum; i++) |
1898 | CurrentIncomingValues[OrigRootPhi.getIncomingValue(i)] = |
1899 | OrigRootPhi.getIncomingBlock(i); |
1900 | |
1901 | // Both current and base PHIs should have same incoming values and |
1902 | // the same basic blocks corresponding to the incoming values. |
1903 | for (unsigned i = 0; i < PhiNum; i++) { |
1904 | auto CIVI = |
1905 | CurrentIncomingValues.find(AlternateRootPhi.getIncomingValue(i)); |
1906 | if (CIVI == CurrentIncomingValues.end()) |
1907 | return false; |
1908 | BasicBlock *CurrentIncomingBB = CIVI->second; |
1909 | if (CurrentIncomingBB != AlternateRootPhi.getIncomingBlock(i)) |
1910 | return false; |
1911 | } |
1912 | return true; |
1913 | |
1914 | } |
1915 | |
1916 | // From the statepoint live set pick values that are cheaper to recompute then |
1917 | // to relocate. Remove this values from the live set, rematerialize them after |
1918 | // statepoint and record them in "Info" structure. Note that similar to |
1919 | // relocated values we don't do any user adjustments here. |
1920 | static void rematerializeLiveValues(CallSite CS, |
1921 | PartiallyConstructedSafepointRecord &Info, |
1922 | TargetTransformInfo &TTI) { |
1923 | const unsigned int ChainLengthThreshold = 10; |
1924 | |
1925 | // Record values we are going to delete from this statepoint live set. |
1926 | // We can not di this in following loop due to iterator invalidation. |
1927 | SmallVector<Value *, 32> LiveValuesToBeDeleted; |
1928 | |
1929 | for (Value *LiveValue: Info.LiveSet) { |
1930 | // For each live pointer find it's defining chain |
1931 | SmallVector<Instruction *, 3> ChainToBase; |
1932 | assert(Info.PointerToBase.count(LiveValue))((Info.PointerToBase.count(LiveValue)) ? static_cast<void> (0) : __assert_fail ("Info.PointerToBase.count(LiveValue)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1932, __PRETTY_FUNCTION__)); |
1933 | Value *RootOfChain = |
1934 | findRematerializableChainToBasePointer(ChainToBase, |
1935 | LiveValue); |
1936 | |
1937 | // Nothing to do, or chain is too long |
1938 | if ( ChainToBase.size() == 0 || |
1939 | ChainToBase.size() > ChainLengthThreshold) |
1940 | continue; |
1941 | |
1942 | // Handle the scenario where the RootOfChain is not equal to the |
1943 | // Base Value, but they are essentially the same phi values. |
1944 | if (RootOfChain != Info.PointerToBase[LiveValue]) { |
1945 | PHINode *OrigRootPhi = dyn_cast<PHINode>(RootOfChain); |
1946 | PHINode *AlternateRootPhi = dyn_cast<PHINode>(Info.PointerToBase[LiveValue]); |
1947 | if (!OrigRootPhi || !AlternateRootPhi) |
1948 | continue; |
1949 | // PHI nodes that have the same incoming values, and belonging to the same |
1950 | // basic blocks are essentially the same SSA value. When the original phi |
1951 | // has incoming values with different base pointers, the original phi is |
1952 | // marked as conflict, and an additional `AlternateRootPhi` with the same |
1953 | // incoming values get generated by the findBasePointer function. We need |
1954 | // to identify the newly generated AlternateRootPhi (.base version of phi) |
1955 | // and RootOfChain (the original phi node itself) are the same, so that we |
1956 | // can rematerialize the gep and casts. This is a workaround for the |
1957 | // deficieny in the findBasePointer algorithm. |
1958 | if (!AreEquivalentPhiNodes(*OrigRootPhi, *AlternateRootPhi)) |
1959 | continue; |
1960 | // Now that the phi nodes are proved to be the same, assert that |
1961 | // findBasePointer's newly generated AlternateRootPhi is present in the |
1962 | // liveset of the call. |
1963 | assert(Info.LiveSet.count(AlternateRootPhi))((Info.LiveSet.count(AlternateRootPhi)) ? static_cast<void > (0) : __assert_fail ("Info.LiveSet.count(AlternateRootPhi)" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 1963, __PRETTY_FUNCTION__)); |
1964 | } |
1965 | // Compute cost of this chain |
1966 | unsigned Cost = chainToBasePointerCost(ChainToBase, TTI); |
1967 | // TODO: We can also account for cases when we will be able to remove some |
1968 | // of the rematerialized values by later optimization passes. I.e if |
1969 | // we rematerialized several intersecting chains. Or if original values |
1970 | // don't have any uses besides this statepoint. |
1971 | |
1972 | // For invokes we need to rematerialize each chain twice - for normal and |
1973 | // for unwind basic blocks. Model this by multiplying cost by two. |
1974 | if (CS.isInvoke()) { |
1975 | Cost *= 2; |
1976 | } |
1977 | // If it's too expensive - skip it |
1978 | if (Cost >= RematerializationThreshold) |
1979 | continue; |
1980 | |
1981 | // Remove value from the live set |
1982 | LiveValuesToBeDeleted.push_back(LiveValue); |
1983 | |
1984 | // Clone instructions and record them inside "Info" structure |
1985 | |
1986 | // Walk backwards to visit top-most instructions first |
1987 | std::reverse(ChainToBase.begin(), ChainToBase.end()); |
1988 | |
1989 | // Utility function which clones all instructions from "ChainToBase" |
1990 | // and inserts them before "InsertBefore". Returns rematerialized value |
1991 | // which should be used after statepoint. |
1992 | auto rematerializeChain = [&ChainToBase]( |
1993 | Instruction *InsertBefore, Value *RootOfChain, Value *AlternateLiveBase) { |
1994 | Instruction *LastClonedValue = nullptr; |
1995 | Instruction *LastValue = nullptr; |
1996 | for (Instruction *Instr: ChainToBase) { |
1997 | // Only GEP's and casts are suported as we need to be careful to not |
1998 | // introduce any new uses of pointers not in the liveset. |
1999 | // Note that it's fine to introduce new uses of pointers which were |
2000 | // otherwise not used after this statepoint. |
2001 | assert(isa<GetElementPtrInst>(Instr) || isa<CastInst>(Instr))((isa<GetElementPtrInst>(Instr) || isa<CastInst>( Instr)) ? static_cast<void> (0) : __assert_fail ("isa<GetElementPtrInst>(Instr) || isa<CastInst>(Instr)" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2001, __PRETTY_FUNCTION__)); |
2002 | |
2003 | Instruction *ClonedValue = Instr->clone(); |
2004 | ClonedValue->insertBefore(InsertBefore); |
2005 | ClonedValue->setName(Instr->getName() + ".remat"); |
2006 | |
2007 | // If it is not first instruction in the chain then it uses previously |
2008 | // cloned value. We should update it to use cloned value. |
2009 | if (LastClonedValue) { |
2010 | assert(LastValue)((LastValue) ? static_cast<void> (0) : __assert_fail ("LastValue" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2010, __PRETTY_FUNCTION__)); |
2011 | ClonedValue->replaceUsesOfWith(LastValue, LastClonedValue); |
2012 | #ifndef NDEBUG |
2013 | for (auto OpValue : ClonedValue->operand_values()) { |
2014 | // Assert that cloned instruction does not use any instructions from |
2015 | // this chain other than LastClonedValue |
2016 | assert(!is_contained(ChainToBase, OpValue) &&((!is_contained(ChainToBase, OpValue) && "incorrect use in rematerialization chain" ) ? static_cast<void> (0) : __assert_fail ("!is_contained(ChainToBase, OpValue) && \"incorrect use in rematerialization chain\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2017, __PRETTY_FUNCTION__)) |
2017 | "incorrect use in rematerialization chain")((!is_contained(ChainToBase, OpValue) && "incorrect use in rematerialization chain" ) ? static_cast<void> (0) : __assert_fail ("!is_contained(ChainToBase, OpValue) && \"incorrect use in rematerialization chain\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2017, __PRETTY_FUNCTION__)); |
2018 | // Assert that the cloned instruction does not use the RootOfChain |
2019 | // or the AlternateLiveBase. |
2020 | assert(OpValue != RootOfChain && OpValue != AlternateLiveBase)((OpValue != RootOfChain && OpValue != AlternateLiveBase ) ? static_cast<void> (0) : __assert_fail ("OpValue != RootOfChain && OpValue != AlternateLiveBase" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2020, __PRETTY_FUNCTION__)); |
2021 | } |
2022 | #endif |
2023 | } else { |
2024 | // For the first instruction, replace the use of unrelocated base i.e. |
2025 | // RootOfChain/OrigRootPhi, with the corresponding PHI present in the |
2026 | // live set. They have been proved to be the same PHI nodes. Note |
2027 | // that the *only* use of the RootOfChain in the ChainToBase list is |
2028 | // the first Value in the list. |
2029 | if (RootOfChain != AlternateLiveBase) |
2030 | ClonedValue->replaceUsesOfWith(RootOfChain, AlternateLiveBase); |
2031 | } |
2032 | |
2033 | LastClonedValue = ClonedValue; |
2034 | LastValue = Instr; |
2035 | } |
2036 | assert(LastClonedValue)((LastClonedValue) ? static_cast<void> (0) : __assert_fail ("LastClonedValue", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2036, __PRETTY_FUNCTION__)); |
2037 | return LastClonedValue; |
2038 | }; |
2039 | |
2040 | // Different cases for calls and invokes. For invokes we need to clone |
2041 | // instructions both on normal and unwind path. |
2042 | if (CS.isCall()) { |
2043 | Instruction *InsertBefore = CS.getInstruction()->getNextNode(); |
2044 | assert(InsertBefore)((InsertBefore) ? static_cast<void> (0) : __assert_fail ("InsertBefore", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2044, __PRETTY_FUNCTION__)); |
2045 | Instruction *RematerializedValue = rematerializeChain( |
2046 | InsertBefore, RootOfChain, Info.PointerToBase[LiveValue]); |
2047 | Info.RematerializedValues[RematerializedValue] = LiveValue; |
2048 | } else { |
2049 | InvokeInst *Invoke = cast<InvokeInst>(CS.getInstruction()); |
2050 | |
2051 | Instruction *NormalInsertBefore = |
2052 | &*Invoke->getNormalDest()->getFirstInsertionPt(); |
2053 | Instruction *UnwindInsertBefore = |
2054 | &*Invoke->getUnwindDest()->getFirstInsertionPt(); |
2055 | |
2056 | Instruction *NormalRematerializedValue = rematerializeChain( |
2057 | NormalInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]); |
2058 | Instruction *UnwindRematerializedValue = rematerializeChain( |
2059 | UnwindInsertBefore, RootOfChain, Info.PointerToBase[LiveValue]); |
2060 | |
2061 | Info.RematerializedValues[NormalRematerializedValue] = LiveValue; |
2062 | Info.RematerializedValues[UnwindRematerializedValue] = LiveValue; |
2063 | } |
2064 | } |
2065 | |
2066 | // Remove rematerializaed values from the live set |
2067 | for (auto LiveValue: LiveValuesToBeDeleted) { |
2068 | Info.LiveSet.remove(LiveValue); |
2069 | } |
2070 | } |
2071 | |
2072 | static bool insertParsePoints(Function &F, DominatorTree &DT, |
2073 | TargetTransformInfo &TTI, |
2074 | SmallVectorImpl<CallSite> &ToUpdate) { |
2075 | #ifndef NDEBUG |
2076 | // sanity check the input |
2077 | std::set<CallSite> Uniqued; |
2078 | Uniqued.insert(ToUpdate.begin(), ToUpdate.end()); |
2079 | assert(Uniqued.size() == ToUpdate.size() && "no duplicates please!")((Uniqued.size() == ToUpdate.size() && "no duplicates please!" ) ? static_cast<void> (0) : __assert_fail ("Uniqued.size() == ToUpdate.size() && \"no duplicates please!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2079, __PRETTY_FUNCTION__)); |
2080 | |
2081 | for (CallSite CS : ToUpdate) |
2082 | assert(CS.getInstruction()->getFunction() == &F)((CS.getInstruction()->getFunction() == &F) ? static_cast <void> (0) : __assert_fail ("CS.getInstruction()->getFunction() == &F" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2082, __PRETTY_FUNCTION__)); |
2083 | #endif |
2084 | |
2085 | // When inserting gc.relocates for invokes, we need to be able to insert at |
2086 | // the top of the successor blocks. See the comment on |
2087 | // normalForInvokeSafepoint on exactly what is needed. Note that this step |
2088 | // may restructure the CFG. |
2089 | for (CallSite CS : ToUpdate) { |
2090 | if (!CS.isInvoke()) |
2091 | continue; |
2092 | auto *II = cast<InvokeInst>(CS.getInstruction()); |
2093 | normalizeForInvokeSafepoint(II->getNormalDest(), II->getParent(), DT); |
2094 | normalizeForInvokeSafepoint(II->getUnwindDest(), II->getParent(), DT); |
2095 | } |
2096 | |
2097 | // A list of dummy calls added to the IR to keep various values obviously |
2098 | // live in the IR. We'll remove all of these when done. |
2099 | SmallVector<CallInst *, 64> Holders; |
2100 | |
2101 | // Insert a dummy call with all of the deopt operands we'll need for the |
2102 | // actual safepoint insertion as arguments. This ensures reference operands |
2103 | // in the deopt argument list are considered live through the safepoint (and |
2104 | // thus makes sure they get relocated.) |
2105 | for (CallSite CS : ToUpdate) { |
2106 | SmallVector<Value *, 64> DeoptValues; |
2107 | |
2108 | for (Value *Arg : GetDeoptBundleOperands(CS)) { |
2109 | assert(!isUnhandledGCPointerType(Arg->getType()) &&((!isUnhandledGCPointerType(Arg->getType()) && "support for FCA unimplemented" ) ? static_cast<void> (0) : __assert_fail ("!isUnhandledGCPointerType(Arg->getType()) && \"support for FCA unimplemented\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2110, __PRETTY_FUNCTION__)) |
2110 | "support for FCA unimplemented")((!isUnhandledGCPointerType(Arg->getType()) && "support for FCA unimplemented" ) ? static_cast<void> (0) : __assert_fail ("!isUnhandledGCPointerType(Arg->getType()) && \"support for FCA unimplemented\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2110, __PRETTY_FUNCTION__)); |
2111 | if (isHandledGCPointerType(Arg->getType())) |
2112 | DeoptValues.push_back(Arg); |
2113 | } |
2114 | |
2115 | insertUseHolderAfter(CS, DeoptValues, Holders); |
2116 | } |
2117 | |
2118 | SmallVector<PartiallyConstructedSafepointRecord, 64> Records(ToUpdate.size()); |
2119 | |
2120 | // A) Identify all gc pointers which are statically live at the given call |
2121 | // site. |
2122 | findLiveReferences(F, DT, ToUpdate, Records); |
2123 | |
2124 | // B) Find the base pointers for each live pointer |
2125 | /* scope for caching */ { |
2126 | // Cache the 'defining value' relation used in the computation and |
2127 | // insertion of base phis and selects. This ensures that we don't insert |
2128 | // large numbers of duplicate base_phis. |
2129 | DefiningValueMapTy DVCache; |
2130 | |
2131 | for (size_t i = 0; i < Records.size(); i++) { |
2132 | PartiallyConstructedSafepointRecord &info = Records[i]; |
2133 | findBasePointers(DT, DVCache, ToUpdate[i], info); |
2134 | } |
2135 | } // end of cache scope |
2136 | |
2137 | // The base phi insertion logic (for any safepoint) may have inserted new |
2138 | // instructions which are now live at some safepoint. The simplest such |
2139 | // example is: |
2140 | // loop: |
2141 | // phi a <-- will be a new base_phi here |
2142 | // safepoint 1 <-- that needs to be live here |
2143 | // gep a + 1 |
2144 | // safepoint 2 |
2145 | // br loop |
2146 | // We insert some dummy calls after each safepoint to definitely hold live |
2147 | // the base pointers which were identified for that safepoint. We'll then |
2148 | // ask liveness for _every_ base inserted to see what is now live. Then we |
2149 | // remove the dummy calls. |
2150 | Holders.reserve(Holders.size() + Records.size()); |
2151 | for (size_t i = 0; i < Records.size(); i++) { |
2152 | PartiallyConstructedSafepointRecord &Info = Records[i]; |
2153 | |
2154 | SmallVector<Value *, 128> Bases; |
2155 | for (auto Pair : Info.PointerToBase) |
2156 | Bases.push_back(Pair.second); |
2157 | |
2158 | insertUseHolderAfter(ToUpdate[i], Bases, Holders); |
2159 | } |
2160 | |
2161 | // By selecting base pointers, we've effectively inserted new uses. Thus, we |
2162 | // need to rerun liveness. We may *also* have inserted new defs, but that's |
2163 | // not the key issue. |
2164 | recomputeLiveInValues(F, DT, ToUpdate, Records); |
2165 | |
2166 | if (PrintBasePointers) { |
2167 | for (auto &Info : Records) { |
2168 | errs() << "Base Pairs: (w/Relocation)\n"; |
2169 | for (auto Pair : Info.PointerToBase) { |
2170 | errs() << " derived "; |
2171 | Pair.first->printAsOperand(errs(), false); |
2172 | errs() << " base "; |
2173 | Pair.second->printAsOperand(errs(), false); |
2174 | errs() << "\n"; |
2175 | } |
2176 | } |
2177 | } |
2178 | |
2179 | // It is possible that non-constant live variables have a constant base. For |
2180 | // example, a GEP with a variable offset from a global. In this case we can |
2181 | // remove it from the liveset. We already don't add constants to the liveset |
2182 | // because we assume they won't move at runtime and the GC doesn't need to be |
2183 | // informed about them. The same reasoning applies if the base is constant. |
2184 | // Note that the relocation placement code relies on this filtering for |
2185 | // correctness as it expects the base to be in the liveset, which isn't true |
2186 | // if the base is constant. |
2187 | for (auto &Info : Records) |
2188 | for (auto &BasePair : Info.PointerToBase) |
2189 | if (isa<Constant>(BasePair.second)) |
2190 | Info.LiveSet.remove(BasePair.first); |
2191 | |
2192 | for (CallInst *CI : Holders) |
2193 | CI->eraseFromParent(); |
2194 | |
2195 | Holders.clear(); |
2196 | |
2197 | // In order to reduce live set of statepoint we might choose to rematerialize |
2198 | // some values instead of relocating them. This is purely an optimization and |
2199 | // does not influence correctness. |
2200 | for (size_t i = 0; i < Records.size(); i++) |
2201 | rematerializeLiveValues(ToUpdate[i], Records[i], TTI); |
2202 | |
2203 | // We need this to safely RAUW and delete call or invoke return values that |
2204 | // may themselves be live over a statepoint. For details, please see usage in |
2205 | // makeStatepointExplicitImpl. |
2206 | std::vector<DeferredReplacement> Replacements; |
2207 | |
2208 | // Now run through and replace the existing statepoints with new ones with |
2209 | // the live variables listed. We do not yet update uses of the values being |
2210 | // relocated. We have references to live variables that need to |
2211 | // survive to the last iteration of this loop. (By construction, the |
2212 | // previous statepoint can not be a live variable, thus we can and remove |
2213 | // the old statepoint calls as we go.) |
2214 | for (size_t i = 0; i < Records.size(); i++) |
2215 | makeStatepointExplicit(DT, ToUpdate[i], Records[i], Replacements); |
2216 | |
2217 | ToUpdate.clear(); // prevent accident use of invalid CallSites |
2218 | |
2219 | for (auto &PR : Replacements) |
2220 | PR.doReplacement(); |
2221 | |
2222 | Replacements.clear(); |
2223 | |
2224 | for (auto &Info : Records) { |
2225 | // These live sets may contain state Value pointers, since we replaced calls |
2226 | // with operand bundles with calls wrapped in gc.statepoint, and some of |
2227 | // those calls may have been def'ing live gc pointers. Clear these out to |
2228 | // avoid accidentally using them. |
2229 | // |
2230 | // TODO: We should create a separate data structure that does not contain |
2231 | // these live sets, and migrate to using that data structure from this point |
2232 | // onward. |
2233 | Info.LiveSet.clear(); |
2234 | Info.PointerToBase.clear(); |
2235 | } |
2236 | |
2237 | // Do all the fixups of the original live variables to their relocated selves |
2238 | SmallVector<Value *, 128> Live; |
2239 | for (size_t i = 0; i < Records.size(); i++) { |
2240 | PartiallyConstructedSafepointRecord &Info = Records[i]; |
2241 | |
2242 | // We can't simply save the live set from the original insertion. One of |
2243 | // the live values might be the result of a call which needs a safepoint. |
2244 | // That Value* no longer exists and we need to use the new gc_result. |
2245 | // Thankfully, the live set is embedded in the statepoint (and updated), so |
2246 | // we just grab that. |
2247 | Statepoint Statepoint(Info.StatepointToken); |
2248 | Live.insert(Live.end(), Statepoint.gc_args_begin(), |
2249 | Statepoint.gc_args_end()); |
2250 | #ifndef NDEBUG |
2251 | // Do some basic sanity checks on our liveness results before performing |
2252 | // relocation. Relocation can and will turn mistakes in liveness results |
2253 | // into non-sensical code which is must harder to debug. |
2254 | // TODO: It would be nice to test consistency as well |
2255 | assert(DT.isReachableFromEntry(Info.StatepointToken->getParent()) &&((DT.isReachableFromEntry(Info.StatepointToken->getParent( )) && "statepoint must be reachable or liveness is meaningless" ) ? static_cast<void> (0) : __assert_fail ("DT.isReachableFromEntry(Info.StatepointToken->getParent()) && \"statepoint must be reachable or liveness is meaningless\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2256, __PRETTY_FUNCTION__)) |
2256 | "statepoint must be reachable or liveness is meaningless")((DT.isReachableFromEntry(Info.StatepointToken->getParent( )) && "statepoint must be reachable or liveness is meaningless" ) ? static_cast<void> (0) : __assert_fail ("DT.isReachableFromEntry(Info.StatepointToken->getParent()) && \"statepoint must be reachable or liveness is meaningless\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2256, __PRETTY_FUNCTION__)); |
2257 | for (Value *V : Statepoint.gc_args()) { |
2258 | if (!isa<Instruction>(V)) |
2259 | // Non-instruction values trivial dominate all possible uses |
2260 | continue; |
2261 | auto *LiveInst = cast<Instruction>(V); |
2262 | assert(DT.isReachableFromEntry(LiveInst->getParent()) &&((DT.isReachableFromEntry(LiveInst->getParent()) && "unreachable values should never be live") ? static_cast< void> (0) : __assert_fail ("DT.isReachableFromEntry(LiveInst->getParent()) && \"unreachable values should never be live\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2263, __PRETTY_FUNCTION__)) |
2263 | "unreachable values should never be live")((DT.isReachableFromEntry(LiveInst->getParent()) && "unreachable values should never be live") ? static_cast< void> (0) : __assert_fail ("DT.isReachableFromEntry(LiveInst->getParent()) && \"unreachable values should never be live\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2263, __PRETTY_FUNCTION__)); |
2264 | assert(DT.dominates(LiveInst, Info.StatepointToken) &&((DT.dominates(LiveInst, Info.StatepointToken) && "basic SSA liveness expectation violated by liveness analysis" ) ? static_cast<void> (0) : __assert_fail ("DT.dominates(LiveInst, Info.StatepointToken) && \"basic SSA liveness expectation violated by liveness analysis\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2265, __PRETTY_FUNCTION__)) |
2265 | "basic SSA liveness expectation violated by liveness analysis")((DT.dominates(LiveInst, Info.StatepointToken) && "basic SSA liveness expectation violated by liveness analysis" ) ? static_cast<void> (0) : __assert_fail ("DT.dominates(LiveInst, Info.StatepointToken) && \"basic SSA liveness expectation violated by liveness analysis\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2265, __PRETTY_FUNCTION__)); |
2266 | } |
2267 | #endif |
2268 | } |
2269 | unique_unsorted(Live); |
2270 | |
2271 | #ifndef NDEBUG |
2272 | // sanity check |
2273 | for (auto *Ptr : Live) |
2274 | assert(isHandledGCPointerType(Ptr->getType()) &&((isHandledGCPointerType(Ptr->getType()) && "must be a gc pointer type" ) ? static_cast<void> (0) : __assert_fail ("isHandledGCPointerType(Ptr->getType()) && \"must be a gc pointer type\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2275, __PRETTY_FUNCTION__)) |
2275 | "must be a gc pointer type")((isHandledGCPointerType(Ptr->getType()) && "must be a gc pointer type" ) ? static_cast<void> (0) : __assert_fail ("isHandledGCPointerType(Ptr->getType()) && \"must be a gc pointer type\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2275, __PRETTY_FUNCTION__)); |
2276 | #endif |
2277 | |
2278 | relocationViaAlloca(F, DT, Live, Records); |
2279 | return !Records.empty(); |
2280 | } |
2281 | |
2282 | // Handles both return values and arguments for Functions and CallSites. |
2283 | template <typename AttrHolder> |
2284 | static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH, |
2285 | unsigned Index) { |
2286 | AttrBuilder R; |
2287 | if (AH.getDereferenceableBytes(Index)) |
2288 | R.addAttribute(Attribute::get(Ctx, Attribute::Dereferenceable, |
2289 | AH.getDereferenceableBytes(Index))); |
2290 | if (AH.getDereferenceableOrNullBytes(Index)) |
2291 | R.addAttribute(Attribute::get(Ctx, Attribute::DereferenceableOrNull, |
2292 | AH.getDereferenceableOrNullBytes(Index))); |
2293 | if (AH.getAttributes().hasAttribute(Index, Attribute::NoAlias)) |
2294 | R.addAttribute(Attribute::NoAlias); |
2295 | |
2296 | if (!R.empty()) |
2297 | AH.setAttributes(AH.getAttributes().removeAttributes(Ctx, Index, R)); |
2298 | } |
2299 | |
2300 | void |
2301 | RewriteStatepointsForGC::stripNonValidAttributesFromPrototype(Function &F) { |
2302 | LLVMContext &Ctx = F.getContext(); |
2303 | |
2304 | for (Argument &A : F.args()) |
2305 | if (isa<PointerType>(A.getType())) |
2306 | RemoveNonValidAttrAtIndex(Ctx, F, |
2307 | A.getArgNo() + AttributeList::FirstArgIndex); |
2308 | |
2309 | if (isa<PointerType>(F.getReturnType())) |
2310 | RemoveNonValidAttrAtIndex(Ctx, F, AttributeList::ReturnIndex); |
2311 | } |
2312 | |
2313 | void RewriteStatepointsForGC::stripInvalidMetadataFromInstruction(Instruction &I) { |
2314 | |
2315 | if (!isa<LoadInst>(I) && !isa<StoreInst>(I)) |
2316 | return; |
2317 | // These are the attributes that are still valid on loads and stores after |
2318 | // RS4GC. |
2319 | // The metadata implying dereferenceability and noalias are (conservatively) |
2320 | // dropped. This is because semantically, after RewriteStatepointsForGC runs, |
2321 | // all calls to gc.statepoint "free" the entire heap. Also, gc.statepoint can |
2322 | // touch the entire heap including noalias objects. Note: The reasoning is |
2323 | // same as stripping the dereferenceability and noalias attributes that are |
2324 | // analogous to the metadata counterparts. |
2325 | // We also drop the invariant.load metadata on the load because that metadata |
2326 | // implies the address operand to the load points to memory that is never |
2327 | // changed once it became dereferenceable. This is no longer true after RS4GC. |
2328 | // Similar reasoning applies to invariant.group metadata, which applies to |
2329 | // loads within a group. |
2330 | unsigned ValidMetadataAfterRS4GC[] = {LLVMContext::MD_tbaa, |
2331 | LLVMContext::MD_range, |
2332 | LLVMContext::MD_alias_scope, |
2333 | LLVMContext::MD_nontemporal, |
2334 | LLVMContext::MD_nonnull, |
2335 | LLVMContext::MD_align, |
2336 | LLVMContext::MD_type}; |
2337 | |
2338 | // Drops all metadata on the instruction other than ValidMetadataAfterRS4GC. |
2339 | I.dropUnknownNonDebugMetadata(ValidMetadataAfterRS4GC); |
2340 | |
2341 | } |
2342 | |
2343 | void RewriteStatepointsForGC::stripNonValidAttributesAndMetadataFromBody(Function &F) { |
2344 | if (F.empty()) |
2345 | return; |
2346 | |
2347 | LLVMContext &Ctx = F.getContext(); |
2348 | MDBuilder Builder(Ctx); |
2349 | |
2350 | |
2351 | for (Instruction &I : instructions(F)) { |
2352 | if (const MDNode *MD = I.getMetadata(LLVMContext::MD_tbaa)) { |
2353 | assert(MD->getNumOperands() < 5 && "unrecognized metadata shape!")((MD->getNumOperands() < 5 && "unrecognized metadata shape!" ) ? static_cast<void> (0) : __assert_fail ("MD->getNumOperands() < 5 && \"unrecognized metadata shape!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2353, __PRETTY_FUNCTION__)); |
2354 | bool IsImmutableTBAA = |
2355 | MD->getNumOperands() == 4 && |
2356 | mdconst::extract<ConstantInt>(MD->getOperand(3))->getValue() == 1; |
2357 | |
2358 | if (!IsImmutableTBAA) |
2359 | continue; // no work to do, MD_tbaa is already marked mutable |
2360 | |
2361 | MDNode *Base = cast<MDNode>(MD->getOperand(0)); |
2362 | MDNode *Access = cast<MDNode>(MD->getOperand(1)); |
2363 | uint64_t Offset = |
2364 | mdconst::extract<ConstantInt>(MD->getOperand(2))->getZExtValue(); |
2365 | |
2366 | MDNode *MutableTBAA = |
2367 | Builder.createTBAAStructTagNode(Base, Access, Offset); |
2368 | I.setMetadata(LLVMContext::MD_tbaa, MutableTBAA); |
2369 | } |
2370 | |
2371 | stripInvalidMetadataFromInstruction(I); |
2372 | |
2373 | if (CallSite CS = CallSite(&I)) { |
2374 | for (int i = 0, e = CS.arg_size(); i != e; i++) |
2375 | if (isa<PointerType>(CS.getArgument(i)->getType())) |
2376 | RemoveNonValidAttrAtIndex(Ctx, CS, i + AttributeList::FirstArgIndex); |
2377 | if (isa<PointerType>(CS.getType())) |
2378 | RemoveNonValidAttrAtIndex(Ctx, CS, AttributeList::ReturnIndex); |
2379 | } |
2380 | } |
2381 | } |
2382 | |
2383 | /// Returns true if this function should be rewritten by this pass. The main |
2384 | /// point of this function is as an extension point for custom logic. |
2385 | static bool shouldRewriteStatepointsIn(Function &F) { |
2386 | // TODO: This should check the GCStrategy |
2387 | if (F.hasGC()) { |
2388 | const auto &FunctionGCName = F.getGC(); |
2389 | const StringRef StatepointExampleName("statepoint-example"); |
2390 | const StringRef CoreCLRName("coreclr"); |
2391 | return (StatepointExampleName == FunctionGCName) || |
2392 | (CoreCLRName == FunctionGCName); |
2393 | } else |
2394 | return false; |
2395 | } |
2396 | |
2397 | void RewriteStatepointsForGC::stripNonValidAttributesAndMetadata(Module &M) { |
2398 | #ifndef NDEBUG |
2399 | assert(any_of(M, shouldRewriteStatepointsIn) && "precondition!")((any_of(M, shouldRewriteStatepointsIn) && "precondition!" ) ? static_cast<void> (0) : __assert_fail ("any_of(M, shouldRewriteStatepointsIn) && \"precondition!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2399, __PRETTY_FUNCTION__)); |
2400 | #endif |
2401 | |
2402 | for (Function &F : M) |
2403 | stripNonValidAttributesFromPrototype(F); |
2404 | |
2405 | for (Function &F : M) |
2406 | stripNonValidAttributesAndMetadataFromBody(F); |
2407 | } |
2408 | |
2409 | bool RewriteStatepointsForGC::runOnFunction(Function &F) { |
2410 | // Nothing to do for declarations. |
2411 | if (F.isDeclaration() || F.empty()) |
2412 | return false; |
2413 | |
2414 | // Policy choice says not to rewrite - the most common reason is that we're |
2415 | // compiling code without a GCStrategy. |
2416 | if (!shouldRewriteStatepointsIn(F)) |
2417 | return false; |
2418 | |
2419 | DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(F).getDomTree(); |
2420 | TargetTransformInfo &TTI = |
2421 | getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); |
2422 | |
2423 | auto NeedsRewrite = [](Instruction &I) { |
2424 | if (ImmutableCallSite CS = ImmutableCallSite(&I)) |
2425 | return !callsGCLeafFunction(CS) && !isStatepoint(CS); |
2426 | return false; |
2427 | }; |
2428 | |
2429 | // Gather all the statepoints which need rewritten. Be careful to only |
2430 | // consider those in reachable code since we need to ask dominance queries |
2431 | // when rewriting. We'll delete the unreachable ones in a moment. |
2432 | SmallVector<CallSite, 64> ParsePointNeeded; |
2433 | bool HasUnreachableStatepoint = false; |
2434 | for (Instruction &I : instructions(F)) { |
2435 | // TODO: only the ones with the flag set! |
2436 | if (NeedsRewrite(I)) { |
2437 | if (DT.isReachableFromEntry(I.getParent())) |
2438 | ParsePointNeeded.push_back(CallSite(&I)); |
2439 | else |
2440 | HasUnreachableStatepoint = true; |
2441 | } |
2442 | } |
2443 | |
2444 | bool MadeChange = false; |
2445 | |
2446 | // Delete any unreachable statepoints so that we don't have unrewritten |
2447 | // statepoints surviving this pass. This makes testing easier and the |
2448 | // resulting IR less confusing to human readers. Rather than be fancy, we |
2449 | // just reuse a utility function which removes the unreachable blocks. |
2450 | if (HasUnreachableStatepoint) |
2451 | MadeChange |= removeUnreachableBlocks(F); |
2452 | |
2453 | // Return early if no work to do. |
2454 | if (ParsePointNeeded.empty()) |
2455 | return MadeChange; |
2456 | |
2457 | // As a prepass, go ahead and aggressively destroy single entry phi nodes. |
2458 | // These are created by LCSSA. They have the effect of increasing the size |
2459 | // of liveness sets for no good reason. It may be harder to do this post |
2460 | // insertion since relocations and base phis can confuse things. |
2461 | for (BasicBlock &BB : F) |
2462 | if (BB.getUniquePredecessor()) { |
2463 | MadeChange = true; |
2464 | FoldSingleEntryPHINodes(&BB); |
2465 | } |
2466 | |
2467 | // Before we start introducing relocations, we want to tweak the IR a bit to |
2468 | // avoid unfortunate code generation effects. The main example is that we |
2469 | // want to try to make sure the comparison feeding a branch is after any |
2470 | // safepoints. Otherwise, we end up with a comparison of pre-relocation |
2471 | // values feeding a branch after relocation. This is semantically correct, |
2472 | // but results in extra register pressure since both the pre-relocation and |
2473 | // post-relocation copies must be available in registers. For code without |
2474 | // relocations this is handled elsewhere, but teaching the scheduler to |
2475 | // reverse the transform we're about to do would be slightly complex. |
2476 | // Note: This may extend the live range of the inputs to the icmp and thus |
2477 | // increase the liveset of any statepoint we move over. This is profitable |
2478 | // as long as all statepoints are in rare blocks. If we had in-register |
2479 | // lowering for live values this would be a much safer transform. |
2480 | auto getConditionInst = [](TerminatorInst *TI) -> Instruction* { |
2481 | if (auto *BI = dyn_cast<BranchInst>(TI)) |
2482 | if (BI->isConditional()) |
2483 | return dyn_cast<Instruction>(BI->getCondition()); |
2484 | // TODO: Extend this to handle switches |
2485 | return nullptr; |
2486 | }; |
2487 | for (BasicBlock &BB : F) { |
2488 | TerminatorInst *TI = BB.getTerminator(); |
2489 | if (auto *Cond = getConditionInst(TI)) |
2490 | // TODO: Handle more than just ICmps here. We should be able to move |
2491 | // most instructions without side effects or memory access. |
2492 | if (isa<ICmpInst>(Cond) && Cond->hasOneUse()) { |
2493 | MadeChange = true; |
2494 | Cond->moveBefore(TI); |
2495 | } |
2496 | } |
2497 | |
2498 | MadeChange |= insertParsePoints(F, DT, TTI, ParsePointNeeded); |
2499 | return MadeChange; |
2500 | } |
2501 | |
2502 | // liveness computation via standard dataflow |
2503 | // ------------------------------------------------------------------- |
2504 | |
2505 | // TODO: Consider using bitvectors for liveness, the set of potentially |
2506 | // interesting values should be small and easy to pre-compute. |
2507 | |
2508 | /// Compute the live-in set for the location rbegin starting from |
2509 | /// the live-out set of the basic block |
2510 | static void computeLiveInValues(BasicBlock::reverse_iterator Begin, |
2511 | BasicBlock::reverse_iterator End, |
2512 | SetVector<Value *> &LiveTmp) { |
2513 | for (auto &I : make_range(Begin, End)) { |
2514 | // KILL/Def - Remove this definition from LiveIn |
2515 | LiveTmp.remove(&I); |
2516 | |
2517 | // Don't consider *uses* in PHI nodes, we handle their contribution to |
2518 | // predecessor blocks when we seed the LiveOut sets |
2519 | if (isa<PHINode>(I)) |
2520 | continue; |
2521 | |
2522 | // USE - Add to the LiveIn set for this instruction |
2523 | for (Value *V : I.operands()) { |
2524 | assert(!isUnhandledGCPointerType(V->getType()) &&((!isUnhandledGCPointerType(V->getType()) && "support for FCA unimplemented" ) ? static_cast<void> (0) : __assert_fail ("!isUnhandledGCPointerType(V->getType()) && \"support for FCA unimplemented\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2525, __PRETTY_FUNCTION__)) |
2525 | "support for FCA unimplemented")((!isUnhandledGCPointerType(V->getType()) && "support for FCA unimplemented" ) ? static_cast<void> (0) : __assert_fail ("!isUnhandledGCPointerType(V->getType()) && \"support for FCA unimplemented\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2525, __PRETTY_FUNCTION__)); |
2526 | if (isHandledGCPointerType(V->getType()) && !isa<Constant>(V)) { |
2527 | // The choice to exclude all things constant here is slightly subtle. |
2528 | // There are two independent reasons: |
2529 | // - We assume that things which are constant (from LLVM's definition) |
2530 | // do not move at runtime. For example, the address of a global |
2531 | // variable is fixed, even though it's contents may not be. |
2532 | // - Second, we can't disallow arbitrary inttoptr constants even |
2533 | // if the language frontend does. Optimization passes are free to |
2534 | // locally exploit facts without respect to global reachability. This |
2535 | // can create sections of code which are dynamically unreachable and |
2536 | // contain just about anything. (see constants.ll in tests) |
2537 | LiveTmp.insert(V); |
2538 | } |
2539 | } |
2540 | } |
2541 | } |
2542 | |
2543 | static void computeLiveOutSeed(BasicBlock *BB, SetVector<Value *> &LiveTmp) { |
2544 | for (BasicBlock *Succ : successors(BB)) { |
2545 | for (auto &I : *Succ) { |
2546 | PHINode *PN = dyn_cast<PHINode>(&I); |
2547 | if (!PN) |
2548 | break; |
2549 | |
2550 | Value *V = PN->getIncomingValueForBlock(BB); |
2551 | assert(!isUnhandledGCPointerType(V->getType()) &&((!isUnhandledGCPointerType(V->getType()) && "support for FCA unimplemented" ) ? static_cast<void> (0) : __assert_fail ("!isUnhandledGCPointerType(V->getType()) && \"support for FCA unimplemented\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2552, __PRETTY_FUNCTION__)) |
2552 | "support for FCA unimplemented")((!isUnhandledGCPointerType(V->getType()) && "support for FCA unimplemented" ) ? static_cast<void> (0) : __assert_fail ("!isUnhandledGCPointerType(V->getType()) && \"support for FCA unimplemented\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2552, __PRETTY_FUNCTION__)); |
2553 | if (isHandledGCPointerType(V->getType()) && !isa<Constant>(V)) |
2554 | LiveTmp.insert(V); |
2555 | } |
2556 | } |
2557 | } |
2558 | |
2559 | static SetVector<Value *> computeKillSet(BasicBlock *BB) { |
2560 | SetVector<Value *> KillSet; |
2561 | for (Instruction &I : *BB) |
2562 | if (isHandledGCPointerType(I.getType())) |
2563 | KillSet.insert(&I); |
2564 | return KillSet; |
2565 | } |
2566 | |
2567 | #ifndef NDEBUG |
2568 | /// Check that the items in 'Live' dominate 'TI'. This is used as a basic |
2569 | /// sanity check for the liveness computation. |
2570 | static void checkBasicSSA(DominatorTree &DT, SetVector<Value *> &Live, |
2571 | TerminatorInst *TI, bool TermOkay = false) { |
2572 | for (Value *V : Live) { |
2573 | if (auto *I = dyn_cast<Instruction>(V)) { |
2574 | // The terminator can be a member of the LiveOut set. LLVM's definition |
2575 | // of instruction dominance states that V does not dominate itself. As |
2576 | // such, we need to special case this to allow it. |
2577 | if (TermOkay && TI == I) |
2578 | continue; |
2579 | assert(DT.dominates(I, TI) &&((DT.dominates(I, TI) && "basic SSA liveness expectation violated by liveness analysis" ) ? static_cast<void> (0) : __assert_fail ("DT.dominates(I, TI) && \"basic SSA liveness expectation violated by liveness analysis\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2580, __PRETTY_FUNCTION__)) |
2580 | "basic SSA liveness expectation violated by liveness analysis")((DT.dominates(I, TI) && "basic SSA liveness expectation violated by liveness analysis" ) ? static_cast<void> (0) : __assert_fail ("DT.dominates(I, TI) && \"basic SSA liveness expectation violated by liveness analysis\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2580, __PRETTY_FUNCTION__)); |
2581 | } |
2582 | } |
2583 | } |
2584 | |
2585 | /// Check that all the liveness sets used during the computation of liveness |
2586 | /// obey basic SSA properties. This is useful for finding cases where we miss |
2587 | /// a def. |
2588 | static void checkBasicSSA(DominatorTree &DT, GCPtrLivenessData &Data, |
2589 | BasicBlock &BB) { |
2590 | checkBasicSSA(DT, Data.LiveSet[&BB], BB.getTerminator()); |
2591 | checkBasicSSA(DT, Data.LiveOut[&BB], BB.getTerminator(), true); |
2592 | checkBasicSSA(DT, Data.LiveIn[&BB], BB.getTerminator()); |
2593 | } |
2594 | #endif |
2595 | |
2596 | static void computeLiveInValues(DominatorTree &DT, Function &F, |
2597 | GCPtrLivenessData &Data) { |
2598 | SmallSetVector<BasicBlock *, 32> Worklist; |
2599 | |
2600 | // Seed the liveness for each individual block |
2601 | for (BasicBlock &BB : F) { |
2602 | Data.KillSet[&BB] = computeKillSet(&BB); |
2603 | Data.LiveSet[&BB].clear(); |
2604 | computeLiveInValues(BB.rbegin(), BB.rend(), Data.LiveSet[&BB]); |
2605 | |
2606 | #ifndef NDEBUG |
2607 | for (Value *Kill : Data.KillSet[&BB]) |
2608 | assert(!Data.LiveSet[&BB].count(Kill) && "live set contains kill")((!Data.LiveSet[&BB].count(Kill) && "live set contains kill" ) ? static_cast<void> (0) : __assert_fail ("!Data.LiveSet[&BB].count(Kill) && \"live set contains kill\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2608, __PRETTY_FUNCTION__)); |
2609 | #endif |
2610 | |
2611 | Data.LiveOut[&BB] = SetVector<Value *>(); |
2612 | computeLiveOutSeed(&BB, Data.LiveOut[&BB]); |
2613 | Data.LiveIn[&BB] = Data.LiveSet[&BB]; |
2614 | Data.LiveIn[&BB].set_union(Data.LiveOut[&BB]); |
2615 | Data.LiveIn[&BB].set_subtract(Data.KillSet[&BB]); |
2616 | if (!Data.LiveIn[&BB].empty()) |
2617 | Worklist.insert(pred_begin(&BB), pred_end(&BB)); |
2618 | } |
2619 | |
2620 | // Propagate that liveness until stable |
2621 | while (!Worklist.empty()) { |
2622 | BasicBlock *BB = Worklist.pop_back_val(); |
2623 | |
2624 | // Compute our new liveout set, then exit early if it hasn't changed despite |
2625 | // the contribution of our successor. |
2626 | SetVector<Value *> LiveOut = Data.LiveOut[BB]; |
2627 | const auto OldLiveOutSize = LiveOut.size(); |
2628 | for (BasicBlock *Succ : successors(BB)) { |
2629 | assert(Data.LiveIn.count(Succ))((Data.LiveIn.count(Succ)) ? static_cast<void> (0) : __assert_fail ("Data.LiveIn.count(Succ)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2629, __PRETTY_FUNCTION__)); |
2630 | LiveOut.set_union(Data.LiveIn[Succ]); |
2631 | } |
2632 | // assert OutLiveOut is a subset of LiveOut |
2633 | if (OldLiveOutSize == LiveOut.size()) { |
2634 | // If the sets are the same size, then we didn't actually add anything |
2635 | // when unioning our successors LiveIn. Thus, the LiveIn of this block |
2636 | // hasn't changed. |
2637 | continue; |
2638 | } |
2639 | Data.LiveOut[BB] = LiveOut; |
2640 | |
2641 | // Apply the effects of this basic block |
2642 | SetVector<Value *> LiveTmp = LiveOut; |
2643 | LiveTmp.set_union(Data.LiveSet[BB]); |
2644 | LiveTmp.set_subtract(Data.KillSet[BB]); |
2645 | |
2646 | assert(Data.LiveIn.count(BB))((Data.LiveIn.count(BB)) ? static_cast<void> (0) : __assert_fail ("Data.LiveIn.count(BB)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2646, __PRETTY_FUNCTION__)); |
2647 | const SetVector<Value *> &OldLiveIn = Data.LiveIn[BB]; |
2648 | // assert: OldLiveIn is a subset of LiveTmp |
2649 | if (OldLiveIn.size() != LiveTmp.size()) { |
2650 | Data.LiveIn[BB] = LiveTmp; |
2651 | Worklist.insert(pred_begin(BB), pred_end(BB)); |
2652 | } |
2653 | } // while (!Worklist.empty()) |
2654 | |
2655 | #ifndef NDEBUG |
2656 | // Sanity check our output against SSA properties. This helps catch any |
2657 | // missing kills during the above iteration. |
2658 | for (BasicBlock &BB : F) |
2659 | checkBasicSSA(DT, Data, BB); |
2660 | #endif |
2661 | } |
2662 | |
2663 | static void findLiveSetAtInst(Instruction *Inst, GCPtrLivenessData &Data, |
2664 | StatepointLiveSetTy &Out) { |
2665 | |
2666 | BasicBlock *BB = Inst->getParent(); |
2667 | |
2668 | // Note: The copy is intentional and required |
2669 | assert(Data.LiveOut.count(BB))((Data.LiveOut.count(BB)) ? static_cast<void> (0) : __assert_fail ("Data.LiveOut.count(BB)", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2669, __PRETTY_FUNCTION__)); |
2670 | SetVector<Value *> LiveOut = Data.LiveOut[BB]; |
2671 | |
2672 | // We want to handle the statepoint itself oddly. It's |
2673 | // call result is not live (normal), nor are it's arguments |
2674 | // (unless they're used again later). This adjustment is |
2675 | // specifically what we need to relocate |
2676 | computeLiveInValues(BB->rbegin(), ++Inst->getIterator().getReverse(), |
2677 | LiveOut); |
2678 | LiveOut.remove(Inst); |
2679 | Out.insert(LiveOut.begin(), LiveOut.end()); |
2680 | } |
2681 | |
2682 | static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, |
2683 | CallSite CS, |
2684 | PartiallyConstructedSafepointRecord &Info) { |
2685 | Instruction *Inst = CS.getInstruction(); |
2686 | StatepointLiveSetTy Updated; |
2687 | findLiveSetAtInst(Inst, RevisedLivenessData, Updated); |
2688 | |
2689 | #ifndef NDEBUG |
2690 | DenseSet<Value *> Bases; |
2691 | for (auto KVPair : Info.PointerToBase) |
2692 | Bases.insert(KVPair.second); |
2693 | #endif |
2694 | |
2695 | // We may have base pointers which are now live that weren't before. We need |
2696 | // to update the PointerToBase structure to reflect this. |
2697 | for (auto V : Updated) |
2698 | if (Info.PointerToBase.insert({V, V}).second) { |
2699 | assert(Bases.count(V) && "Can't find base for unexpected live value!")((Bases.count(V) && "Can't find base for unexpected live value!" ) ? static_cast<void> (0) : __assert_fail ("Bases.count(V) && \"Can't find base for unexpected live value!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2699, __PRETTY_FUNCTION__)); |
2700 | continue; |
2701 | } |
2702 | |
2703 | #ifndef NDEBUG |
2704 | for (auto V : Updated) |
2705 | assert(Info.PointerToBase.count(V) &&((Info.PointerToBase.count(V) && "Must be able to find base for live value!" ) ? static_cast<void> (0) : __assert_fail ("Info.PointerToBase.count(V) && \"Must be able to find base for live value!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2706, __PRETTY_FUNCTION__)) |
2706 | "Must be able to find base for live value!")((Info.PointerToBase.count(V) && "Must be able to find base for live value!" ) ? static_cast<void> (0) : __assert_fail ("Info.PointerToBase.count(V) && \"Must be able to find base for live value!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2706, __PRETTY_FUNCTION__)); |
2707 | #endif |
2708 | |
2709 | // Remove any stale base mappings - this can happen since our liveness is |
2710 | // more precise then the one inherent in the base pointer analysis. |
2711 | DenseSet<Value *> ToErase; |
2712 | for (auto KVPair : Info.PointerToBase) |
2713 | if (!Updated.count(KVPair.first)) |
2714 | ToErase.insert(KVPair.first); |
2715 | |
2716 | for (auto *V : ToErase) |
2717 | Info.PointerToBase.erase(V); |
2718 | |
2719 | #ifndef NDEBUG |
2720 | for (auto KVPair : Info.PointerToBase) |
2721 | assert(Updated.count(KVPair.first) && "record for non-live value")((Updated.count(KVPair.first) && "record for non-live value" ) ? static_cast<void> (0) : __assert_fail ("Updated.count(KVPair.first) && \"record for non-live value\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn306458/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp" , 2721, __PRETTY_FUNCTION__)); |
2722 | #endif |
2723 | |
2724 | Info.LiveSet = Updated; |
2725 | } |