File: | build/source/llvm/lib/Transforms/Scalar/SROA.cpp |
Warning: | line 3138, column 54 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | /// \file | |||
9 | /// This transformation implements the well known scalar replacement of | |||
10 | /// aggregates transformation. It tries to identify promotable elements of an | |||
11 | /// aggregate alloca, and promote them to registers. It will also try to | |||
12 | /// convert uses of an element (or set of elements) of an alloca into a vector | |||
13 | /// or bitfield-style integer scalar if appropriate. | |||
14 | /// | |||
15 | /// It works to do this with minimal slicing of the alloca so that regions | |||
16 | /// which are merely transferred in and out of external memory remain unchanged | |||
17 | /// and are not decomposed to scalar code. | |||
18 | /// | |||
19 | /// Because this also performs alloca promotion, it can be thought of as also | |||
20 | /// serving the purpose of SSA formation. The algorithm iterates on the | |||
21 | /// function until all opportunities for promotion have been realized. | |||
22 | /// | |||
23 | //===----------------------------------------------------------------------===// | |||
24 | ||||
25 | #include "llvm/Transforms/Scalar/SROA.h" | |||
26 | #include "llvm/ADT/APInt.h" | |||
27 | #include "llvm/ADT/ArrayRef.h" | |||
28 | #include "llvm/ADT/DenseMap.h" | |||
29 | #include "llvm/ADT/PointerIntPair.h" | |||
30 | #include "llvm/ADT/STLExtras.h" | |||
31 | #include "llvm/ADT/SetVector.h" | |||
32 | #include "llvm/ADT/SmallBitVector.h" | |||
33 | #include "llvm/ADT/SmallPtrSet.h" | |||
34 | #include "llvm/ADT/SmallVector.h" | |||
35 | #include "llvm/ADT/Statistic.h" | |||
36 | #include "llvm/ADT/StringRef.h" | |||
37 | #include "llvm/ADT/Twine.h" | |||
38 | #include "llvm/ADT/iterator.h" | |||
39 | #include "llvm/ADT/iterator_range.h" | |||
40 | #include "llvm/Analysis/AssumptionCache.h" | |||
41 | #include "llvm/Analysis/DomTreeUpdater.h" | |||
42 | #include "llvm/Analysis/GlobalsModRef.h" | |||
43 | #include "llvm/Analysis/Loads.h" | |||
44 | #include "llvm/Analysis/PtrUseVisitor.h" | |||
45 | #include "llvm/Config/llvm-config.h" | |||
46 | #include "llvm/IR/BasicBlock.h" | |||
47 | #include "llvm/IR/Constant.h" | |||
48 | #include "llvm/IR/ConstantFolder.h" | |||
49 | #include "llvm/IR/Constants.h" | |||
50 | #include "llvm/IR/DIBuilder.h" | |||
51 | #include "llvm/IR/DataLayout.h" | |||
52 | #include "llvm/IR/DebugInfo.h" | |||
53 | #include "llvm/IR/DebugInfoMetadata.h" | |||
54 | #include "llvm/IR/DerivedTypes.h" | |||
55 | #include "llvm/IR/Dominators.h" | |||
56 | #include "llvm/IR/Function.h" | |||
57 | #include "llvm/IR/GetElementPtrTypeIterator.h" | |||
58 | #include "llvm/IR/GlobalAlias.h" | |||
59 | #include "llvm/IR/IRBuilder.h" | |||
60 | #include "llvm/IR/InstVisitor.h" | |||
61 | #include "llvm/IR/Instruction.h" | |||
62 | #include "llvm/IR/Instructions.h" | |||
63 | #include "llvm/IR/IntrinsicInst.h" | |||
64 | #include "llvm/IR/LLVMContext.h" | |||
65 | #include "llvm/IR/Metadata.h" | |||
66 | #include "llvm/IR/Module.h" | |||
67 | #include "llvm/IR/Operator.h" | |||
68 | #include "llvm/IR/PassManager.h" | |||
69 | #include "llvm/IR/Type.h" | |||
70 | #include "llvm/IR/Use.h" | |||
71 | #include "llvm/IR/User.h" | |||
72 | #include "llvm/IR/Value.h" | |||
73 | #include "llvm/InitializePasses.h" | |||
74 | #include "llvm/Pass.h" | |||
75 | #include "llvm/Support/Casting.h" | |||
76 | #include "llvm/Support/CommandLine.h" | |||
77 | #include "llvm/Support/Compiler.h" | |||
78 | #include "llvm/Support/Debug.h" | |||
79 | #include "llvm/Support/ErrorHandling.h" | |||
80 | #include "llvm/Support/raw_ostream.h" | |||
81 | #include "llvm/Transforms/Scalar.h" | |||
82 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | |||
83 | #include "llvm/Transforms/Utils/Local.h" | |||
84 | #include "llvm/Transforms/Utils/PromoteMemToReg.h" | |||
85 | #include <algorithm> | |||
86 | #include <cassert> | |||
87 | #include <cstddef> | |||
88 | #include <cstdint> | |||
89 | #include <cstring> | |||
90 | #include <iterator> | |||
91 | #include <string> | |||
92 | #include <tuple> | |||
93 | #include <utility> | |||
94 | #include <vector> | |||
95 | ||||
96 | using namespace llvm; | |||
97 | using namespace llvm::sroa; | |||
98 | ||||
99 | #define DEBUG_TYPE"sroa" "sroa" | |||
100 | ||||
101 | STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement")static llvm::Statistic NumAllocasAnalyzed = {"sroa", "NumAllocasAnalyzed" , "Number of allocas analyzed for replacement"}; | |||
102 | STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed")static llvm::Statistic NumAllocaPartitions = {"sroa", "NumAllocaPartitions" , "Number of alloca partitions formed"}; | |||
103 | STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca")static llvm::Statistic MaxPartitionsPerAlloca = {"sroa", "MaxPartitionsPerAlloca" , "Maximum number of partitions per alloca"}; | |||
104 | STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten")static llvm::Statistic NumAllocaPartitionUses = {"sroa", "NumAllocaPartitionUses" , "Number of alloca partition uses rewritten"}; | |||
105 | STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition")static llvm::Statistic MaxUsesPerAllocaPartition = {"sroa", "MaxUsesPerAllocaPartition" , "Maximum number of uses of a partition"}; | |||
106 | STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced")static llvm::Statistic NumNewAllocas = {"sroa", "NumNewAllocas" , "Number of new, smaller allocas introduced"}; | |||
107 | STATISTIC(NumPromoted, "Number of allocas promoted to SSA values")static llvm::Statistic NumPromoted = {"sroa", "NumPromoted", "Number of allocas promoted to SSA values" }; | |||
108 | STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion")static llvm::Statistic NumLoadsSpeculated = {"sroa", "NumLoadsSpeculated" , "Number of loads speculated to allow promotion"}; | |||
109 | STATISTIC(NumLoadsPredicated,static llvm::Statistic NumLoadsPredicated = {"sroa", "NumLoadsPredicated" , "Number of loads rewritten into predicated loads to allow promotion" } | |||
110 | "Number of loads rewritten into predicated loads to allow promotion")static llvm::Statistic NumLoadsPredicated = {"sroa", "NumLoadsPredicated" , "Number of loads rewritten into predicated loads to allow promotion" }; | |||
111 | STATISTIC(static llvm::Statistic NumStoresPredicated = {"sroa", "NumStoresPredicated" , "Number of stores rewritten into predicated loads to allow promotion" } | |||
112 | NumStoresPredicated,static llvm::Statistic NumStoresPredicated = {"sroa", "NumStoresPredicated" , "Number of stores rewritten into predicated loads to allow promotion" } | |||
113 | "Number of stores rewritten into predicated loads to allow promotion")static llvm::Statistic NumStoresPredicated = {"sroa", "NumStoresPredicated" , "Number of stores rewritten into predicated loads to allow promotion" }; | |||
114 | STATISTIC(NumDeleted, "Number of instructions deleted")static llvm::Statistic NumDeleted = {"sroa", "NumDeleted", "Number of instructions deleted" }; | |||
115 | STATISTIC(NumVectorized, "Number of vectorized aggregates")static llvm::Statistic NumVectorized = {"sroa", "NumVectorized" , "Number of vectorized aggregates"}; | |||
116 | ||||
117 | /// Hidden option to experiment with completely strict handling of inbounds | |||
118 | /// GEPs. | |||
119 | static cl::opt<bool> SROAStrictInbounds("sroa-strict-inbounds", cl::init(false), | |||
120 | cl::Hidden); | |||
121 | /// Disable running mem2reg during SROA in order to test or debug SROA. | |||
122 | static cl::opt<bool> SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(false), | |||
123 | cl::Hidden); | |||
124 | namespace { | |||
125 | ||||
126 | /// Calculate the fragment of a variable to use when slicing a store | |||
127 | /// based on the slice dimensions, existing fragment, and base storage | |||
128 | /// fragment. | |||
129 | /// Results: | |||
130 | /// UseFrag - Use Target as the new fragment. | |||
131 | /// UseNoFrag - The new slice already covers the whole variable. | |||
132 | /// Skip - The new alloca slice doesn't include this variable. | |||
133 | /// FIXME: Can we use calculateFragmentIntersect instead? | |||
134 | enum FragCalcResult { UseFrag, UseNoFrag, Skip }; | |||
135 | static FragCalcResult | |||
136 | calculateFragment(DILocalVariable *Variable, | |||
137 | uint64_t NewStorageSliceOffsetInBits, | |||
138 | uint64_t NewStorageSliceSizeInBits, | |||
139 | std::optional<DIExpression::FragmentInfo> StorageFragment, | |||
140 | std::optional<DIExpression::FragmentInfo> CurrentFragment, | |||
141 | DIExpression::FragmentInfo &Target) { | |||
142 | // If the base storage describes part of the variable apply the offset and | |||
143 | // the size constraint. | |||
144 | if (StorageFragment) { | |||
145 | Target.SizeInBits = | |||
146 | std::min(NewStorageSliceSizeInBits, StorageFragment->SizeInBits); | |||
147 | Target.OffsetInBits = | |||
148 | NewStorageSliceOffsetInBits + StorageFragment->OffsetInBits; | |||
149 | } else { | |||
150 | Target.SizeInBits = NewStorageSliceSizeInBits; | |||
151 | Target.OffsetInBits = NewStorageSliceOffsetInBits; | |||
152 | } | |||
153 | ||||
154 | // If this slice extracts the entirety of an independent variable from a | |||
155 | // larger alloca, do not produce a fragment expression, as the variable is | |||
156 | // not fragmented. | |||
157 | if (!CurrentFragment) { | |||
158 | if (auto Size = Variable->getSizeInBits()) { | |||
159 | // Treat the current fragment as covering the whole variable. | |||
160 | CurrentFragment = DIExpression::FragmentInfo(*Size, 0); | |||
161 | if (Target == CurrentFragment) | |||
162 | return UseNoFrag; | |||
163 | } | |||
164 | } | |||
165 | ||||
166 | // No additional work to do if there isn't a fragment already, or there is | |||
167 | // but it already exactly describes the new assignment. | |||
168 | if (!CurrentFragment || *CurrentFragment == Target) | |||
169 | return UseFrag; | |||
170 | ||||
171 | // Reject the target fragment if it doesn't fit wholly within the current | |||
172 | // fragment. TODO: We could instead chop up the target to fit in the case of | |||
173 | // a partial overlap. | |||
174 | if (Target.startInBits() < CurrentFragment->startInBits() || | |||
175 | Target.endInBits() > CurrentFragment->endInBits()) | |||
176 | return Skip; | |||
177 | ||||
178 | // Target fits within the current fragment, return it. | |||
179 | return UseFrag; | |||
180 | } | |||
181 | ||||
182 | static DebugVariable getAggregateVariable(DbgVariableIntrinsic *DVI) { | |||
183 | return DebugVariable(DVI->getVariable(), std::nullopt, | |||
184 | DVI->getDebugLoc().getInlinedAt()); | |||
185 | } | |||
186 | ||||
187 | /// Find linked dbg.assign and generate a new one with the correct | |||
188 | /// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the | |||
189 | /// value component is copied from the old dbg.assign to the new. | |||
190 | /// \param OldAlloca Alloca for the variable before splitting. | |||
191 | /// \param IsSplit True if the store (not necessarily alloca) | |||
192 | /// is being split. | |||
193 | /// \param OldAllocaOffsetInBits Offset of the slice taken from OldAlloca. | |||
194 | /// \param SliceSizeInBits New number of bits being written to. | |||
195 | /// \param OldInst Instruction that is being split. | |||
196 | /// \param Inst New instruction performing this part of the | |||
197 | /// split store. | |||
198 | /// \param Dest Store destination. | |||
199 | /// \param Value Stored value. | |||
200 | /// \param DL Datalayout. | |||
201 | static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, | |||
202 | uint64_t OldAllocaOffsetInBits, | |||
203 | uint64_t SliceSizeInBits, Instruction *OldInst, | |||
204 | Instruction *Inst, Value *Dest, Value *Value, | |||
205 | const DataLayout &DL) { | |||
206 | auto MarkerRange = at::getAssignmentMarkers(OldInst); | |||
207 | // Nothing to do if OldInst has no linked dbg.assign intrinsics. | |||
208 | if (MarkerRange.empty()) | |||
209 | return; | |||
210 | ||||
211 | LLVM_DEBUG(dbgs() << " migrateDebugInfo\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " migrateDebugInfo\n"; } } while (false); | |||
212 | LLVM_DEBUG(dbgs() << " OldAlloca: " << *OldAlloca << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " OldAlloca: " << *OldAlloca << "\n"; } } while (false); | |||
213 | LLVM_DEBUG(dbgs() << " IsSplit: " << IsSplit << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " IsSplit: " << IsSplit << "\n"; } } while (false); | |||
214 | LLVM_DEBUG(dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBitsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits << "\n"; } } while (false) | |||
215 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits << "\n"; } } while (false); | |||
216 | LLVM_DEBUG(dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n"; } } while (false); | |||
217 | LLVM_DEBUG(dbgs() << " OldInst: " << *OldInst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " OldInst: " << *OldInst << "\n"; } } while (false); | |||
218 | LLVM_DEBUG(dbgs() << " Inst: " << *Inst << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Inst: " << *Inst << "\n"; } } while (false); | |||
219 | LLVM_DEBUG(dbgs() << " Dest: " << *Dest << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Dest: " << *Dest << "\n"; } } while (false); | |||
220 | if (Value) | |||
221 | LLVM_DEBUG(dbgs() << " Value: " << *Value << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Value: " << *Value << "\n"; } } while (false); | |||
222 | ||||
223 | /// Map of aggregate variables to their fragment associated with OldAlloca. | |||
224 | DenseMap<DebugVariable, std::optional<DIExpression::FragmentInfo>> | |||
225 | BaseFragments; | |||
226 | for (auto *DAI : at::getAssignmentMarkers(OldAlloca)) | |||
227 | BaseFragments[getAggregateVariable(DAI)] = | |||
228 | DAI->getExpression()->getFragmentInfo(); | |||
229 | ||||
230 | // The new inst needs a DIAssignID unique metadata tag (if OldInst has | |||
231 | // one). It shouldn't already have one: assert this assumption. | |||
232 | assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID))(static_cast <bool> (!Inst->getMetadata(LLVMContext:: MD_DIAssignID)) ? void (0) : __assert_fail ("!Inst->getMetadata(LLVMContext::MD_DIAssignID)" , "llvm/lib/Transforms/Scalar/SROA.cpp", 232, __extension__ __PRETTY_FUNCTION__ )); | |||
233 | DIAssignID *NewID = nullptr; | |||
234 | auto &Ctx = Inst->getContext(); | |||
235 | DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false); | |||
236 | assert(OldAlloca->isStaticAlloca())(static_cast <bool> (OldAlloca->isStaticAlloca()) ? void (0) : __assert_fail ("OldAlloca->isStaticAlloca()", "llvm/lib/Transforms/Scalar/SROA.cpp" , 236, __extension__ __PRETTY_FUNCTION__)); | |||
237 | ||||
238 | for (DbgAssignIntrinsic *DbgAssign : MarkerRange) { | |||
239 | LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssigndo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " existing dbg.assign is: " << *DbgAssign << "\n"; } } while (false) | |||
240 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " existing dbg.assign is: " << *DbgAssign << "\n"; } } while (false); | |||
241 | auto *Expr = DbgAssign->getExpression(); | |||
242 | bool SetKillLocation = false; | |||
243 | ||||
244 | if (IsSplit) { | |||
245 | std::optional<DIExpression::FragmentInfo> BaseFragment; | |||
246 | { | |||
247 | auto R = BaseFragments.find(getAggregateVariable(DbgAssign)); | |||
248 | if (R == BaseFragments.end()) | |||
249 | continue; | |||
250 | BaseFragment = R->second; | |||
251 | } | |||
252 | std::optional<DIExpression::FragmentInfo> CurrentFragment = | |||
253 | Expr->getFragmentInfo(); | |||
254 | DIExpression::FragmentInfo NewFragment; | |||
255 | FragCalcResult Result = calculateFragment( | |||
256 | DbgAssign->getVariable(), OldAllocaOffsetInBits, SliceSizeInBits, | |||
257 | BaseFragment, CurrentFragment, NewFragment); | |||
258 | ||||
259 | if (Result == Skip) | |||
260 | continue; | |||
261 | if (Result == UseFrag && !(NewFragment == CurrentFragment)) { | |||
262 | if (CurrentFragment) { | |||
263 | // Rewrite NewFragment to be relative to the existing one (this is | |||
264 | // what createFragmentExpression wants). CalculateFragment has | |||
265 | // already resolved the size for us. FIXME: Should it return the | |||
266 | // relative fragment too? | |||
267 | NewFragment.OffsetInBits -= CurrentFragment->OffsetInBits; | |||
268 | } | |||
269 | // Add the new fragment info to the existing expression if possible. | |||
270 | if (auto E = DIExpression::createFragmentExpression( | |||
271 | Expr, NewFragment.OffsetInBits, NewFragment.SizeInBits)) { | |||
272 | Expr = *E; | |||
273 | } else { | |||
274 | // Otherwise, add the new fragment info to an empty expression and | |||
275 | // discard the value component of this dbg.assign as the value cannot | |||
276 | // be computed with the new fragment. | |||
277 | Expr = *DIExpression::createFragmentExpression( | |||
278 | DIExpression::get(Expr->getContext(), std::nullopt), | |||
279 | NewFragment.OffsetInBits, NewFragment.SizeInBits); | |||
280 | SetKillLocation = true; | |||
281 | } | |||
282 | } | |||
283 | } | |||
284 | ||||
285 | // If we haven't created a DIAssignID ID do that now and attach it to Inst. | |||
286 | if (!NewID) { | |||
287 | NewID = DIAssignID::getDistinct(Ctx); | |||
288 | Inst->setMetadata(LLVMContext::MD_DIAssignID, NewID); | |||
289 | } | |||
290 | ||||
291 | ::Value *NewValue = Value ? Value : DbgAssign->getValue(); | |||
292 | auto *NewAssign = DIB.insertDbgAssign( | |||
293 | Inst, NewValue, DbgAssign->getVariable(), Expr, Dest, | |||
294 | DIExpression::get(Ctx, std::nullopt), DbgAssign->getDebugLoc()); | |||
295 | ||||
296 | // If we've updated the value but the original dbg.assign has an arglist | |||
297 | // then kill it now - we can't use the requested new value. | |||
298 | // We can't replace the DIArgList with the new value as it'd leave | |||
299 | // the DIExpression in an invalid state (DW_OP_LLVM_arg operands without | |||
300 | // an arglist). And we can't keep the DIArgList in case the linked store | |||
301 | // is being split - in which case the DIArgList + expression may no longer | |||
302 | // be computing the correct value. | |||
303 | // This should be a very rare situation as it requires the value being | |||
304 | // stored to differ from the dbg.assign (i.e., the value has been | |||
305 | // represented differently in the debug intrinsic for some reason). | |||
306 | SetKillLocation |= | |||
307 | Value && (DbgAssign->hasArgList() || | |||
308 | !DbgAssign->getExpression()->isSingleLocationExpression()); | |||
309 | if (SetKillLocation) | |||
310 | NewAssign->setKillLocation(); | |||
311 | ||||
312 | // We could use more precision here at the cost of some additional (code) | |||
313 | // complexity - if the original dbg.assign was adjacent to its store, we | |||
314 | // could position this new dbg.assign adjacent to its store rather than the | |||
315 | // old dbg.assgn. That would result in interleaved dbg.assigns rather than | |||
316 | // what we get now: | |||
317 | // split store !1 | |||
318 | // split store !2 | |||
319 | // dbg.assign !1 | |||
320 | // dbg.assign !2 | |||
321 | // This (current behaviour) results results in debug assignments being | |||
322 | // noted as slightly offset (in code) from the store. In practice this | |||
323 | // should have little effect on the debugging experience due to the fact | |||
324 | // that all the split stores should get the same line number. | |||
325 | NewAssign->moveBefore(DbgAssign); | |||
326 | ||||
327 | NewAssign->setDebugLoc(DbgAssign->getDebugLoc()); | |||
328 | LLVM_DEBUG(dbgs() << "Created new assign intrinsic: " << *NewAssigndo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Created new assign intrinsic: " << *NewAssign << "\n"; } } while (false) | |||
329 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Created new assign intrinsic: " << *NewAssign << "\n"; } } while (false); | |||
330 | } | |||
331 | } | |||
332 | ||||
333 | /// A custom IRBuilder inserter which prefixes all names, but only in | |||
334 | /// Assert builds. | |||
335 | class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter { | |||
336 | std::string Prefix; | |||
337 | ||||
338 | Twine getNameWithPrefix(const Twine &Name) const { | |||
339 | return Name.isTriviallyEmpty() ? Name : Prefix + Name; | |||
340 | } | |||
341 | ||||
342 | public: | |||
343 | void SetNamePrefix(const Twine &P) { Prefix = P.str(); } | |||
344 | ||||
345 | void InsertHelper(Instruction *I, const Twine &Name, BasicBlock *BB, | |||
346 | BasicBlock::iterator InsertPt) const override { | |||
347 | IRBuilderDefaultInserter::InsertHelper(I, getNameWithPrefix(Name), BB, | |||
348 | InsertPt); | |||
349 | } | |||
350 | }; | |||
351 | ||||
352 | /// Provide a type for IRBuilder that drops names in release builds. | |||
353 | using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>; | |||
354 | ||||
355 | /// A used slice of an alloca. | |||
356 | /// | |||
357 | /// This structure represents a slice of an alloca used by some instruction. It | |||
358 | /// stores both the begin and end offsets of this use, a pointer to the use | |||
359 | /// itself, and a flag indicating whether we can classify the use as splittable | |||
360 | /// or not when forming partitions of the alloca. | |||
361 | class Slice { | |||
362 | /// The beginning offset of the range. | |||
363 | uint64_t BeginOffset = 0; | |||
364 | ||||
365 | /// The ending offset, not included in the range. | |||
366 | uint64_t EndOffset = 0; | |||
367 | ||||
368 | /// Storage for both the use of this slice and whether it can be | |||
369 | /// split. | |||
370 | PointerIntPair<Use *, 1, bool> UseAndIsSplittable; | |||
371 | ||||
372 | public: | |||
373 | Slice() = default; | |||
374 | ||||
375 | Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable) | |||
376 | : BeginOffset(BeginOffset), EndOffset(EndOffset), | |||
377 | UseAndIsSplittable(U, IsSplittable) {} | |||
378 | ||||
379 | uint64_t beginOffset() const { return BeginOffset; } | |||
380 | uint64_t endOffset() const { return EndOffset; } | |||
381 | ||||
382 | bool isSplittable() const { return UseAndIsSplittable.getInt(); } | |||
383 | void makeUnsplittable() { UseAndIsSplittable.setInt(false); } | |||
384 | ||||
385 | Use *getUse() const { return UseAndIsSplittable.getPointer(); } | |||
386 | ||||
387 | bool isDead() const { return getUse() == nullptr; } | |||
388 | void kill() { UseAndIsSplittable.setPointer(nullptr); } | |||
389 | ||||
390 | /// Support for ordering ranges. | |||
391 | /// | |||
392 | /// This provides an ordering over ranges such that start offsets are | |||
393 | /// always increasing, and within equal start offsets, the end offsets are | |||
394 | /// decreasing. Thus the spanning range comes first in a cluster with the | |||
395 | /// same start position. | |||
396 | bool operator<(const Slice &RHS) const { | |||
397 | if (beginOffset() < RHS.beginOffset()) | |||
398 | return true; | |||
399 | if (beginOffset() > RHS.beginOffset()) | |||
400 | return false; | |||
401 | if (isSplittable() != RHS.isSplittable()) | |||
402 | return !isSplittable(); | |||
403 | if (endOffset() > RHS.endOffset()) | |||
404 | return true; | |||
405 | return false; | |||
406 | } | |||
407 | ||||
408 | /// Support comparison with a single offset to allow binary searches. | |||
409 | friend LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__)) bool operator<(const Slice &LHS, | |||
410 | uint64_t RHSOffset) { | |||
411 | return LHS.beginOffset() < RHSOffset; | |||
412 | } | |||
413 | friend LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__)) bool operator<(uint64_t LHSOffset, | |||
414 | const Slice &RHS) { | |||
415 | return LHSOffset < RHS.beginOffset(); | |||
416 | } | |||
417 | ||||
418 | bool operator==(const Slice &RHS) const { | |||
419 | return isSplittable() == RHS.isSplittable() && | |||
420 | beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset(); | |||
421 | } | |||
422 | bool operator!=(const Slice &RHS) const { return !operator==(RHS); } | |||
423 | }; | |||
424 | ||||
425 | } // end anonymous namespace | |||
426 | ||||
427 | /// Representation of the alloca slices. | |||
428 | /// | |||
429 | /// This class represents the slices of an alloca which are formed by its | |||
430 | /// various uses. If a pointer escapes, we can't fully build a representation | |||
431 | /// for the slices used and we reflect that in this structure. The uses are | |||
432 | /// stored, sorted by increasing beginning offset and with unsplittable slices | |||
433 | /// starting at a particular offset before splittable slices. | |||
434 | class llvm::sroa::AllocaSlices { | |||
435 | public: | |||
436 | /// Construct the slices of a particular alloca. | |||
437 | AllocaSlices(const DataLayout &DL, AllocaInst &AI); | |||
438 | ||||
439 | /// Test whether a pointer to the allocation escapes our analysis. | |||
440 | /// | |||
441 | /// If this is true, the slices are never fully built and should be | |||
442 | /// ignored. | |||
443 | bool isEscaped() const { return PointerEscapingInstr; } | |||
444 | ||||
445 | /// Support for iterating over the slices. | |||
446 | /// @{ | |||
447 | using iterator = SmallVectorImpl<Slice>::iterator; | |||
448 | using range = iterator_range<iterator>; | |||
449 | ||||
450 | iterator begin() { return Slices.begin(); } | |||
451 | iterator end() { return Slices.end(); } | |||
452 | ||||
453 | using const_iterator = SmallVectorImpl<Slice>::const_iterator; | |||
454 | using const_range = iterator_range<const_iterator>; | |||
455 | ||||
456 | const_iterator begin() const { return Slices.begin(); } | |||
457 | const_iterator end() const { return Slices.end(); } | |||
458 | /// @} | |||
459 | ||||
460 | /// Erase a range of slices. | |||
461 | void erase(iterator Start, iterator Stop) { Slices.erase(Start, Stop); } | |||
462 | ||||
463 | /// Insert new slices for this alloca. | |||
464 | /// | |||
465 | /// This moves the slices into the alloca's slices collection, and re-sorts | |||
466 | /// everything so that the usual ordering properties of the alloca's slices | |||
467 | /// hold. | |||
468 | void insert(ArrayRef<Slice> NewSlices) { | |||
469 | int OldSize = Slices.size(); | |||
470 | Slices.append(NewSlices.begin(), NewSlices.end()); | |||
471 | auto SliceI = Slices.begin() + OldSize; | |||
472 | llvm::sort(SliceI, Slices.end()); | |||
473 | std::inplace_merge(Slices.begin(), SliceI, Slices.end()); | |||
474 | } | |||
475 | ||||
476 | // Forward declare the iterator and range accessor for walking the | |||
477 | // partitions. | |||
478 | class partition_iterator; | |||
479 | iterator_range<partition_iterator> partitions(); | |||
480 | ||||
481 | /// Access the dead users for this alloca. | |||
482 | ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; } | |||
483 | ||||
484 | /// Access Uses that should be dropped if the alloca is promotable. | |||
485 | ArrayRef<Use *> getDeadUsesIfPromotable() const { | |||
486 | return DeadUseIfPromotable; | |||
487 | } | |||
488 | ||||
489 | /// Access the dead operands referring to this alloca. | |||
490 | /// | |||
491 | /// These are operands which have cannot actually be used to refer to the | |||
492 | /// alloca as they are outside its range and the user doesn't correct for | |||
493 | /// that. These mostly consist of PHI node inputs and the like which we just | |||
494 | /// need to replace with undef. | |||
495 | ArrayRef<Use *> getDeadOperands() const { return DeadOperands; } | |||
496 | ||||
497 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
498 | void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const; | |||
499 | void printSlice(raw_ostream &OS, const_iterator I, | |||
500 | StringRef Indent = " ") const; | |||
501 | void printUse(raw_ostream &OS, const_iterator I, | |||
502 | StringRef Indent = " ") const; | |||
503 | void print(raw_ostream &OS) const; | |||
504 | void dump(const_iterator I) const; | |||
505 | void dump() const; | |||
506 | #endif | |||
507 | ||||
508 | private: | |||
509 | template <typename DerivedT, typename RetT = void> class BuilderBase; | |||
510 | class SliceBuilder; | |||
511 | ||||
512 | friend class AllocaSlices::SliceBuilder; | |||
513 | ||||
514 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
515 | /// Handle to alloca instruction to simplify method interfaces. | |||
516 | AllocaInst &AI; | |||
517 | #endif | |||
518 | ||||
519 | /// The instruction responsible for this alloca not having a known set | |||
520 | /// of slices. | |||
521 | /// | |||
522 | /// When an instruction (potentially) escapes the pointer to the alloca, we | |||
523 | /// store a pointer to that here and abort trying to form slices of the | |||
524 | /// alloca. This will be null if the alloca slices are analyzed successfully. | |||
525 | Instruction *PointerEscapingInstr; | |||
526 | ||||
527 | /// The slices of the alloca. | |||
528 | /// | |||
529 | /// We store a vector of the slices formed by uses of the alloca here. This | |||
530 | /// vector is sorted by increasing begin offset, and then the unsplittable | |||
531 | /// slices before the splittable ones. See the Slice inner class for more | |||
532 | /// details. | |||
533 | SmallVector<Slice, 8> Slices; | |||
534 | ||||
535 | /// Instructions which will become dead if we rewrite the alloca. | |||
536 | /// | |||
537 | /// Note that these are not separated by slice. This is because we expect an | |||
538 | /// alloca to be completely rewritten or not rewritten at all. If rewritten, | |||
539 | /// all these instructions can simply be removed and replaced with poison as | |||
540 | /// they come from outside of the allocated space. | |||
541 | SmallVector<Instruction *, 8> DeadUsers; | |||
542 | ||||
543 | /// Uses which will become dead if can promote the alloca. | |||
544 | SmallVector<Use *, 8> DeadUseIfPromotable; | |||
545 | ||||
546 | /// Operands which will become dead if we rewrite the alloca. | |||
547 | /// | |||
548 | /// These are operands that in their particular use can be replaced with | |||
549 | /// poison when we rewrite the alloca. These show up in out-of-bounds inputs | |||
550 | /// to PHI nodes and the like. They aren't entirely dead (there might be | |||
551 | /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we | |||
552 | /// want to swap this particular input for poison to simplify the use lists of | |||
553 | /// the alloca. | |||
554 | SmallVector<Use *, 8> DeadOperands; | |||
555 | }; | |||
556 | ||||
557 | /// A partition of the slices. | |||
558 | /// | |||
559 | /// An ephemeral representation for a range of slices which can be viewed as | |||
560 | /// a partition of the alloca. This range represents a span of the alloca's | |||
561 | /// memory which cannot be split, and provides access to all of the slices | |||
562 | /// overlapping some part of the partition. | |||
563 | /// | |||
564 | /// Objects of this type are produced by traversing the alloca's slices, but | |||
565 | /// are only ephemeral and not persistent. | |||
566 | class llvm::sroa::Partition { | |||
567 | private: | |||
568 | friend class AllocaSlices; | |||
569 | friend class AllocaSlices::partition_iterator; | |||
570 | ||||
571 | using iterator = AllocaSlices::iterator; | |||
572 | ||||
573 | /// The beginning and ending offsets of the alloca for this | |||
574 | /// partition. | |||
575 | uint64_t BeginOffset = 0, EndOffset = 0; | |||
576 | ||||
577 | /// The start and end iterators of this partition. | |||
578 | iterator SI, SJ; | |||
579 | ||||
580 | /// A collection of split slice tails overlapping the partition. | |||
581 | SmallVector<Slice *, 4> SplitTails; | |||
582 | ||||
583 | /// Raw constructor builds an empty partition starting and ending at | |||
584 | /// the given iterator. | |||
585 | Partition(iterator SI) : SI(SI), SJ(SI) {} | |||
586 | ||||
587 | public: | |||
588 | /// The start offset of this partition. | |||
589 | /// | |||
590 | /// All of the contained slices start at or after this offset. | |||
591 | uint64_t beginOffset() const { return BeginOffset; } | |||
592 | ||||
593 | /// The end offset of this partition. | |||
594 | /// | |||
595 | /// All of the contained slices end at or before this offset. | |||
596 | uint64_t endOffset() const { return EndOffset; } | |||
597 | ||||
598 | /// The size of the partition. | |||
599 | /// | |||
600 | /// Note that this can never be zero. | |||
601 | uint64_t size() const { | |||
602 | assert(BeginOffset < EndOffset && "Partitions must span some bytes!")(static_cast <bool> (BeginOffset < EndOffset && "Partitions must span some bytes!") ? void (0) : __assert_fail ("BeginOffset < EndOffset && \"Partitions must span some bytes!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 602, __extension__ __PRETTY_FUNCTION__ )); | |||
603 | return EndOffset - BeginOffset; | |||
604 | } | |||
605 | ||||
606 | /// Test whether this partition contains no slices, and merely spans | |||
607 | /// a region occupied by split slices. | |||
608 | bool empty() const { return SI == SJ; } | |||
609 | ||||
610 | /// \name Iterate slices that start within the partition. | |||
611 | /// These may be splittable or unsplittable. They have a begin offset >= the | |||
612 | /// partition begin offset. | |||
613 | /// @{ | |||
614 | // FIXME: We should probably define a "concat_iterator" helper and use that | |||
615 | // to stitch together pointee_iterators over the split tails and the | |||
616 | // contiguous iterators of the partition. That would give a much nicer | |||
617 | // interface here. We could then additionally expose filtered iterators for | |||
618 | // split, unsplit, and unsplittable splices based on the usage patterns. | |||
619 | iterator begin() const { return SI; } | |||
620 | iterator end() const { return SJ; } | |||
621 | /// @} | |||
622 | ||||
623 | /// Get the sequence of split slice tails. | |||
624 | /// | |||
625 | /// These tails are of slices which start before this partition but are | |||
626 | /// split and overlap into the partition. We accumulate these while forming | |||
627 | /// partitions. | |||
628 | ArrayRef<Slice *> splitSliceTails() const { return SplitTails; } | |||
629 | }; | |||
630 | ||||
631 | /// An iterator over partitions of the alloca's slices. | |||
632 | /// | |||
633 | /// This iterator implements the core algorithm for partitioning the alloca's | |||
634 | /// slices. It is a forward iterator as we don't support backtracking for | |||
635 | /// efficiency reasons, and re-use a single storage area to maintain the | |||
636 | /// current set of split slices. | |||
637 | /// | |||
638 | /// It is templated on the slice iterator type to use so that it can operate | |||
639 | /// with either const or non-const slice iterators. | |||
640 | class AllocaSlices::partition_iterator | |||
641 | : public iterator_facade_base<partition_iterator, std::forward_iterator_tag, | |||
642 | Partition> { | |||
643 | friend class AllocaSlices; | |||
644 | ||||
645 | /// Most of the state for walking the partitions is held in a class | |||
646 | /// with a nice interface for examining them. | |||
647 | Partition P; | |||
648 | ||||
649 | /// We need to keep the end of the slices to know when to stop. | |||
650 | AllocaSlices::iterator SE; | |||
651 | ||||
652 | /// We also need to keep track of the maximum split end offset seen. | |||
653 | /// FIXME: Do we really? | |||
654 | uint64_t MaxSplitSliceEndOffset = 0; | |||
655 | ||||
656 | /// Sets the partition to be empty at given iterator, and sets the | |||
657 | /// end iterator. | |||
658 | partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE) | |||
659 | : P(SI), SE(SE) { | |||
660 | // If not already at the end, advance our state to form the initial | |||
661 | // partition. | |||
662 | if (SI != SE) | |||
663 | advance(); | |||
664 | } | |||
665 | ||||
666 | /// Advance the iterator to the next partition. | |||
667 | /// | |||
668 | /// Requires that the iterator not be at the end of the slices. | |||
669 | void advance() { | |||
670 | assert((P.SI != SE || !P.SplitTails.empty()) &&(static_cast <bool> ((P.SI != SE || !P.SplitTails.empty ()) && "Cannot advance past the end of the slices!") ? void (0) : __assert_fail ("(P.SI != SE || !P.SplitTails.empty()) && \"Cannot advance past the end of the slices!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 671, __extension__ __PRETTY_FUNCTION__ )) | |||
671 | "Cannot advance past the end of the slices!")(static_cast <bool> ((P.SI != SE || !P.SplitTails.empty ()) && "Cannot advance past the end of the slices!") ? void (0) : __assert_fail ("(P.SI != SE || !P.SplitTails.empty()) && \"Cannot advance past the end of the slices!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 671, __extension__ __PRETTY_FUNCTION__ )); | |||
672 | ||||
673 | // Clear out any split uses which have ended. | |||
674 | if (!P.SplitTails.empty()) { | |||
675 | if (P.EndOffset >= MaxSplitSliceEndOffset) { | |||
676 | // If we've finished all splits, this is easy. | |||
677 | P.SplitTails.clear(); | |||
678 | MaxSplitSliceEndOffset = 0; | |||
679 | } else { | |||
680 | // Remove the uses which have ended in the prior partition. This | |||
681 | // cannot change the max split slice end because we just checked that | |||
682 | // the prior partition ended prior to that max. | |||
683 | llvm::erase_if(P.SplitTails, | |||
684 | [&](Slice *S) { return S->endOffset() <= P.EndOffset; }); | |||
685 | assert(llvm::any_of(P.SplitTails,(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 689, __extension__ __PRETTY_FUNCTION__ )) | |||
686 | [&](Slice *S) {(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 689, __extension__ __PRETTY_FUNCTION__ )) | |||
687 | return S->endOffset() == MaxSplitSliceEndOffset;(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 689, __extension__ __PRETTY_FUNCTION__ )) | |||
688 | }) &&(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 689, __extension__ __PRETTY_FUNCTION__ )) | |||
689 | "Could not find the current max split slice offset!")(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 689, __extension__ __PRETTY_FUNCTION__ )); | |||
690 | assert(llvm::all_of(P.SplitTails,(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 694, __extension__ __PRETTY_FUNCTION__ )) | |||
691 | [&](Slice *S) {(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 694, __extension__ __PRETTY_FUNCTION__ )) | |||
692 | return S->endOffset() <= MaxSplitSliceEndOffset;(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 694, __extension__ __PRETTY_FUNCTION__ )) | |||
693 | }) &&(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 694, __extension__ __PRETTY_FUNCTION__ )) | |||
694 | "Max split slice end offset is not actually the max!")(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 694, __extension__ __PRETTY_FUNCTION__ )); | |||
695 | } | |||
696 | } | |||
697 | ||||
698 | // If P.SI is already at the end, then we've cleared the split tail and | |||
699 | // now have an end iterator. | |||
700 | if (P.SI == SE) { | |||
701 | assert(P.SplitTails.empty() && "Failed to clear the split slices!")(static_cast <bool> (P.SplitTails.empty() && "Failed to clear the split slices!" ) ? void (0) : __assert_fail ("P.SplitTails.empty() && \"Failed to clear the split slices!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 701, __extension__ __PRETTY_FUNCTION__ )); | |||
702 | return; | |||
703 | } | |||
704 | ||||
705 | // If we had a non-empty partition previously, set up the state for | |||
706 | // subsequent partitions. | |||
707 | if (P.SI != P.SJ) { | |||
708 | // Accumulate all the splittable slices which started in the old | |||
709 | // partition into the split list. | |||
710 | for (Slice &S : P) | |||
711 | if (S.isSplittable() && S.endOffset() > P.EndOffset) { | |||
712 | P.SplitTails.push_back(&S); | |||
713 | MaxSplitSliceEndOffset = | |||
714 | std::max(S.endOffset(), MaxSplitSliceEndOffset); | |||
715 | } | |||
716 | ||||
717 | // Start from the end of the previous partition. | |||
718 | P.SI = P.SJ; | |||
719 | ||||
720 | // If P.SI is now at the end, we at most have a tail of split slices. | |||
721 | if (P.SI == SE) { | |||
722 | P.BeginOffset = P.EndOffset; | |||
723 | P.EndOffset = MaxSplitSliceEndOffset; | |||
724 | return; | |||
725 | } | |||
726 | ||||
727 | // If the we have split slices and the next slice is after a gap and is | |||
728 | // not splittable immediately form an empty partition for the split | |||
729 | // slices up until the next slice begins. | |||
730 | if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset && | |||
731 | !P.SI->isSplittable()) { | |||
732 | P.BeginOffset = P.EndOffset; | |||
733 | P.EndOffset = P.SI->beginOffset(); | |||
734 | return; | |||
735 | } | |||
736 | } | |||
737 | ||||
738 | // OK, we need to consume new slices. Set the end offset based on the | |||
739 | // current slice, and step SJ past it. The beginning offset of the | |||
740 | // partition is the beginning offset of the next slice unless we have | |||
741 | // pre-existing split slices that are continuing, in which case we begin | |||
742 | // at the prior end offset. | |||
743 | P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset; | |||
744 | P.EndOffset = P.SI->endOffset(); | |||
745 | ++P.SJ; | |||
746 | ||||
747 | // There are two strategies to form a partition based on whether the | |||
748 | // partition starts with an unsplittable slice or a splittable slice. | |||
749 | if (!P.SI->isSplittable()) { | |||
750 | // When we're forming an unsplittable region, it must always start at | |||
751 | // the first slice and will extend through its end. | |||
752 | assert(P.BeginOffset == P.SI->beginOffset())(static_cast <bool> (P.BeginOffset == P.SI->beginOffset ()) ? void (0) : __assert_fail ("P.BeginOffset == P.SI->beginOffset()" , "llvm/lib/Transforms/Scalar/SROA.cpp", 752, __extension__ __PRETTY_FUNCTION__ )); | |||
753 | ||||
754 | // Form a partition including all of the overlapping slices with this | |||
755 | // unsplittable slice. | |||
756 | while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) { | |||
757 | if (!P.SJ->isSplittable()) | |||
758 | P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset()); | |||
759 | ++P.SJ; | |||
760 | } | |||
761 | ||||
762 | // We have a partition across a set of overlapping unsplittable | |||
763 | // partitions. | |||
764 | return; | |||
765 | } | |||
766 | ||||
767 | // If we're starting with a splittable slice, then we need to form | |||
768 | // a synthetic partition spanning it and any other overlapping splittable | |||
769 | // splices. | |||
770 | assert(P.SI->isSplittable() && "Forming a splittable partition!")(static_cast <bool> (P.SI->isSplittable() && "Forming a splittable partition!") ? void (0) : __assert_fail ("P.SI->isSplittable() && \"Forming a splittable partition!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 770, __extension__ __PRETTY_FUNCTION__ )); | |||
771 | ||||
772 | // Collect all of the overlapping splittable slices. | |||
773 | while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset && | |||
774 | P.SJ->isSplittable()) { | |||
775 | P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset()); | |||
776 | ++P.SJ; | |||
777 | } | |||
778 | ||||
779 | // Back upiP.EndOffset if we ended the span early when encountering an | |||
780 | // unsplittable slice. This synthesizes the early end offset of | |||
781 | // a partition spanning only splittable slices. | |||
782 | if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) { | |||
783 | assert(!P.SJ->isSplittable())(static_cast <bool> (!P.SJ->isSplittable()) ? void ( 0) : __assert_fail ("!P.SJ->isSplittable()", "llvm/lib/Transforms/Scalar/SROA.cpp" , 783, __extension__ __PRETTY_FUNCTION__)); | |||
784 | P.EndOffset = P.SJ->beginOffset(); | |||
785 | } | |||
786 | } | |||
787 | ||||
788 | public: | |||
789 | bool operator==(const partition_iterator &RHS) const { | |||
790 | assert(SE == RHS.SE &&(static_cast <bool> (SE == RHS.SE && "End iterators don't match between compared partition iterators!" ) ? void (0) : __assert_fail ("SE == RHS.SE && \"End iterators don't match between compared partition iterators!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 791, __extension__ __PRETTY_FUNCTION__ )) | |||
791 | "End iterators don't match between compared partition iterators!")(static_cast <bool> (SE == RHS.SE && "End iterators don't match between compared partition iterators!" ) ? void (0) : __assert_fail ("SE == RHS.SE && \"End iterators don't match between compared partition iterators!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 791, __extension__ __PRETTY_FUNCTION__ )); | |||
792 | ||||
793 | // The observed positions of partitions is marked by the P.SI iterator and | |||
794 | // the emptiness of the split slices. The latter is only relevant when | |||
795 | // P.SI == SE, as the end iterator will additionally have an empty split | |||
796 | // slices list, but the prior may have the same P.SI and a tail of split | |||
797 | // slices. | |||
798 | if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) { | |||
799 | assert(P.SJ == RHS.P.SJ &&(static_cast <bool> (P.SJ == RHS.P.SJ && "Same set of slices formed two different sized partitions!" ) ? void (0) : __assert_fail ("P.SJ == RHS.P.SJ && \"Same set of slices formed two different sized partitions!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 800, __extension__ __PRETTY_FUNCTION__ )) | |||
800 | "Same set of slices formed two different sized partitions!")(static_cast <bool> (P.SJ == RHS.P.SJ && "Same set of slices formed two different sized partitions!" ) ? void (0) : __assert_fail ("P.SJ == RHS.P.SJ && \"Same set of slices formed two different sized partitions!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 800, __extension__ __PRETTY_FUNCTION__ )); | |||
801 | assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&(static_cast <bool> (P.SplitTails.size() == RHS.P.SplitTails .size() && "Same slice position with differently sized non-empty split " "slice tails!") ? void (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 803, __extension__ __PRETTY_FUNCTION__ )) | |||
802 | "Same slice position with differently sized non-empty split "(static_cast <bool> (P.SplitTails.size() == RHS.P.SplitTails .size() && "Same slice position with differently sized non-empty split " "slice tails!") ? void (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 803, __extension__ __PRETTY_FUNCTION__ )) | |||
803 | "slice tails!")(static_cast <bool> (P.SplitTails.size() == RHS.P.SplitTails .size() && "Same slice position with differently sized non-empty split " "slice tails!") ? void (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 803, __extension__ __PRETTY_FUNCTION__ )); | |||
804 | return true; | |||
805 | } | |||
806 | return false; | |||
807 | } | |||
808 | ||||
809 | partition_iterator &operator++() { | |||
810 | advance(); | |||
811 | return *this; | |||
812 | } | |||
813 | ||||
814 | Partition &operator*() { return P; } | |||
815 | }; | |||
816 | ||||
817 | /// A forward range over the partitions of the alloca's slices. | |||
818 | /// | |||
819 | /// This accesses an iterator range over the partitions of the alloca's | |||
820 | /// slices. It computes these partitions on the fly based on the overlapping | |||
821 | /// offsets of the slices and the ability to split them. It will visit "empty" | |||
822 | /// partitions to cover regions of the alloca only accessed via split | |||
823 | /// slices. | |||
824 | iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() { | |||
825 | return make_range(partition_iterator(begin(), end()), | |||
826 | partition_iterator(end(), end())); | |||
827 | } | |||
828 | ||||
829 | static Value *foldSelectInst(SelectInst &SI) { | |||
830 | // If the condition being selected on is a constant or the same value is | |||
831 | // being selected between, fold the select. Yes this does (rarely) happen | |||
832 | // early on. | |||
833 | if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition())) | |||
834 | return SI.getOperand(1 + CI->isZero()); | |||
835 | if (SI.getOperand(1) == SI.getOperand(2)) | |||
836 | return SI.getOperand(1); | |||
837 | ||||
838 | return nullptr; | |||
839 | } | |||
840 | ||||
841 | /// A helper that folds a PHI node or a select. | |||
842 | static Value *foldPHINodeOrSelectInst(Instruction &I) { | |||
843 | if (PHINode *PN = dyn_cast<PHINode>(&I)) { | |||
844 | // If PN merges together the same value, return that value. | |||
845 | return PN->hasConstantValue(); | |||
846 | } | |||
847 | return foldSelectInst(cast<SelectInst>(I)); | |||
848 | } | |||
849 | ||||
850 | /// Builder for the alloca slices. | |||
851 | /// | |||
852 | /// This class builds a set of alloca slices by recursively visiting the uses | |||
853 | /// of an alloca and making a slice for each load and store at each offset. | |||
854 | class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> { | |||
855 | friend class PtrUseVisitor<SliceBuilder>; | |||
856 | friend class InstVisitor<SliceBuilder>; | |||
857 | ||||
858 | using Base = PtrUseVisitor<SliceBuilder>; | |||
859 | ||||
860 | const uint64_t AllocSize; | |||
861 | AllocaSlices &AS; | |||
862 | ||||
863 | SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap; | |||
864 | SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes; | |||
865 | ||||
866 | /// Set to de-duplicate dead instructions found in the use walk. | |||
867 | SmallPtrSet<Instruction *, 4> VisitedDeadInsts; | |||
868 | ||||
869 | public: | |||
870 | SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS) | |||
871 | : PtrUseVisitor<SliceBuilder>(DL), | |||
872 | AllocSize(DL.getTypeAllocSize(AI.getAllocatedType()).getFixedValue()), | |||
873 | AS(AS) {} | |||
874 | ||||
875 | private: | |||
876 | void markAsDead(Instruction &I) { | |||
877 | if (VisitedDeadInsts.insert(&I).second) | |||
878 | AS.DeadUsers.push_back(&I); | |||
879 | } | |||
880 | ||||
881 | void insertUse(Instruction &I, const APInt &Offset, uint64_t Size, | |||
882 | bool IsSplittable = false) { | |||
883 | // Completely skip uses which have a zero size or start either before or | |||
884 | // past the end of the allocation. | |||
885 | if (Size == 0 || Offset.uge(AllocSize)) { | |||
886 | LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
887 | << Offsetdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
888 | << " which has zero size or starts outside of the "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
889 | << AllocSize << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
890 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
891 | << " use: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false); | |||
892 | return markAsDead(I); | |||
893 | } | |||
894 | ||||
895 | uint64_t BeginOffset = Offset.getZExtValue(); | |||
896 | uint64_t EndOffset = BeginOffset + Size; | |||
897 | ||||
898 | // Clamp the end offset to the end of the allocation. Note that this is | |||
899 | // formulated to handle even the case where "BeginOffset + Size" overflows. | |||
900 | // This may appear superficially to be something we could ignore entirely, | |||
901 | // but that is not so! There may be widened loads or PHI-node uses where | |||
902 | // some instructions are dead but not others. We can't completely ignore | |||
903 | // them, and so have to record at least the information here. | |||
904 | assert(AllocSize >= BeginOffset)(static_cast <bool> (AllocSize >= BeginOffset) ? void (0) : __assert_fail ("AllocSize >= BeginOffset", "llvm/lib/Transforms/Scalar/SROA.cpp" , 904, __extension__ __PRETTY_FUNCTION__)); // Established above. | |||
905 | if (Size > AllocSize - BeginOffset) { | |||
906 | LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
907 | << Offset << " to remain within the " << AllocSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
908 | << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
909 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
910 | << " use: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false); | |||
911 | EndOffset = AllocSize; | |||
912 | } | |||
913 | ||||
914 | AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable)); | |||
915 | } | |||
916 | ||||
917 | void visitBitCastInst(BitCastInst &BC) { | |||
918 | if (BC.use_empty()) | |||
919 | return markAsDead(BC); | |||
920 | ||||
921 | return Base::visitBitCastInst(BC); | |||
922 | } | |||
923 | ||||
924 | void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) { | |||
925 | if (ASC.use_empty()) | |||
926 | return markAsDead(ASC); | |||
927 | ||||
928 | return Base::visitAddrSpaceCastInst(ASC); | |||
929 | } | |||
930 | ||||
931 | void visitGetElementPtrInst(GetElementPtrInst &GEPI) { | |||
932 | if (GEPI.use_empty()) | |||
933 | return markAsDead(GEPI); | |||
934 | ||||
935 | if (SROAStrictInbounds && GEPI.isInBounds()) { | |||
936 | // FIXME: This is a manually un-factored variant of the basic code inside | |||
937 | // of GEPs with checking of the inbounds invariant specified in the | |||
938 | // langref in a very strict sense. If we ever want to enable | |||
939 | // SROAStrictInbounds, this code should be factored cleanly into | |||
940 | // PtrUseVisitor, but it is easier to experiment with SROAStrictInbounds | |||
941 | // by writing out the code here where we have the underlying allocation | |||
942 | // size readily available. | |||
943 | APInt GEPOffset = Offset; | |||
944 | const DataLayout &DL = GEPI.getModule()->getDataLayout(); | |||
945 | for (gep_type_iterator GTI = gep_type_begin(GEPI), | |||
946 | GTE = gep_type_end(GEPI); | |||
947 | GTI != GTE; ++GTI) { | |||
948 | ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand()); | |||
949 | if (!OpC) | |||
950 | break; | |||
951 | ||||
952 | // Handle a struct index, which adds its field offset to the pointer. | |||
953 | if (StructType *STy = GTI.getStructTypeOrNull()) { | |||
954 | unsigned ElementIdx = OpC->getZExtValue(); | |||
955 | const StructLayout *SL = DL.getStructLayout(STy); | |||
956 | GEPOffset += | |||
957 | APInt(Offset.getBitWidth(), SL->getElementOffset(ElementIdx)); | |||
958 | } else { | |||
959 | // For array or vector indices, scale the index by the size of the | |||
960 | // type. | |||
961 | APInt Index = OpC->getValue().sextOrTrunc(Offset.getBitWidth()); | |||
962 | GEPOffset += | |||
963 | Index * | |||
964 | APInt(Offset.getBitWidth(), | |||
965 | DL.getTypeAllocSize(GTI.getIndexedType()).getFixedValue()); | |||
966 | } | |||
967 | ||||
968 | // If this index has computed an intermediate pointer which is not | |||
969 | // inbounds, then the result of the GEP is a poison value and we can | |||
970 | // delete it and all uses. | |||
971 | if (GEPOffset.ugt(AllocSize)) | |||
972 | return markAsDead(GEPI); | |||
973 | } | |||
974 | } | |||
975 | ||||
976 | return Base::visitGetElementPtrInst(GEPI); | |||
977 | } | |||
978 | ||||
979 | void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset, | |||
980 | uint64_t Size, bool IsVolatile) { | |||
981 | // We allow splitting of non-volatile loads and stores where the type is an | |||
982 | // integer type. These may be used to implement 'memcpy' or other "transfer | |||
983 | // of bits" patterns. | |||
984 | bool IsSplittable = | |||
985 | Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty); | |||
986 | ||||
987 | insertUse(I, Offset, Size, IsSplittable); | |||
988 | } | |||
989 | ||||
990 | void visitLoadInst(LoadInst &LI) { | |||
991 | assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&(static_cast <bool> ((!LI.isSimple() || LI.getType()-> isSingleValueType()) && "All simple FCA loads should have been pre-split" ) ? void (0) : __assert_fail ("(!LI.isSimple() || LI.getType()->isSingleValueType()) && \"All simple FCA loads should have been pre-split\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 992, __extension__ __PRETTY_FUNCTION__ )) | |||
992 | "All simple FCA loads should have been pre-split")(static_cast <bool> ((!LI.isSimple() || LI.getType()-> isSingleValueType()) && "All simple FCA loads should have been pre-split" ) ? void (0) : __assert_fail ("(!LI.isSimple() || LI.getType()->isSingleValueType()) && \"All simple FCA loads should have been pre-split\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 992, __extension__ __PRETTY_FUNCTION__ )); | |||
993 | ||||
994 | if (!IsOffsetKnown) | |||
995 | return PI.setAborted(&LI); | |||
996 | ||||
997 | TypeSize Size = DL.getTypeStoreSize(LI.getType()); | |||
998 | if (Size.isScalable()) | |||
999 | return PI.setAborted(&LI); | |||
1000 | ||||
1001 | return handleLoadOrStore(LI.getType(), LI, Offset, Size.getFixedValue(), | |||
1002 | LI.isVolatile()); | |||
1003 | } | |||
1004 | ||||
1005 | void visitStoreInst(StoreInst &SI) { | |||
1006 | Value *ValOp = SI.getValueOperand(); | |||
1007 | if (ValOp == *U) | |||
1008 | return PI.setEscapedAndAborted(&SI); | |||
1009 | if (!IsOffsetKnown) | |||
1010 | return PI.setAborted(&SI); | |||
1011 | ||||
1012 | TypeSize StoreSize = DL.getTypeStoreSize(ValOp->getType()); | |||
1013 | if (StoreSize.isScalable()) | |||
1014 | return PI.setAborted(&SI); | |||
1015 | ||||
1016 | uint64_t Size = StoreSize.getFixedValue(); | |||
1017 | ||||
1018 | // If this memory access can be shown to *statically* extend outside the | |||
1019 | // bounds of the allocation, it's behavior is undefined, so simply | |||
1020 | // ignore it. Note that this is more strict than the generic clamping | |||
1021 | // behavior of insertUse. We also try to handle cases which might run the | |||
1022 | // risk of overflow. | |||
1023 | // FIXME: We should instead consider the pointer to have escaped if this | |||
1024 | // function is being instrumented for addressing bugs or race conditions. | |||
1025 | if (Size > AllocSize || Offset.ugt(AllocSize - Size)) { | |||
1026 | LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||
1027 | << Offset << " which extends past the end of the "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||
1028 | << AllocSize << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||
1029 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||
1030 | << " use: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false); | |||
1031 | return markAsDead(SI); | |||
1032 | } | |||
1033 | ||||
1034 | assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&(static_cast <bool> ((!SI.isSimple() || ValOp->getType ()->isSingleValueType()) && "All simple FCA stores should have been pre-split" ) ? void (0) : __assert_fail ("(!SI.isSimple() || ValOp->getType()->isSingleValueType()) && \"All simple FCA stores should have been pre-split\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1035, __extension__ __PRETTY_FUNCTION__ )) | |||
1035 | "All simple FCA stores should have been pre-split")(static_cast <bool> ((!SI.isSimple() || ValOp->getType ()->isSingleValueType()) && "All simple FCA stores should have been pre-split" ) ? void (0) : __assert_fail ("(!SI.isSimple() || ValOp->getType()->isSingleValueType()) && \"All simple FCA stores should have been pre-split\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1035, __extension__ __PRETTY_FUNCTION__ )); | |||
1036 | handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile()); | |||
1037 | } | |||
1038 | ||||
1039 | void visitMemSetInst(MemSetInst &II) { | |||
1040 | assert(II.getRawDest() == *U && "Pointer use is not the destination?")(static_cast <bool> (II.getRawDest() == *U && "Pointer use is not the destination?" ) ? void (0) : __assert_fail ("II.getRawDest() == *U && \"Pointer use is not the destination?\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1040, __extension__ __PRETTY_FUNCTION__ )); | |||
1041 | ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); | |||
1042 | if ((Length && Length->getValue() == 0) || | |||
1043 | (IsOffsetKnown && Offset.uge(AllocSize))) | |||
1044 | // Zero-length mem transfer intrinsics can be ignored entirely. | |||
1045 | return markAsDead(II); | |||
1046 | ||||
1047 | if (!IsOffsetKnown) | |||
1048 | return PI.setAborted(&II); | |||
1049 | ||||
1050 | insertUse(II, Offset, Length ? Length->getLimitedValue() | |||
1051 | : AllocSize - Offset.getLimitedValue(), | |||
1052 | (bool)Length); | |||
1053 | } | |||
1054 | ||||
1055 | void visitMemTransferInst(MemTransferInst &II) { | |||
1056 | ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); | |||
1057 | if (Length && Length->getValue() == 0) | |||
1058 | // Zero-length mem transfer intrinsics can be ignored entirely. | |||
1059 | return markAsDead(II); | |||
1060 | ||||
1061 | // Because we can visit these intrinsics twice, also check to see if the | |||
1062 | // first time marked this instruction as dead. If so, skip it. | |||
1063 | if (VisitedDeadInsts.count(&II)) | |||
1064 | return; | |||
1065 | ||||
1066 | if (!IsOffsetKnown) | |||
1067 | return PI.setAborted(&II); | |||
1068 | ||||
1069 | // This side of the transfer is completely out-of-bounds, and so we can | |||
1070 | // nuke the entire transfer. However, we also need to nuke the other side | |||
1071 | // if already added to our partitions. | |||
1072 | // FIXME: Yet another place we really should bypass this when | |||
1073 | // instrumenting for ASan. | |||
1074 | if (Offset.uge(AllocSize)) { | |||
1075 | SmallDenseMap<Instruction *, unsigned>::iterator MTPI = | |||
1076 | MemTransferSliceMap.find(&II); | |||
1077 | if (MTPI != MemTransferSliceMap.end()) | |||
1078 | AS.Slices[MTPI->second].kill(); | |||
1079 | return markAsDead(II); | |||
1080 | } | |||
1081 | ||||
1082 | uint64_t RawOffset = Offset.getLimitedValue(); | |||
1083 | uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset; | |||
1084 | ||||
1085 | // Check for the special case where the same exact value is used for both | |||
1086 | // source and dest. | |||
1087 | if (*U == II.getRawDest() && *U == II.getRawSource()) { | |||
1088 | // For non-volatile transfers this is a no-op. | |||
1089 | if (!II.isVolatile()) | |||
1090 | return markAsDead(II); | |||
1091 | ||||
1092 | return insertUse(II, Offset, Size, /*IsSplittable=*/false); | |||
1093 | } | |||
1094 | ||||
1095 | // If we have seen both source and destination for a mem transfer, then | |||
1096 | // they both point to the same alloca. | |||
1097 | bool Inserted; | |||
1098 | SmallDenseMap<Instruction *, unsigned>::iterator MTPI; | |||
1099 | std::tie(MTPI, Inserted) = | |||
1100 | MemTransferSliceMap.insert(std::make_pair(&II, AS.Slices.size())); | |||
1101 | unsigned PrevIdx = MTPI->second; | |||
1102 | if (!Inserted) { | |||
1103 | Slice &PrevP = AS.Slices[PrevIdx]; | |||
1104 | ||||
1105 | // Check if the begin offsets match and this is a non-volatile transfer. | |||
1106 | // In that case, we can completely elide the transfer. | |||
1107 | if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) { | |||
1108 | PrevP.kill(); | |||
1109 | return markAsDead(II); | |||
1110 | } | |||
1111 | ||||
1112 | // Otherwise we have an offset transfer within the same alloca. We can't | |||
1113 | // split those. | |||
1114 | PrevP.makeUnsplittable(); | |||
1115 | } | |||
1116 | ||||
1117 | // Insert the use now that we've fixed up the splittable nature. | |||
1118 | insertUse(II, Offset, Size, /*IsSplittable=*/Inserted && Length); | |||
1119 | ||||
1120 | // Check that we ended up with a valid index in the map. | |||
1121 | assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&(static_cast <bool> (AS.Slices[PrevIdx].getUse()->getUser () == &II && "Map index doesn't point back to a slice with this user." ) ? void (0) : __assert_fail ("AS.Slices[PrevIdx].getUse()->getUser() == &II && \"Map index doesn't point back to a slice with this user.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1122, __extension__ __PRETTY_FUNCTION__ )) | |||
1122 | "Map index doesn't point back to a slice with this user.")(static_cast <bool> (AS.Slices[PrevIdx].getUse()->getUser () == &II && "Map index doesn't point back to a slice with this user." ) ? void (0) : __assert_fail ("AS.Slices[PrevIdx].getUse()->getUser() == &II && \"Map index doesn't point back to a slice with this user.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1122, __extension__ __PRETTY_FUNCTION__ )); | |||
1123 | } | |||
1124 | ||||
1125 | // Disable SRoA for any intrinsics except for lifetime invariants and | |||
1126 | // invariant group. | |||
1127 | // FIXME: What about debug intrinsics? This matches old behavior, but | |||
1128 | // doesn't make sense. | |||
1129 | void visitIntrinsicInst(IntrinsicInst &II) { | |||
1130 | if (II.isDroppable()) { | |||
1131 | AS.DeadUseIfPromotable.push_back(U); | |||
1132 | return; | |||
1133 | } | |||
1134 | ||||
1135 | if (!IsOffsetKnown) | |||
1136 | return PI.setAborted(&II); | |||
1137 | ||||
1138 | if (II.isLifetimeStartOrEnd()) { | |||
1139 | ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0)); | |||
1140 | uint64_t Size = std::min(AllocSize - Offset.getLimitedValue(), | |||
1141 | Length->getLimitedValue()); | |||
1142 | insertUse(II, Offset, Size, true); | |||
1143 | return; | |||
1144 | } | |||
1145 | ||||
1146 | if (II.isLaunderOrStripInvariantGroup()) { | |||
1147 | enqueueUsers(II); | |||
1148 | return; | |||
1149 | } | |||
1150 | ||||
1151 | Base::visitIntrinsicInst(II); | |||
1152 | } | |||
1153 | ||||
1154 | Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) { | |||
1155 | // We consider any PHI or select that results in a direct load or store of | |||
1156 | // the same offset to be a viable use for slicing purposes. These uses | |||
1157 | // are considered unsplittable and the size is the maximum loaded or stored | |||
1158 | // size. | |||
1159 | SmallPtrSet<Instruction *, 4> Visited; | |||
1160 | SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses; | |||
1161 | Visited.insert(Root); | |||
1162 | Uses.push_back(std::make_pair(cast<Instruction>(*U), Root)); | |||
1163 | const DataLayout &DL = Root->getModule()->getDataLayout(); | |||
1164 | // If there are no loads or stores, the access is dead. We mark that as | |||
1165 | // a size zero access. | |||
1166 | Size = 0; | |||
1167 | do { | |||
1168 | Instruction *I, *UsedI; | |||
1169 | std::tie(UsedI, I) = Uses.pop_back_val(); | |||
1170 | ||||
1171 | if (LoadInst *LI = dyn_cast<LoadInst>(I)) { | |||
1172 | Size = | |||
1173 | std::max(Size, DL.getTypeStoreSize(LI->getType()).getFixedValue()); | |||
1174 | continue; | |||
1175 | } | |||
1176 | if (StoreInst *SI = dyn_cast<StoreInst>(I)) { | |||
1177 | Value *Op = SI->getOperand(0); | |||
1178 | if (Op == UsedI) | |||
1179 | return SI; | |||
1180 | Size = | |||
1181 | std::max(Size, DL.getTypeStoreSize(Op->getType()).getFixedValue()); | |||
1182 | continue; | |||
1183 | } | |||
1184 | ||||
1185 | if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) { | |||
1186 | if (!GEP->hasAllZeroIndices()) | |||
1187 | return GEP; | |||
1188 | } else if (!isa<BitCastInst>(I) && !isa<PHINode>(I) && | |||
1189 | !isa<SelectInst>(I) && !isa<AddrSpaceCastInst>(I)) { | |||
1190 | return I; | |||
1191 | } | |||
1192 | ||||
1193 | for (User *U : I->users()) | |||
1194 | if (Visited.insert(cast<Instruction>(U)).second) | |||
1195 | Uses.push_back(std::make_pair(I, cast<Instruction>(U))); | |||
1196 | } while (!Uses.empty()); | |||
1197 | ||||
1198 | return nullptr; | |||
1199 | } | |||
1200 | ||||
1201 | void visitPHINodeOrSelectInst(Instruction &I) { | |||
1202 | assert(isa<PHINode>(I) || isa<SelectInst>(I))(static_cast <bool> (isa<PHINode>(I) || isa<SelectInst >(I)) ? void (0) : __assert_fail ("isa<PHINode>(I) || isa<SelectInst>(I)" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1202, __extension__ __PRETTY_FUNCTION__ )); | |||
1203 | if (I.use_empty()) | |||
1204 | return markAsDead(I); | |||
1205 | ||||
1206 | // If this is a PHI node before a catchswitch, we cannot insert any non-PHI | |||
1207 | // instructions in this BB, which may be required during rewriting. Bail out | |||
1208 | // on these cases. | |||
1209 | if (isa<PHINode>(I) && | |||
1210 | I.getParent()->getFirstInsertionPt() == I.getParent()->end()) | |||
1211 | return PI.setAborted(&I); | |||
1212 | ||||
1213 | // TODO: We could use simplifyInstruction here to fold PHINodes and | |||
1214 | // SelectInsts. However, doing so requires to change the current | |||
1215 | // dead-operand-tracking mechanism. For instance, suppose neither loading | |||
1216 | // from %U nor %other traps. Then "load (select undef, %U, %other)" does not | |||
1217 | // trap either. However, if we simply replace %U with undef using the | |||
1218 | // current dead-operand-tracking mechanism, "load (select undef, undef, | |||
1219 | // %other)" may trap because the select may return the first operand | |||
1220 | // "undef". | |||
1221 | if (Value *Result = foldPHINodeOrSelectInst(I)) { | |||
1222 | if (Result == *U) | |||
1223 | // If the result of the constant fold will be the pointer, recurse | |||
1224 | // through the PHI/select as if we had RAUW'ed it. | |||
1225 | enqueueUsers(I); | |||
1226 | else | |||
1227 | // Otherwise the operand to the PHI/select is dead, and we can replace | |||
1228 | // it with poison. | |||
1229 | AS.DeadOperands.push_back(U); | |||
1230 | ||||
1231 | return; | |||
1232 | } | |||
1233 | ||||
1234 | if (!IsOffsetKnown) | |||
1235 | return PI.setAborted(&I); | |||
1236 | ||||
1237 | // See if we already have computed info on this node. | |||
1238 | uint64_t &Size = PHIOrSelectSizes[&I]; | |||
1239 | if (!Size) { | |||
1240 | // This is a new PHI/Select, check for an unsafe use of it. | |||
1241 | if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(&I, Size)) | |||
1242 | return PI.setAborted(UnsafeI); | |||
1243 | } | |||
1244 | ||||
1245 | // For PHI and select operands outside the alloca, we can't nuke the entire | |||
1246 | // phi or select -- the other side might still be relevant, so we special | |||
1247 | // case them here and use a separate structure to track the operands | |||
1248 | // themselves which should be replaced with poison. | |||
1249 | // FIXME: This should instead be escaped in the event we're instrumenting | |||
1250 | // for address sanitization. | |||
1251 | if (Offset.uge(AllocSize)) { | |||
1252 | AS.DeadOperands.push_back(U); | |||
1253 | return; | |||
1254 | } | |||
1255 | ||||
1256 | insertUse(I, Offset, Size); | |||
1257 | } | |||
1258 | ||||
1259 | void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(PN); } | |||
1260 | ||||
1261 | void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); } | |||
1262 | ||||
1263 | /// Disable SROA entirely if there are unhandled users of the alloca. | |||
1264 | void visitInstruction(Instruction &I) { PI.setAborted(&I); } | |||
1265 | }; | |||
1266 | ||||
1267 | AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI) | |||
1268 | : | |||
1269 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1270 | AI(AI), | |||
1271 | #endif | |||
1272 | PointerEscapingInstr(nullptr) { | |||
1273 | SliceBuilder PB(DL, AI, *this); | |||
1274 | SliceBuilder::PtrInfo PtrI = PB.visitPtr(AI); | |||
1275 | if (PtrI.isEscaped() || PtrI.isAborted()) { | |||
1276 | // FIXME: We should sink the escape vs. abort info into the caller nicely, | |||
1277 | // possibly by just storing the PtrInfo in the AllocaSlices. | |||
1278 | PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst() | |||
1279 | : PtrI.getAbortingInst(); | |||
1280 | assert(PointerEscapingInstr && "Did not track a bad instruction")(static_cast <bool> (PointerEscapingInstr && "Did not track a bad instruction" ) ? void (0) : __assert_fail ("PointerEscapingInstr && \"Did not track a bad instruction\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1280, __extension__ __PRETTY_FUNCTION__ )); | |||
1281 | return; | |||
1282 | } | |||
1283 | ||||
1284 | llvm::erase_if(Slices, [](const Slice &S) { return S.isDead(); }); | |||
1285 | ||||
1286 | // Sort the uses. This arranges for the offsets to be in ascending order, | |||
1287 | // and the sizes to be in descending order. | |||
1288 | llvm::stable_sort(Slices); | |||
1289 | } | |||
1290 | ||||
1291 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1292 | ||||
1293 | void AllocaSlices::print(raw_ostream &OS, const_iterator I, | |||
1294 | StringRef Indent) const { | |||
1295 | printSlice(OS, I, Indent); | |||
1296 | OS << "\n"; | |||
1297 | printUse(OS, I, Indent); | |||
1298 | } | |||
1299 | ||||
1300 | void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I, | |||
1301 | StringRef Indent) const { | |||
1302 | OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")" | |||
1303 | << " slice #" << (I - begin()) | |||
1304 | << (I->isSplittable() ? " (splittable)" : ""); | |||
1305 | } | |||
1306 | ||||
1307 | void AllocaSlices::printUse(raw_ostream &OS, const_iterator I, | |||
1308 | StringRef Indent) const { | |||
1309 | OS << Indent << " used by: " << *I->getUse()->getUser() << "\n"; | |||
1310 | } | |||
1311 | ||||
1312 | void AllocaSlices::print(raw_ostream &OS) const { | |||
1313 | if (PointerEscapingInstr) { | |||
1314 | OS << "Can't analyze slices for alloca: " << AI << "\n" | |||
1315 | << " A pointer to this alloca escaped by:\n" | |||
1316 | << " " << *PointerEscapingInstr << "\n"; | |||
1317 | return; | |||
1318 | } | |||
1319 | ||||
1320 | OS << "Slices of alloca: " << AI << "\n"; | |||
1321 | for (const_iterator I = begin(), E = end(); I != E; ++I) | |||
1322 | print(OS, I); | |||
1323 | } | |||
1324 | ||||
1325 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void AllocaSlices::dump(const_iterator I) const { | |||
1326 | print(dbgs(), I); | |||
1327 | } | |||
1328 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void AllocaSlices::dump() const { print(dbgs()); } | |||
1329 | ||||
1330 | #endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1331 | ||||
1332 | /// Walk the range of a partitioning looking for a common type to cover this | |||
1333 | /// sequence of slices. | |||
1334 | static std::pair<Type *, IntegerType *> | |||
1335 | findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E, | |||
1336 | uint64_t EndOffset) { | |||
1337 | Type *Ty = nullptr; | |||
1338 | bool TyIsCommon = true; | |||
1339 | IntegerType *ITy = nullptr; | |||
1340 | ||||
1341 | // Note that we need to look at *every* alloca slice's Use to ensure we | |||
1342 | // always get consistent results regardless of the order of slices. | |||
1343 | for (AllocaSlices::const_iterator I = B; I != E; ++I) { | |||
1344 | Use *U = I->getUse(); | |||
1345 | if (isa<IntrinsicInst>(*U->getUser())) | |||
1346 | continue; | |||
1347 | if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset) | |||
1348 | continue; | |||
1349 | ||||
1350 | Type *UserTy = nullptr; | |||
1351 | if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | |||
1352 | UserTy = LI->getType(); | |||
1353 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | |||
1354 | UserTy = SI->getValueOperand()->getType(); | |||
1355 | } | |||
1356 | ||||
1357 | if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(UserTy)) { | |||
1358 | // If the type is larger than the partition, skip it. We only encounter | |||
1359 | // this for split integer operations where we want to use the type of the | |||
1360 | // entity causing the split. Also skip if the type is not a byte width | |||
1361 | // multiple. | |||
1362 | if (UserITy->getBitWidth() % 8 != 0 || | |||
1363 | UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset())) | |||
1364 | continue; | |||
1365 | ||||
1366 | // Track the largest bitwidth integer type used in this way in case there | |||
1367 | // is no common type. | |||
1368 | if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth()) | |||
1369 | ITy = UserITy; | |||
1370 | } | |||
1371 | ||||
1372 | // To avoid depending on the order of slices, Ty and TyIsCommon must not | |||
1373 | // depend on types skipped above. | |||
1374 | if (!UserTy || (Ty && Ty != UserTy)) | |||
1375 | TyIsCommon = false; // Give up on anything but an iN type. | |||
1376 | else | |||
1377 | Ty = UserTy; | |||
1378 | } | |||
1379 | ||||
1380 | return {TyIsCommon ? Ty : nullptr, ITy}; | |||
1381 | } | |||
1382 | ||||
1383 | /// PHI instructions that use an alloca and are subsequently loaded can be | |||
1384 | /// rewritten to load both input pointers in the pred blocks and then PHI the | |||
1385 | /// results, allowing the load of the alloca to be promoted. | |||
1386 | /// From this: | |||
1387 | /// %P2 = phi [i32* %Alloca, i32* %Other] | |||
1388 | /// %V = load i32* %P2 | |||
1389 | /// to: | |||
1390 | /// %V1 = load i32* %Alloca -> will be mem2reg'd | |||
1391 | /// ... | |||
1392 | /// %V2 = load i32* %Other | |||
1393 | /// ... | |||
1394 | /// %V = phi [i32 %V1, i32 %V2] | |||
1395 | /// | |||
1396 | /// We can do this to a select if its only uses are loads and if the operands | |||
1397 | /// to the select can be loaded unconditionally. | |||
1398 | /// | |||
1399 | /// FIXME: This should be hoisted into a generic utility, likely in | |||
1400 | /// Transforms/Util/Local.h | |||
1401 | static bool isSafePHIToSpeculate(PHINode &PN) { | |||
1402 | const DataLayout &DL = PN.getModule()->getDataLayout(); | |||
1403 | ||||
1404 | // For now, we can only do this promotion if the load is in the same block | |||
1405 | // as the PHI, and if there are no stores between the phi and load. | |||
1406 | // TODO: Allow recursive phi users. | |||
1407 | // TODO: Allow stores. | |||
1408 | BasicBlock *BB = PN.getParent(); | |||
1409 | Align MaxAlign; | |||
1410 | uint64_t APWidth = DL.getIndexTypeSizeInBits(PN.getType()); | |||
1411 | Type *LoadType = nullptr; | |||
1412 | for (User *U : PN.users()) { | |||
1413 | LoadInst *LI = dyn_cast<LoadInst>(U); | |||
1414 | if (!LI || !LI->isSimple()) | |||
1415 | return false; | |||
1416 | ||||
1417 | // For now we only allow loads in the same block as the PHI. This is | |||
1418 | // a common case that happens when instcombine merges two loads through | |||
1419 | // a PHI. | |||
1420 | if (LI->getParent() != BB) | |||
1421 | return false; | |||
1422 | ||||
1423 | if (LoadType) { | |||
1424 | if (LoadType != LI->getType()) | |||
1425 | return false; | |||
1426 | } else { | |||
1427 | LoadType = LI->getType(); | |||
1428 | } | |||
1429 | ||||
1430 | // Ensure that there are no instructions between the PHI and the load that | |||
1431 | // could store. | |||
1432 | for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI) | |||
1433 | if (BBI->mayWriteToMemory()) | |||
1434 | return false; | |||
1435 | ||||
1436 | MaxAlign = std::max(MaxAlign, LI->getAlign()); | |||
1437 | } | |||
1438 | ||||
1439 | if (!LoadType) | |||
1440 | return false; | |||
1441 | ||||
1442 | APInt LoadSize = | |||
1443 | APInt(APWidth, DL.getTypeStoreSize(LoadType).getFixedValue()); | |||
1444 | ||||
1445 | // We can only transform this if it is safe to push the loads into the | |||
1446 | // predecessor blocks. The only thing to watch out for is that we can't put | |||
1447 | // a possibly trapping load in the predecessor if it is a critical edge. | |||
1448 | for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { | |||
1449 | Instruction *TI = PN.getIncomingBlock(Idx)->getTerminator(); | |||
1450 | Value *InVal = PN.getIncomingValue(Idx); | |||
1451 | ||||
1452 | // If the value is produced by the terminator of the predecessor (an | |||
1453 | // invoke) or it has side-effects, there is no valid place to put a load | |||
1454 | // in the predecessor. | |||
1455 | if (TI == InVal || TI->mayHaveSideEffects()) | |||
1456 | return false; | |||
1457 | ||||
1458 | // If the predecessor has a single successor, then the edge isn't | |||
1459 | // critical. | |||
1460 | if (TI->getNumSuccessors() == 1) | |||
1461 | continue; | |||
1462 | ||||
1463 | // If this pointer is always safe to load, or if we can prove that there | |||
1464 | // is already a load in the block, then we can move the load to the pred | |||
1465 | // block. | |||
1466 | if (isSafeToLoadUnconditionally(InVal, MaxAlign, LoadSize, DL, TI)) | |||
1467 | continue; | |||
1468 | ||||
1469 | return false; | |||
1470 | } | |||
1471 | ||||
1472 | return true; | |||
1473 | } | |||
1474 | ||||
1475 | static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) { | |||
1476 | LLVM_DEBUG(dbgs() << " original: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << PN << "\n"; } } while (false); | |||
1477 | ||||
1478 | LoadInst *SomeLoad = cast<LoadInst>(PN.user_back()); | |||
1479 | Type *LoadTy = SomeLoad->getType(); | |||
1480 | IRB.SetInsertPoint(&PN); | |||
1481 | PHINode *NewPN = IRB.CreatePHI(LoadTy, PN.getNumIncomingValues(), | |||
1482 | PN.getName() + ".sroa.speculated"); | |||
1483 | ||||
1484 | // Get the AA tags and alignment to use from one of the loads. It does not | |||
1485 | // matter which one we get and if any differ. | |||
1486 | AAMDNodes AATags = SomeLoad->getAAMetadata(); | |||
1487 | Align Alignment = SomeLoad->getAlign(); | |||
1488 | ||||
1489 | // Rewrite all loads of the PN to use the new PHI. | |||
1490 | while (!PN.use_empty()) { | |||
1491 | LoadInst *LI = cast<LoadInst>(PN.user_back()); | |||
1492 | LI->replaceAllUsesWith(NewPN); | |||
1493 | LI->eraseFromParent(); | |||
1494 | } | |||
1495 | ||||
1496 | // Inject loads into all of the pred blocks. | |||
1497 | DenseMap<BasicBlock*, Value*> InjectedLoads; | |||
1498 | for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { | |||
1499 | BasicBlock *Pred = PN.getIncomingBlock(Idx); | |||
1500 | Value *InVal = PN.getIncomingValue(Idx); | |||
1501 | ||||
1502 | // A PHI node is allowed to have multiple (duplicated) entries for the same | |||
1503 | // basic block, as long as the value is the same. So if we already injected | |||
1504 | // a load in the predecessor, then we should reuse the same load for all | |||
1505 | // duplicated entries. | |||
1506 | if (Value* V = InjectedLoads.lookup(Pred)) { | |||
1507 | NewPN->addIncoming(V, Pred); | |||
1508 | continue; | |||
1509 | } | |||
1510 | ||||
1511 | Instruction *TI = Pred->getTerminator(); | |||
1512 | IRB.SetInsertPoint(TI); | |||
1513 | ||||
1514 | LoadInst *Load = IRB.CreateAlignedLoad( | |||
1515 | LoadTy, InVal, Alignment, | |||
1516 | (PN.getName() + ".sroa.speculate.load." + Pred->getName())); | |||
1517 | ++NumLoadsSpeculated; | |||
1518 | if (AATags) | |||
1519 | Load->setAAMetadata(AATags); | |||
1520 | NewPN->addIncoming(Load, Pred); | |||
1521 | InjectedLoads[Pred] = Load; | |||
1522 | } | |||
1523 | ||||
1524 | LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " speculated to: " << *NewPN << "\n"; } } while (false); | |||
1525 | PN.eraseFromParent(); | |||
1526 | } | |||
1527 | ||||
1528 | sroa::SelectHandSpeculativity & | |||
1529 | sroa::SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) { | |||
1530 | if (isTrueVal) | |||
1531 | Bitfield::set<sroa::SelectHandSpeculativity::TrueVal>(Storage, true); | |||
1532 | else | |||
1533 | Bitfield::set<sroa::SelectHandSpeculativity::FalseVal>(Storage, true); | |||
1534 | return *this; | |||
1535 | } | |||
1536 | ||||
1537 | bool sroa::SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const { | |||
1538 | return isTrueVal | |||
1539 | ? Bitfield::get<sroa::SelectHandSpeculativity::TrueVal>(Storage) | |||
1540 | : Bitfield::get<sroa::SelectHandSpeculativity::FalseVal>(Storage); | |||
1541 | } | |||
1542 | ||||
1543 | bool sroa::SelectHandSpeculativity::areAllSpeculatable() const { | |||
1544 | return isSpeculatable(/*isTrueVal=*/true) && | |||
1545 | isSpeculatable(/*isTrueVal=*/false); | |||
1546 | } | |||
1547 | ||||
1548 | bool sroa::SelectHandSpeculativity::areAnySpeculatable() const { | |||
1549 | return isSpeculatable(/*isTrueVal=*/true) || | |||
1550 | isSpeculatable(/*isTrueVal=*/false); | |||
1551 | } | |||
1552 | bool sroa::SelectHandSpeculativity::areNoneSpeculatable() const { | |||
1553 | return !areAnySpeculatable(); | |||
1554 | } | |||
1555 | ||||
1556 | static sroa::SelectHandSpeculativity | |||
1557 | isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG) { | |||
1558 | assert(LI.isSimple() && "Only for simple loads")(static_cast <bool> (LI.isSimple() && "Only for simple loads" ) ? void (0) : __assert_fail ("LI.isSimple() && \"Only for simple loads\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1558, __extension__ __PRETTY_FUNCTION__ )); | |||
1559 | sroa::SelectHandSpeculativity Spec; | |||
1560 | ||||
1561 | const DataLayout &DL = SI.getModule()->getDataLayout(); | |||
1562 | for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()}) | |||
1563 | if (isSafeToLoadUnconditionally(Value, LI.getType(), LI.getAlign(), DL, | |||
1564 | &LI)) | |||
1565 | Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue()); | |||
1566 | else if (PreserveCFG) | |||
1567 | return Spec; | |||
1568 | ||||
1569 | return Spec; | |||
1570 | } | |||
1571 | ||||
1572 | std::optional<sroa::RewriteableMemOps> | |||
1573 | SROAPass::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) { | |||
1574 | RewriteableMemOps Ops; | |||
1575 | ||||
1576 | for (User *U : SI.users()) { | |||
1577 | if (auto *BC = dyn_cast<BitCastInst>(U); BC && BC->hasOneUse()) | |||
1578 | U = *BC->user_begin(); | |||
1579 | ||||
1580 | if (auto *Store = dyn_cast<StoreInst>(U)) { | |||
1581 | // Note that atomic stores can be transformed; atomic semantics do not | |||
1582 | // have any meaning for a local alloca. Stores are not speculatable, | |||
1583 | // however, so if we can't turn it into a predicated store, we are done. | |||
1584 | if (Store->isVolatile() || PreserveCFG) | |||
1585 | return {}; // Give up on this `select`. | |||
1586 | Ops.emplace_back(Store); | |||
1587 | continue; | |||
1588 | } | |||
1589 | ||||
1590 | auto *LI = dyn_cast<LoadInst>(U); | |||
1591 | ||||
1592 | // Note that atomic loads can be transformed; | |||
1593 | // atomic semantics do not have any meaning for a local alloca. | |||
1594 | if (!LI || LI->isVolatile()) | |||
1595 | return {}; // Give up on this `select`. | |||
1596 | ||||
1597 | PossiblySpeculatableLoad Load(LI); | |||
1598 | if (!LI->isSimple()) { | |||
1599 | // If the `load` is not simple, we can't speculatively execute it, | |||
1600 | // but we could handle this via a CFG modification. But can we? | |||
1601 | if (PreserveCFG) | |||
1602 | return {}; // Give up on this `select`. | |||
1603 | Ops.emplace_back(Load); | |||
1604 | continue; | |||
1605 | } | |||
1606 | ||||
1607 | sroa::SelectHandSpeculativity Spec = | |||
1608 | isSafeLoadOfSelectToSpeculate(*LI, SI, PreserveCFG); | |||
1609 | if (PreserveCFG && !Spec.areAllSpeculatable()) | |||
1610 | return {}; // Give up on this `select`. | |||
1611 | ||||
1612 | Load.setInt(Spec); | |||
1613 | Ops.emplace_back(Load); | |||
1614 | } | |||
1615 | ||||
1616 | return Ops; | |||
1617 | } | |||
1618 | ||||
1619 | static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI, | |||
1620 | IRBuilderTy &IRB) { | |||
1621 | LLVM_DEBUG(dbgs() << " original load: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original load: " << SI << "\n"; } } while (false); | |||
1622 | ||||
1623 | Value *TV = SI.getTrueValue(); | |||
1624 | Value *FV = SI.getFalseValue(); | |||
1625 | // Replace the given load of the select with a select of two loads. | |||
1626 | ||||
1627 | assert(LI.isSimple() && "We only speculate simple loads")(static_cast <bool> (LI.isSimple() && "We only speculate simple loads" ) ? void (0) : __assert_fail ("LI.isSimple() && \"We only speculate simple loads\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1627, __extension__ __PRETTY_FUNCTION__ )); | |||
1628 | ||||
1629 | IRB.SetInsertPoint(&LI); | |||
1630 | ||||
1631 | if (auto *TypedPtrTy = LI.getPointerOperandType(); | |||
1632 | !TypedPtrTy->isOpaquePointerTy() && SI.getType() != TypedPtrTy) { | |||
1633 | TV = IRB.CreateBitOrPointerCast(TV, TypedPtrTy, ""); | |||
1634 | FV = IRB.CreateBitOrPointerCast(FV, TypedPtrTy, ""); | |||
1635 | } | |||
1636 | ||||
1637 | LoadInst *TL = | |||
1638 | IRB.CreateAlignedLoad(LI.getType(), TV, LI.getAlign(), | |||
1639 | LI.getName() + ".sroa.speculate.load.true"); | |||
1640 | LoadInst *FL = | |||
1641 | IRB.CreateAlignedLoad(LI.getType(), FV, LI.getAlign(), | |||
1642 | LI.getName() + ".sroa.speculate.load.false"); | |||
1643 | NumLoadsSpeculated += 2; | |||
1644 | ||||
1645 | // Transfer alignment and AA info if present. | |||
1646 | TL->setAlignment(LI.getAlign()); | |||
1647 | FL->setAlignment(LI.getAlign()); | |||
1648 | ||||
1649 | AAMDNodes Tags = LI.getAAMetadata(); | |||
1650 | if (Tags) { | |||
1651 | TL->setAAMetadata(Tags); | |||
1652 | FL->setAAMetadata(Tags); | |||
1653 | } | |||
1654 | ||||
1655 | Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, | |||
1656 | LI.getName() + ".sroa.speculated"); | |||
1657 | ||||
1658 | LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " speculated to: " << *V << "\n"; } } while (false); | |||
1659 | LI.replaceAllUsesWith(V); | |||
1660 | } | |||
1661 | ||||
1662 | template <typename T> | |||
1663 | static void rewriteMemOpOfSelect(SelectInst &SI, T &I, | |||
1664 | sroa::SelectHandSpeculativity Spec, | |||
1665 | DomTreeUpdater &DTU) { | |||
1666 | assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Only for load and store!")(static_cast <bool> ((isa<LoadInst>(I) || isa< StoreInst>(I)) && "Only for load and store!") ? void (0) : __assert_fail ("(isa<LoadInst>(I) || isa<StoreInst>(I)) && \"Only for load and store!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1666, __extension__ __PRETTY_FUNCTION__ )); | |||
1667 | LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original mem op: " << I << "\n"; } } while (false); | |||
1668 | BasicBlock *Head = I.getParent(); | |||
1669 | Instruction *ThenTerm = nullptr; | |||
1670 | Instruction *ElseTerm = nullptr; | |||
1671 | if (Spec.areNoneSpeculatable()) | |||
1672 | SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm, | |||
1673 | SI.getMetadata(LLVMContext::MD_prof), &DTU); | |||
1674 | else { | |||
1675 | SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false, | |||
1676 | SI.getMetadata(LLVMContext::MD_prof), &DTU, | |||
1677 | /*LI=*/nullptr, /*ThenBlock=*/nullptr); | |||
1678 | if (Spec.isSpeculatable(/*isTrueVal=*/true)) | |||
1679 | cast<BranchInst>(Head->getTerminator())->swapSuccessors(); | |||
1680 | } | |||
1681 | auto *HeadBI = cast<BranchInst>(Head->getTerminator()); | |||
1682 | Spec = {}; // Do not use `Spec` beyond this point. | |||
1683 | BasicBlock *Tail = I.getParent(); | |||
1684 | Tail->setName(Head->getName() + ".cont"); | |||
1685 | PHINode *PN; | |||
1686 | if (isa<LoadInst>(I)) | |||
1687 | PN = PHINode::Create(I.getType(), 2, "", &I); | |||
1688 | for (BasicBlock *SuccBB : successors(Head)) { | |||
1689 | bool IsThen = SuccBB == HeadBI->getSuccessor(0); | |||
1690 | int SuccIdx = IsThen ? 0 : 1; | |||
1691 | auto *NewMemOpBB = SuccBB == Tail ? Head : SuccBB; | |||
1692 | auto &CondMemOp = cast<T>(*I.clone()); | |||
1693 | if (NewMemOpBB != Head) { | |||
1694 | NewMemOpBB->setName(Head->getName() + (IsThen ? ".then" : ".else")); | |||
1695 | if (isa<LoadInst>(I)) | |||
1696 | ++NumLoadsPredicated; | |||
1697 | else | |||
1698 | ++NumStoresPredicated; | |||
1699 | } else { | |||
1700 | CondMemOp.dropUBImplyingAttrsAndMetadata(); | |||
1701 | ++NumLoadsSpeculated; | |||
1702 | } | |||
1703 | CondMemOp.insertBefore(NewMemOpBB->getTerminator()); | |||
1704 | Value *Ptr = SI.getOperand(1 + SuccIdx); | |||
1705 | if (auto *PtrTy = Ptr->getType(); | |||
1706 | !PtrTy->isOpaquePointerTy() && | |||
1707 | PtrTy != CondMemOp.getPointerOperandType()) | |||
1708 | Ptr = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( | |||
1709 | Ptr, CondMemOp.getPointerOperandType(), "", &CondMemOp); | |||
1710 | CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr); | |||
1711 | if (isa<LoadInst>(I)) { | |||
1712 | CondMemOp.setName(I.getName() + (IsThen ? ".then" : ".else") + ".val"); | |||
1713 | PN->addIncoming(&CondMemOp, NewMemOpBB); | |||
1714 | } else | |||
1715 | LLVM_DEBUG(dbgs() << " to: " << CondMemOp << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << CondMemOp << "\n"; } } while (false); | |||
1716 | } | |||
1717 | if (isa<LoadInst>(I)) { | |||
1718 | PN->takeName(&I); | |||
1719 | LLVM_DEBUG(dbgs() << " to: " << *PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *PN << "\n"; } } while (false); | |||
1720 | I.replaceAllUsesWith(PN); | |||
1721 | } | |||
1722 | } | |||
1723 | ||||
1724 | static void rewriteMemOpOfSelect(SelectInst &SelInst, Instruction &I, | |||
1725 | sroa::SelectHandSpeculativity Spec, | |||
1726 | DomTreeUpdater &DTU) { | |||
1727 | if (auto *LI = dyn_cast<LoadInst>(&I)) | |||
1728 | rewriteMemOpOfSelect(SelInst, *LI, Spec, DTU); | |||
1729 | else if (auto *SI = dyn_cast<StoreInst>(&I)) | |||
1730 | rewriteMemOpOfSelect(SelInst, *SI, Spec, DTU); | |||
1731 | else | |||
1732 | llvm_unreachable_internal("Only for load and store."); | |||
1733 | } | |||
1734 | ||||
1735 | static bool rewriteSelectInstMemOps(SelectInst &SI, | |||
1736 | const sroa::RewriteableMemOps &Ops, | |||
1737 | IRBuilderTy &IRB, DomTreeUpdater *DTU) { | |||
1738 | bool CFGChanged = false; | |||
1739 | LLVM_DEBUG(dbgs() << " original select: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original select: " << SI << "\n"; } } while (false); | |||
1740 | ||||
1741 | for (const RewriteableMemOp &Op : Ops) { | |||
1742 | sroa::SelectHandSpeculativity Spec; | |||
1743 | Instruction *I; | |||
1744 | if (auto *const *US = std::get_if<UnspeculatableStore>(&Op)) { | |||
1745 | I = *US; | |||
1746 | } else { | |||
1747 | auto PSL = std::get<PossiblySpeculatableLoad>(Op); | |||
1748 | I = PSL.getPointer(); | |||
1749 | Spec = PSL.getInt(); | |||
1750 | } | |||
1751 | if (Spec.areAllSpeculatable()) { | |||
1752 | speculateSelectInstLoads(SI, cast<LoadInst>(*I), IRB); | |||
1753 | } else { | |||
1754 | assert(DTU && "Should not get here when not allowed to modify the CFG!")(static_cast <bool> (DTU && "Should not get here when not allowed to modify the CFG!" ) ? void (0) : __assert_fail ("DTU && \"Should not get here when not allowed to modify the CFG!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1754, __extension__ __PRETTY_FUNCTION__ )); | |||
1755 | rewriteMemOpOfSelect(SI, *I, Spec, *DTU); | |||
1756 | CFGChanged = true; | |||
1757 | } | |||
1758 | I->eraseFromParent(); | |||
1759 | } | |||
1760 | ||||
1761 | for (User *U : make_early_inc_range(SI.users())) | |||
1762 | cast<BitCastInst>(U)->eraseFromParent(); | |||
1763 | SI.eraseFromParent(); | |||
1764 | return CFGChanged; | |||
1765 | } | |||
1766 | ||||
1767 | /// Compute an adjusted pointer from Ptr by Offset bytes where the | |||
1768 | /// resulting pointer has PointerTy. | |||
1769 | static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr, | |||
1770 | APInt Offset, Type *PointerTy, | |||
1771 | const Twine &NamePrefix) { | |||
1772 | assert(Ptr->getType()->isOpaquePointerTy() &&(static_cast <bool> (Ptr->getType()->isOpaquePointerTy () && "Only opaque pointers supported") ? void (0) : __assert_fail ("Ptr->getType()->isOpaquePointerTy() && \"Only opaque pointers supported\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1773, __extension__ __PRETTY_FUNCTION__ )) | |||
1773 | "Only opaque pointers supported")(static_cast <bool> (Ptr->getType()->isOpaquePointerTy () && "Only opaque pointers supported") ? void (0) : __assert_fail ("Ptr->getType()->isOpaquePointerTy() && \"Only opaque pointers supported\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1773, __extension__ __PRETTY_FUNCTION__ )); | |||
1774 | if (Offset != 0) | |||
1775 | Ptr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), Ptr, IRB.getInt(Offset), | |||
1776 | NamePrefix + "sroa_idx"); | |||
1777 | return IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, PointerTy, | |||
1778 | NamePrefix + "sroa_cast"); | |||
1779 | } | |||
1780 | ||||
1781 | /// Compute the adjusted alignment for a load or store from an offset. | |||
1782 | static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) { | |||
1783 | return commonAlignment(getLoadStoreAlignment(I), Offset); | |||
1784 | } | |||
1785 | ||||
1786 | /// Test whether we can convert a value from the old to the new type. | |||
1787 | /// | |||
1788 | /// This predicate should be used to guard calls to convertValue in order to | |||
1789 | /// ensure that we only try to convert viable values. The strategy is that we | |||
1790 | /// will peel off single element struct and array wrappings to get to an | |||
1791 | /// underlying value, and convert that value. | |||
1792 | static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) { | |||
1793 | if (OldTy == NewTy) | |||
1794 | return true; | |||
1795 | ||||
1796 | // For integer types, we can't handle any bit-width differences. This would | |||
1797 | // break both vector conversions with extension and introduce endianness | |||
1798 | // issues when in conjunction with loads and stores. | |||
1799 | if (isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) { | |||
1800 | assert(cast<IntegerType>(OldTy)->getBitWidth() !=(static_cast <bool> (cast<IntegerType>(OldTy)-> getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth () && "We can't have the same bitwidth for different int types" ) ? void (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1802, __extension__ __PRETTY_FUNCTION__ )) | |||
1801 | cast<IntegerType>(NewTy)->getBitWidth() &&(static_cast <bool> (cast<IntegerType>(OldTy)-> getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth () && "We can't have the same bitwidth for different int types" ) ? void (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1802, __extension__ __PRETTY_FUNCTION__ )) | |||
1802 | "We can't have the same bitwidth for different int types")(static_cast <bool> (cast<IntegerType>(OldTy)-> getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth () && "We can't have the same bitwidth for different int types" ) ? void (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1802, __extension__ __PRETTY_FUNCTION__ )); | |||
1803 | return false; | |||
1804 | } | |||
1805 | ||||
1806 | if (DL.getTypeSizeInBits(NewTy).getFixedValue() != | |||
1807 | DL.getTypeSizeInBits(OldTy).getFixedValue()) | |||
1808 | return false; | |||
1809 | if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType()) | |||
1810 | return false; | |||
1811 | ||||
1812 | // We can convert pointers to integers and vice-versa. Same for vectors | |||
1813 | // of pointers and integers. | |||
1814 | OldTy = OldTy->getScalarType(); | |||
1815 | NewTy = NewTy->getScalarType(); | |||
1816 | if (NewTy->isPointerTy() || OldTy->isPointerTy()) { | |||
1817 | if (NewTy->isPointerTy() && OldTy->isPointerTy()) { | |||
1818 | unsigned OldAS = OldTy->getPointerAddressSpace(); | |||
1819 | unsigned NewAS = NewTy->getPointerAddressSpace(); | |||
1820 | // Convert pointers if they are pointers from the same address space or | |||
1821 | // different integral (not non-integral) address spaces with the same | |||
1822 | // pointer size. | |||
1823 | return OldAS == NewAS || | |||
1824 | (!DL.isNonIntegralAddressSpace(OldAS) && | |||
1825 | !DL.isNonIntegralAddressSpace(NewAS) && | |||
1826 | DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS)); | |||
1827 | } | |||
1828 | ||||
1829 | // We can convert integers to integral pointers, but not to non-integral | |||
1830 | // pointers. | |||
1831 | if (OldTy->isIntegerTy()) | |||
1832 | return !DL.isNonIntegralPointerType(NewTy); | |||
1833 | ||||
1834 | // We can convert integral pointers to integers, but non-integral pointers | |||
1835 | // need to remain pointers. | |||
1836 | if (!DL.isNonIntegralPointerType(OldTy)) | |||
1837 | return NewTy->isIntegerTy(); | |||
1838 | ||||
1839 | return false; | |||
1840 | } | |||
1841 | ||||
1842 | if (OldTy->isTargetExtTy() || NewTy->isTargetExtTy()) | |||
1843 | return false; | |||
1844 | ||||
1845 | return true; | |||
1846 | } | |||
1847 | ||||
1848 | /// Generic routine to convert an SSA value to a value of a different | |||
1849 | /// type. | |||
1850 | /// | |||
1851 | /// This will try various different casting techniques, such as bitcasts, | |||
1852 | /// inttoptr, and ptrtoint casts. Use the \c canConvertValue predicate to test | |||
1853 | /// two types for viability with this routine. | |||
1854 | static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V, | |||
1855 | Type *NewTy) { | |||
1856 | Type *OldTy = V->getType(); | |||
1857 | assert(canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type")(static_cast <bool> (canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type") ? void (0) : __assert_fail ( "canConvertValue(DL, OldTy, NewTy) && \"Value not convertable to type\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1857, __extension__ __PRETTY_FUNCTION__ )); | |||
1858 | ||||
1859 | if (OldTy == NewTy) | |||
1860 | return V; | |||
1861 | ||||
1862 | assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&(static_cast <bool> (!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && "Integer types must be the exact same to convert." ) ? void (0) : __assert_fail ("!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && \"Integer types must be the exact same to convert.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1863, __extension__ __PRETTY_FUNCTION__ )) | |||
1863 | "Integer types must be the exact same to convert.")(static_cast <bool> (!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && "Integer types must be the exact same to convert." ) ? void (0) : __assert_fail ("!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && \"Integer types must be the exact same to convert.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1863, __extension__ __PRETTY_FUNCTION__ )); | |||
1864 | ||||
1865 | // See if we need inttoptr for this type pair. May require additional bitcast. | |||
1866 | if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) { | |||
1867 | // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8* | |||
1868 | // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*> | |||
1869 | // Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*> | |||
1870 | // Directly handle i64 to i8* | |||
1871 | return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)), | |||
1872 | NewTy); | |||
1873 | } | |||
1874 | ||||
1875 | // See if we need ptrtoint for this type pair. May require additional bitcast. | |||
1876 | if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) { | |||
1877 | // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128 | |||
1878 | // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32> | |||
1879 | // Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32> | |||
1880 | // Expand i8* to i64 --> i8* to i64 to i64 | |||
1881 | return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)), | |||
1882 | NewTy); | |||
1883 | } | |||
1884 | ||||
1885 | if (OldTy->isPtrOrPtrVectorTy() && NewTy->isPtrOrPtrVectorTy()) { | |||
1886 | unsigned OldAS = OldTy->getPointerAddressSpace(); | |||
1887 | unsigned NewAS = NewTy->getPointerAddressSpace(); | |||
1888 | // To convert pointers with different address spaces (they are already | |||
1889 | // checked convertible, i.e. they have the same pointer size), so far we | |||
1890 | // cannot use `bitcast` (which has restrict on the same address space) or | |||
1891 | // `addrspacecast` (which is not always no-op casting). Instead, use a pair | |||
1892 | // of no-op `ptrtoint`/`inttoptr` casts through an integer with the same bit | |||
1893 | // size. | |||
1894 | if (OldAS != NewAS) { | |||
1895 | assert(DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS))(static_cast <bool> (DL.getPointerSize(OldAS) == DL.getPointerSize (NewAS)) ? void (0) : __assert_fail ("DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS)" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1895, __extension__ __PRETTY_FUNCTION__ )); | |||
1896 | return IRB.CreateIntToPtr(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)), | |||
1897 | NewTy); | |||
1898 | } | |||
1899 | } | |||
1900 | ||||
1901 | return IRB.CreateBitCast(V, NewTy); | |||
1902 | } | |||
1903 | ||||
1904 | /// Test whether the given slice use can be promoted to a vector. | |||
1905 | /// | |||
1906 | /// This function is called to test each entry in a partition which is slated | |||
1907 | /// for a single slice. | |||
1908 | static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S, | |||
1909 | VectorType *Ty, | |||
1910 | uint64_t ElementSize, | |||
1911 | const DataLayout &DL) { | |||
1912 | // First validate the slice offsets. | |||
1913 | uint64_t BeginOffset = | |||
1914 | std::max(S.beginOffset(), P.beginOffset()) - P.beginOffset(); | |||
1915 | uint64_t BeginIndex = BeginOffset / ElementSize; | |||
1916 | if (BeginIndex * ElementSize != BeginOffset || | |||
1917 | BeginIndex >= cast<FixedVectorType>(Ty)->getNumElements()) | |||
1918 | return false; | |||
1919 | uint64_t EndOffset = | |||
1920 | std::min(S.endOffset(), P.endOffset()) - P.beginOffset(); | |||
1921 | uint64_t EndIndex = EndOffset / ElementSize; | |||
1922 | if (EndIndex * ElementSize != EndOffset || | |||
1923 | EndIndex > cast<FixedVectorType>(Ty)->getNumElements()) | |||
1924 | return false; | |||
1925 | ||||
1926 | assert(EndIndex > BeginIndex && "Empty vector!")(static_cast <bool> (EndIndex > BeginIndex && "Empty vector!") ? void (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1926, __extension__ __PRETTY_FUNCTION__ )); | |||
1927 | uint64_t NumElements = EndIndex - BeginIndex; | |||
1928 | Type *SliceTy = (NumElements == 1) | |||
1929 | ? Ty->getElementType() | |||
1930 | : FixedVectorType::get(Ty->getElementType(), NumElements); | |||
1931 | ||||
1932 | Type *SplitIntTy = | |||
1933 | Type::getIntNTy(Ty->getContext(), NumElements * ElementSize * 8); | |||
1934 | ||||
1935 | Use *U = S.getUse(); | |||
1936 | ||||
1937 | if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { | |||
1938 | if (MI->isVolatile()) | |||
1939 | return false; | |||
1940 | if (!S.isSplittable()) | |||
1941 | return false; // Skip any unsplittable intrinsics. | |||
1942 | } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) { | |||
1943 | if (!II->isLifetimeStartOrEnd() && !II->isDroppable()) | |||
1944 | return false; | |||
1945 | } else if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | |||
1946 | if (LI->isVolatile()) | |||
1947 | return false; | |||
1948 | Type *LTy = LI->getType(); | |||
1949 | // Disable vector promotion when there are loads or stores of an FCA. | |||
1950 | if (LTy->isStructTy()) | |||
1951 | return false; | |||
1952 | if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) { | |||
1953 | assert(LTy->isIntegerTy())(static_cast <bool> (LTy->isIntegerTy()) ? void (0) : __assert_fail ("LTy->isIntegerTy()", "llvm/lib/Transforms/Scalar/SROA.cpp" , 1953, __extension__ __PRETTY_FUNCTION__)); | |||
1954 | LTy = SplitIntTy; | |||
1955 | } | |||
1956 | if (!canConvertValue(DL, SliceTy, LTy)) | |||
1957 | return false; | |||
1958 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | |||
1959 | if (SI->isVolatile()) | |||
1960 | return false; | |||
1961 | Type *STy = SI->getValueOperand()->getType(); | |||
1962 | // Disable vector promotion when there are loads or stores of an FCA. | |||
1963 | if (STy->isStructTy()) | |||
1964 | return false; | |||
1965 | if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) { | |||
1966 | assert(STy->isIntegerTy())(static_cast <bool> (STy->isIntegerTy()) ? void (0) : __assert_fail ("STy->isIntegerTy()", "llvm/lib/Transforms/Scalar/SROA.cpp" , 1966, __extension__ __PRETTY_FUNCTION__)); | |||
1967 | STy = SplitIntTy; | |||
1968 | } | |||
1969 | if (!canConvertValue(DL, STy, SliceTy)) | |||
1970 | return false; | |||
1971 | } else { | |||
1972 | return false; | |||
1973 | } | |||
1974 | ||||
1975 | return true; | |||
1976 | } | |||
1977 | ||||
1978 | /// Test whether a vector type is viable for promotion. | |||
1979 | /// | |||
1980 | /// This implements the necessary checking for \c isVectorPromotionViable over | |||
1981 | /// all slices of the alloca for the given VectorType. | |||
1982 | static bool checkVectorTypeForPromotion(Partition &P, VectorType *VTy, | |||
1983 | const DataLayout &DL) { | |||
1984 | uint64_t ElementSize = | |||
1985 | DL.getTypeSizeInBits(VTy->getElementType()).getFixedValue(); | |||
1986 | ||||
1987 | // While the definition of LLVM vectors is bitpacked, we don't support sizes | |||
1988 | // that aren't byte sized. | |||
1989 | if (ElementSize % 8) | |||
1990 | return false; | |||
1991 | assert((DL.getTypeSizeInBits(VTy).getFixedValue() % 8) == 0 &&(static_cast <bool> ((DL.getTypeSizeInBits(VTy).getFixedValue () % 8) == 0 && "vector size not a multiple of element size?" ) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(VTy).getFixedValue() % 8) == 0 && \"vector size not a multiple of element size?\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1992, __extension__ __PRETTY_FUNCTION__ )) | |||
1992 | "vector size not a multiple of element size?")(static_cast <bool> ((DL.getTypeSizeInBits(VTy).getFixedValue () % 8) == 0 && "vector size not a multiple of element size?" ) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(VTy).getFixedValue() % 8) == 0 && \"vector size not a multiple of element size?\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 1992, __extension__ __PRETTY_FUNCTION__ )); | |||
1993 | ElementSize /= 8; | |||
1994 | ||||
1995 | for (const Slice &S : P) | |||
1996 | if (!isVectorPromotionViableForSlice(P, S, VTy, ElementSize, DL)) | |||
1997 | return false; | |||
1998 | ||||
1999 | for (const Slice *S : P.splitSliceTails()) | |||
2000 | if (!isVectorPromotionViableForSlice(P, *S, VTy, ElementSize, DL)) | |||
2001 | return false; | |||
2002 | ||||
2003 | return true; | |||
2004 | } | |||
2005 | ||||
2006 | /// Test whether the given alloca partitioning and range of slices can be | |||
2007 | /// promoted to a vector. | |||
2008 | /// | |||
2009 | /// This is a quick test to check whether we can rewrite a particular alloca | |||
2010 | /// partition (and its newly formed alloca) into a vector alloca with only | |||
2011 | /// whole-vector loads and stores such that it could be promoted to a vector | |||
2012 | /// SSA value. We only can ensure this for a limited set of operations, and we | |||
2013 | /// don't want to do the rewrites unless we are confident that the result will | |||
2014 | /// be promotable, so we have an early test here. | |||
2015 | static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) { | |||
2016 | // Collect the candidate types for vector-based promotion. Also track whether | |||
2017 | // we have different element types. | |||
2018 | SmallVector<VectorType *, 4> CandidateTys; | |||
2019 | SetVector<Type *> LoadStoreTys; | |||
2020 | Type *CommonEltTy = nullptr; | |||
2021 | VectorType *CommonVecPtrTy = nullptr; | |||
2022 | bool HaveVecPtrTy = false; | |||
2023 | bool HaveCommonEltTy = true; | |||
2024 | bool HaveCommonVecPtrTy = true; | |||
2025 | auto CheckCandidateType = [&](Type *Ty) { | |||
2026 | if (auto *VTy = dyn_cast<VectorType>(Ty)) { | |||
2027 | // Return if bitcast to vectors is different for total size in bits. | |||
2028 | if (!CandidateTys.empty()) { | |||
2029 | VectorType *V = CandidateTys[0]; | |||
2030 | if (DL.getTypeSizeInBits(VTy).getFixedValue() != | |||
2031 | DL.getTypeSizeInBits(V).getFixedValue()) { | |||
2032 | CandidateTys.clear(); | |||
2033 | return; | |||
2034 | } | |||
2035 | } | |||
2036 | CandidateTys.push_back(VTy); | |||
2037 | Type *EltTy = VTy->getElementType(); | |||
2038 | ||||
2039 | if (!CommonEltTy) | |||
2040 | CommonEltTy = EltTy; | |||
2041 | else if (CommonEltTy != EltTy) | |||
2042 | HaveCommonEltTy = false; | |||
2043 | ||||
2044 | if (EltTy->isPointerTy()) { | |||
2045 | HaveVecPtrTy = true; | |||
2046 | if (!CommonVecPtrTy) | |||
2047 | CommonVecPtrTy = VTy; | |||
2048 | else if (CommonVecPtrTy != VTy) | |||
2049 | HaveCommonVecPtrTy = false; | |||
2050 | } | |||
2051 | } | |||
2052 | }; | |||
2053 | // Put load and store types into a set for de-duplication. | |||
2054 | for (const Slice &S : P) { | |||
2055 | Type *Ty; | |||
2056 | if (auto *LI = dyn_cast<LoadInst>(S.getUse()->getUser())) | |||
2057 | Ty = LI->getType(); | |||
2058 | else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser())) | |||
2059 | Ty = SI->getValueOperand()->getType(); | |||
2060 | else | |||
2061 | continue; | |||
2062 | LoadStoreTys.insert(Ty); | |||
2063 | // Consider any loads or stores that are the exact size of the slice. | |||
2064 | if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset()) | |||
2065 | CheckCandidateType(Ty); | |||
2066 | } | |||
2067 | // Consider additional vector types where the element type size is a | |||
2068 | // multiple of load/store element size. | |||
2069 | for (Type *Ty : LoadStoreTys) { | |||
2070 | if (!VectorType::isValidElementType(Ty)) | |||
2071 | continue; | |||
2072 | unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue(); | |||
2073 | // Make a copy of CandidateTys and iterate through it, because we might | |||
2074 | // append to CandidateTys in the loop. | |||
2075 | SmallVector<VectorType *, 4> CandidateTysCopy = CandidateTys; | |||
2076 | for (VectorType *&VTy : CandidateTysCopy) { | |||
2077 | unsigned VectorSize = DL.getTypeSizeInBits(VTy).getFixedValue(); | |||
2078 | unsigned ElementSize = | |||
2079 | DL.getTypeSizeInBits(VTy->getElementType()).getFixedValue(); | |||
2080 | if (TypeSize != VectorSize && TypeSize != ElementSize && | |||
2081 | VectorSize % TypeSize == 0) { | |||
2082 | VectorType *NewVTy = VectorType::get(Ty, VectorSize / TypeSize, false); | |||
2083 | CheckCandidateType(NewVTy); | |||
2084 | } | |||
2085 | } | |||
2086 | } | |||
2087 | ||||
2088 | // If we didn't find a vector type, nothing to do here. | |||
2089 | if (CandidateTys.empty()) | |||
2090 | return nullptr; | |||
2091 | ||||
2092 | // Pointer-ness is sticky, if we had a vector-of-pointers candidate type, | |||
2093 | // then we should choose it, not some other alternative. | |||
2094 | // But, we can't perform a no-op pointer address space change via bitcast, | |||
2095 | // so if we didn't have a common pointer element type, bail. | |||
2096 | if (HaveVecPtrTy && !HaveCommonVecPtrTy) | |||
2097 | return nullptr; | |||
2098 | ||||
2099 | // Try to pick the "best" element type out of the choices. | |||
2100 | if (!HaveCommonEltTy && HaveVecPtrTy) { | |||
2101 | // If there was a pointer element type, there's really only one choice. | |||
2102 | CandidateTys.clear(); | |||
2103 | CandidateTys.push_back(CommonVecPtrTy); | |||
2104 | } else if (!HaveCommonEltTy && !HaveVecPtrTy) { | |||
2105 | // Integer-ify vector types. | |||
2106 | for (VectorType *&VTy : CandidateTys) { | |||
2107 | if (!VTy->getElementType()->isIntegerTy()) | |||
2108 | VTy = cast<VectorType>(VTy->getWithNewType(IntegerType::getIntNTy( | |||
2109 | VTy->getContext(), VTy->getScalarSizeInBits()))); | |||
2110 | } | |||
2111 | ||||
2112 | // Rank the remaining candidate vector types. This is easy because we know | |||
2113 | // they're all integer vectors. We sort by ascending number of elements. | |||
2114 | auto RankVectorTypesComp = [&DL](VectorType *RHSTy, VectorType *LHSTy) { | |||
2115 | (void)DL; | |||
2116 | assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==(static_cast <bool> (DL.getTypeSizeInBits(RHSTy).getFixedValue () == DL.getTypeSizeInBits(LHSTy).getFixedValue() && "Cannot have vector types of different sizes!" ) ? void (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy).getFixedValue() == DL.getTypeSizeInBits(LHSTy).getFixedValue() && \"Cannot have vector types of different sizes!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2118, __extension__ __PRETTY_FUNCTION__ )) | |||
2117 | DL.getTypeSizeInBits(LHSTy).getFixedValue() &&(static_cast <bool> (DL.getTypeSizeInBits(RHSTy).getFixedValue () == DL.getTypeSizeInBits(LHSTy).getFixedValue() && "Cannot have vector types of different sizes!" ) ? void (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy).getFixedValue() == DL.getTypeSizeInBits(LHSTy).getFixedValue() && \"Cannot have vector types of different sizes!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2118, __extension__ __PRETTY_FUNCTION__ )) | |||
2118 | "Cannot have vector types of different sizes!")(static_cast <bool> (DL.getTypeSizeInBits(RHSTy).getFixedValue () == DL.getTypeSizeInBits(LHSTy).getFixedValue() && "Cannot have vector types of different sizes!" ) ? void (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy).getFixedValue() == DL.getTypeSizeInBits(LHSTy).getFixedValue() && \"Cannot have vector types of different sizes!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2118, __extension__ __PRETTY_FUNCTION__ )); | |||
2119 | assert(RHSTy->getElementType()->isIntegerTy() &&(static_cast <bool> (RHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2120, __extension__ __PRETTY_FUNCTION__ )) | |||
2120 | "All non-integer types eliminated!")(static_cast <bool> (RHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2120, __extension__ __PRETTY_FUNCTION__ )); | |||
2121 | assert(LHSTy->getElementType()->isIntegerTy() &&(static_cast <bool> (LHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2122, __extension__ __PRETTY_FUNCTION__ )) | |||
2122 | "All non-integer types eliminated!")(static_cast <bool> (LHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2122, __extension__ __PRETTY_FUNCTION__ )); | |||
2123 | return cast<FixedVectorType>(RHSTy)->getNumElements() < | |||
2124 | cast<FixedVectorType>(LHSTy)->getNumElements(); | |||
2125 | }; | |||
2126 | auto RankVectorTypesEq = [&DL](VectorType *RHSTy, VectorType *LHSTy) { | |||
2127 | (void)DL; | |||
2128 | assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==(static_cast <bool> (DL.getTypeSizeInBits(RHSTy).getFixedValue () == DL.getTypeSizeInBits(LHSTy).getFixedValue() && "Cannot have vector types of different sizes!" ) ? void (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy).getFixedValue() == DL.getTypeSizeInBits(LHSTy).getFixedValue() && \"Cannot have vector types of different sizes!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2130, __extension__ __PRETTY_FUNCTION__ )) | |||
2129 | DL.getTypeSizeInBits(LHSTy).getFixedValue() &&(static_cast <bool> (DL.getTypeSizeInBits(RHSTy).getFixedValue () == DL.getTypeSizeInBits(LHSTy).getFixedValue() && "Cannot have vector types of different sizes!" ) ? void (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy).getFixedValue() == DL.getTypeSizeInBits(LHSTy).getFixedValue() && \"Cannot have vector types of different sizes!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2130, __extension__ __PRETTY_FUNCTION__ )) | |||
2130 | "Cannot have vector types of different sizes!")(static_cast <bool> (DL.getTypeSizeInBits(RHSTy).getFixedValue () == DL.getTypeSizeInBits(LHSTy).getFixedValue() && "Cannot have vector types of different sizes!" ) ? void (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy).getFixedValue() == DL.getTypeSizeInBits(LHSTy).getFixedValue() && \"Cannot have vector types of different sizes!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2130, __extension__ __PRETTY_FUNCTION__ )); | |||
2131 | assert(RHSTy->getElementType()->isIntegerTy() &&(static_cast <bool> (RHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2132, __extension__ __PRETTY_FUNCTION__ )) | |||
2132 | "All non-integer types eliminated!")(static_cast <bool> (RHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2132, __extension__ __PRETTY_FUNCTION__ )); | |||
2133 | assert(LHSTy->getElementType()->isIntegerTy() &&(static_cast <bool> (LHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2134, __extension__ __PRETTY_FUNCTION__ )) | |||
2134 | "All non-integer types eliminated!")(static_cast <bool> (LHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2134, __extension__ __PRETTY_FUNCTION__ )); | |||
2135 | return cast<FixedVectorType>(RHSTy)->getNumElements() == | |||
2136 | cast<FixedVectorType>(LHSTy)->getNumElements(); | |||
2137 | }; | |||
2138 | llvm::sort(CandidateTys, RankVectorTypesComp); | |||
2139 | CandidateTys.erase(std::unique(CandidateTys.begin(), CandidateTys.end(), | |||
2140 | RankVectorTypesEq), | |||
2141 | CandidateTys.end()); | |||
2142 | } else { | |||
2143 | // The only way to have the same element type in every vector type is to | |||
2144 | // have the same vector type. Check that and remove all but one. | |||
2145 | #ifndef NDEBUG | |||
2146 | for (VectorType *VTy : CandidateTys) { | |||
2147 | assert(VTy->getElementType() == CommonEltTy &&(static_cast <bool> (VTy->getElementType() == CommonEltTy && "Unaccounted for element type!") ? void (0) : __assert_fail ("VTy->getElementType() == CommonEltTy && \"Unaccounted for element type!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2148, __extension__ __PRETTY_FUNCTION__ )) | |||
2148 | "Unaccounted for element type!")(static_cast <bool> (VTy->getElementType() == CommonEltTy && "Unaccounted for element type!") ? void (0) : __assert_fail ("VTy->getElementType() == CommonEltTy && \"Unaccounted for element type!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2148, __extension__ __PRETTY_FUNCTION__ )); | |||
2149 | assert(VTy == CandidateTys[0] &&(static_cast <bool> (VTy == CandidateTys[0] && "Different vector types with the same element type!" ) ? void (0) : __assert_fail ("VTy == CandidateTys[0] && \"Different vector types with the same element type!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2150, __extension__ __PRETTY_FUNCTION__ )) | |||
2150 | "Different vector types with the same element type!")(static_cast <bool> (VTy == CandidateTys[0] && "Different vector types with the same element type!" ) ? void (0) : __assert_fail ("VTy == CandidateTys[0] && \"Different vector types with the same element type!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2150, __extension__ __PRETTY_FUNCTION__ )); | |||
2151 | } | |||
2152 | #endif | |||
2153 | CandidateTys.resize(1); | |||
2154 | } | |||
2155 | ||||
2156 | // FIXME: hack. Do we have a named constant for this? | |||
2157 | // SDAG SDNode can't have more than 65535 operands. | |||
2158 | llvm::erase_if(CandidateTys, [](VectorType *VTy) { | |||
2159 | return cast<FixedVectorType>(VTy)->getNumElements() > | |||
2160 | std::numeric_limits<unsigned short>::max(); | |||
2161 | }); | |||
2162 | ||||
2163 | for (VectorType *VTy : CandidateTys) | |||
2164 | if (checkVectorTypeForPromotion(P, VTy, DL)) | |||
2165 | return VTy; | |||
2166 | ||||
2167 | return nullptr; | |||
2168 | } | |||
2169 | ||||
2170 | /// Test whether a slice of an alloca is valid for integer widening. | |||
2171 | /// | |||
2172 | /// This implements the necessary checking for the \c isIntegerWideningViable | |||
2173 | /// test below on a single slice of the alloca. | |||
2174 | static bool isIntegerWideningViableForSlice(const Slice &S, | |||
2175 | uint64_t AllocBeginOffset, | |||
2176 | Type *AllocaTy, | |||
2177 | const DataLayout &DL, | |||
2178 | bool &WholeAllocaOp) { | |||
2179 | uint64_t Size = DL.getTypeStoreSize(AllocaTy).getFixedValue(); | |||
2180 | ||||
2181 | uint64_t RelBegin = S.beginOffset() - AllocBeginOffset; | |||
2182 | uint64_t RelEnd = S.endOffset() - AllocBeginOffset; | |||
2183 | ||||
2184 | Use *U = S.getUse(); | |||
2185 | ||||
2186 | // Lifetime intrinsics operate over the whole alloca whose sizes are usually | |||
2187 | // larger than other load/store slices (RelEnd > Size). But lifetime are | |||
2188 | // always promotable and should not impact other slices' promotability of the | |||
2189 | // partition. | |||
2190 | if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) { | |||
2191 | if (II->isLifetimeStartOrEnd() || II->isDroppable()) | |||
2192 | return true; | |||
2193 | } | |||
2194 | ||||
2195 | // We can't reasonably handle cases where the load or store extends past | |||
2196 | // the end of the alloca's type and into its padding. | |||
2197 | if (RelEnd > Size) | |||
2198 | return false; | |||
2199 | ||||
2200 | if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | |||
2201 | if (LI->isVolatile()) | |||
2202 | return false; | |||
2203 | // We can't handle loads that extend past the allocated memory. | |||
2204 | if (DL.getTypeStoreSize(LI->getType()).getFixedValue() > Size) | |||
2205 | return false; | |||
2206 | // So far, AllocaSliceRewriter does not support widening split slice tails | |||
2207 | // in rewriteIntegerLoad. | |||
2208 | if (S.beginOffset() < AllocBeginOffset) | |||
2209 | return false; | |||
2210 | // Note that we don't count vector loads or stores as whole-alloca | |||
2211 | // operations which enable integer widening because we would prefer to use | |||
2212 | // vector widening instead. | |||
2213 | if (!isa<VectorType>(LI->getType()) && RelBegin == 0 && RelEnd == Size) | |||
2214 | WholeAllocaOp = true; | |||
2215 | if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) { | |||
2216 | if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedValue()) | |||
2217 | return false; | |||
2218 | } else if (RelBegin != 0 || RelEnd != Size || | |||
2219 | !canConvertValue(DL, AllocaTy, LI->getType())) { | |||
2220 | // Non-integer loads need to be convertible from the alloca type so that | |||
2221 | // they are promotable. | |||
2222 | return false; | |||
2223 | } | |||
2224 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | |||
2225 | Type *ValueTy = SI->getValueOperand()->getType(); | |||
2226 | if (SI->isVolatile()) | |||
2227 | return false; | |||
2228 | // We can't handle stores that extend past the allocated memory. | |||
2229 | if (DL.getTypeStoreSize(ValueTy).getFixedValue() > Size) | |||
2230 | return false; | |||
2231 | // So far, AllocaSliceRewriter does not support widening split slice tails | |||
2232 | // in rewriteIntegerStore. | |||
2233 | if (S.beginOffset() < AllocBeginOffset) | |||
2234 | return false; | |||
2235 | // Note that we don't count vector loads or stores as whole-alloca | |||
2236 | // operations which enable integer widening because we would prefer to use | |||
2237 | // vector widening instead. | |||
2238 | if (!isa<VectorType>(ValueTy) && RelBegin == 0 && RelEnd == Size) | |||
2239 | WholeAllocaOp = true; | |||
2240 | if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) { | |||
2241 | if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedValue()) | |||
2242 | return false; | |||
2243 | } else if (RelBegin != 0 || RelEnd != Size || | |||
2244 | !canConvertValue(DL, ValueTy, AllocaTy)) { | |||
2245 | // Non-integer stores need to be convertible to the alloca type so that | |||
2246 | // they are promotable. | |||
2247 | return false; | |||
2248 | } | |||
2249 | } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { | |||
2250 | if (MI->isVolatile() || !isa<Constant>(MI->getLength())) | |||
2251 | return false; | |||
2252 | if (!S.isSplittable()) | |||
2253 | return false; // Skip any unsplittable intrinsics. | |||
2254 | } else { | |||
2255 | return false; | |||
2256 | } | |||
2257 | ||||
2258 | return true; | |||
2259 | } | |||
2260 | ||||
2261 | /// Test whether the given alloca partition's integer operations can be | |||
2262 | /// widened to promotable ones. | |||
2263 | /// | |||
2264 | /// This is a quick test to check whether we can rewrite the integer loads and | |||
2265 | /// stores to a particular alloca into wider loads and stores and be able to | |||
2266 | /// promote the resulting alloca. | |||
2267 | static bool isIntegerWideningViable(Partition &P, Type *AllocaTy, | |||
2268 | const DataLayout &DL) { | |||
2269 | uint64_t SizeInBits = DL.getTypeSizeInBits(AllocaTy).getFixedValue(); | |||
2270 | // Don't create integer types larger than the maximum bitwidth. | |||
2271 | if (SizeInBits > IntegerType::MAX_INT_BITS) | |||
2272 | return false; | |||
2273 | ||||
2274 | // Don't try to handle allocas with bit-padding. | |||
2275 | if (SizeInBits != DL.getTypeStoreSizeInBits(AllocaTy).getFixedValue()) | |||
2276 | return false; | |||
2277 | ||||
2278 | // We need to ensure that an integer type with the appropriate bitwidth can | |||
2279 | // be converted to the alloca type, whatever that is. We don't want to force | |||
2280 | // the alloca itself to have an integer type if there is a more suitable one. | |||
2281 | Type *IntTy = Type::getIntNTy(AllocaTy->getContext(), SizeInBits); | |||
2282 | if (!canConvertValue(DL, AllocaTy, IntTy) || | |||
2283 | !canConvertValue(DL, IntTy, AllocaTy)) | |||
2284 | return false; | |||
2285 | ||||
2286 | // While examining uses, we ensure that the alloca has a covering load or | |||
2287 | // store. We don't want to widen the integer operations only to fail to | |||
2288 | // promote due to some other unsplittable entry (which we may make splittable | |||
2289 | // later). However, if there are only splittable uses, go ahead and assume | |||
2290 | // that we cover the alloca. | |||
2291 | // FIXME: We shouldn't consider split slices that happen to start in the | |||
2292 | // partition here... | |||
2293 | bool WholeAllocaOp = P.empty() && DL.isLegalInteger(SizeInBits); | |||
2294 | ||||
2295 | for (const Slice &S : P) | |||
2296 | if (!isIntegerWideningViableForSlice(S, P.beginOffset(), AllocaTy, DL, | |||
2297 | WholeAllocaOp)) | |||
2298 | return false; | |||
2299 | ||||
2300 | for (const Slice *S : P.splitSliceTails()) | |||
2301 | if (!isIntegerWideningViableForSlice(*S, P.beginOffset(), AllocaTy, DL, | |||
2302 | WholeAllocaOp)) | |||
2303 | return false; | |||
2304 | ||||
2305 | return WholeAllocaOp; | |||
2306 | } | |||
2307 | ||||
2308 | static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V, | |||
2309 | IntegerType *Ty, uint64_t Offset, | |||
2310 | const Twine &Name) { | |||
2311 | LLVM_DEBUG(dbgs() << " start: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " start: " << *V << "\n"; } } while (false); | |||
2312 | IntegerType *IntTy = cast<IntegerType>(V->getType()); | |||
2313 | assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedValue () + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && "Element extends past full value") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedValue() + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && \"Element extends past full value\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2315, __extension__ __PRETTY_FUNCTION__ )) | |||
2314 | DL.getTypeStoreSize(IntTy).getFixedValue() &&(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedValue () + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && "Element extends past full value") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedValue() + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && \"Element extends past full value\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2315, __extension__ __PRETTY_FUNCTION__ )) | |||
2315 | "Element extends past full value")(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedValue () + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && "Element extends past full value") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedValue() + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && \"Element extends past full value\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2315, __extension__ __PRETTY_FUNCTION__ )); | |||
2316 | uint64_t ShAmt = 8 * Offset; | |||
2317 | if (DL.isBigEndian()) | |||
2318 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedValue() - | |||
2319 | DL.getTypeStoreSize(Ty).getFixedValue() - Offset); | |||
2320 | if (ShAmt) { | |||
2321 | V = IRB.CreateLShr(V, ShAmt, Name + ".shift"); | |||
2322 | LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shifted: " << *V << "\n"; } } while (false); | |||
2323 | } | |||
2324 | assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&(static_cast <bool> (Ty->getBitWidth() <= IntTy-> getBitWidth() && "Cannot extract to a larger integer!" ) ? void (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2325, __extension__ __PRETTY_FUNCTION__ )) | |||
2325 | "Cannot extract to a larger integer!")(static_cast <bool> (Ty->getBitWidth() <= IntTy-> getBitWidth() && "Cannot extract to a larger integer!" ) ? void (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2325, __extension__ __PRETTY_FUNCTION__ )); | |||
2326 | if (Ty != IntTy) { | |||
2327 | V = IRB.CreateTrunc(V, Ty, Name + ".trunc"); | |||
2328 | LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " trunced: " << *V << "\n"; } } while (false); | |||
2329 | } | |||
2330 | return V; | |||
2331 | } | |||
2332 | ||||
2333 | static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old, | |||
2334 | Value *V, uint64_t Offset, const Twine &Name) { | |||
2335 | IntegerType *IntTy = cast<IntegerType>(Old->getType()); | |||
2336 | IntegerType *Ty = cast<IntegerType>(V->getType()); | |||
2337 | assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&(static_cast <bool> (Ty->getBitWidth() <= IntTy-> getBitWidth() && "Cannot insert a larger integer!") ? void (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot insert a larger integer!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2338, __extension__ __PRETTY_FUNCTION__ )) | |||
2338 | "Cannot insert a larger integer!")(static_cast <bool> (Ty->getBitWidth() <= IntTy-> getBitWidth() && "Cannot insert a larger integer!") ? void (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot insert a larger integer!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2338, __extension__ __PRETTY_FUNCTION__ )); | |||
2339 | LLVM_DEBUG(dbgs() << " start: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " start: " << *V << "\n"; } } while (false); | |||
2340 | if (Ty != IntTy) { | |||
2341 | V = IRB.CreateZExt(V, IntTy, Name + ".ext"); | |||
2342 | LLVM_DEBUG(dbgs() << " extended: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " extended: " << *V << "\n"; } } while (false); | |||
2343 | } | |||
2344 | assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedValue () + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && "Element store outside of alloca store") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedValue() + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && \"Element store outside of alloca store\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2346, __extension__ __PRETTY_FUNCTION__ )) | |||
2345 | DL.getTypeStoreSize(IntTy).getFixedValue() &&(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedValue () + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && "Element store outside of alloca store") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedValue() + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && \"Element store outside of alloca store\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2346, __extension__ __PRETTY_FUNCTION__ )) | |||
2346 | "Element store outside of alloca store")(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedValue () + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && "Element store outside of alloca store") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedValue() + Offset <= DL.getTypeStoreSize(IntTy).getFixedValue() && \"Element store outside of alloca store\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2346, __extension__ __PRETTY_FUNCTION__ )); | |||
2347 | uint64_t ShAmt = 8 * Offset; | |||
2348 | if (DL.isBigEndian()) | |||
2349 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedValue() - | |||
2350 | DL.getTypeStoreSize(Ty).getFixedValue() - Offset); | |||
2351 | if (ShAmt) { | |||
2352 | V = IRB.CreateShl(V, ShAmt, Name + ".shift"); | |||
2353 | LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shifted: " << *V << "\n"; } } while (false); | |||
2354 | } | |||
2355 | ||||
2356 | if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) { | |||
2357 | APInt Mask = ~Ty->getMask().zext(IntTy->getBitWidth()).shl(ShAmt); | |||
2358 | Old = IRB.CreateAnd(Old, Mask, Name + ".mask"); | |||
2359 | LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " masked: " << *Old << "\n"; } } while (false); | |||
2360 | V = IRB.CreateOr(Old, V, Name + ".insert"); | |||
2361 | LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " inserted: " << *V << "\n"; } } while (false); | |||
2362 | } | |||
2363 | return V; | |||
2364 | } | |||
2365 | ||||
2366 | static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex, | |||
2367 | unsigned EndIndex, const Twine &Name) { | |||
2368 | auto *VecTy = cast<FixedVectorType>(V->getType()); | |||
2369 | unsigned NumElements = EndIndex - BeginIndex; | |||
2370 | assert(NumElements <= VecTy->getNumElements() && "Too many elements!")(static_cast <bool> (NumElements <= VecTy->getNumElements () && "Too many elements!") ? void (0) : __assert_fail ("NumElements <= VecTy->getNumElements() && \"Too many elements!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2370, __extension__ __PRETTY_FUNCTION__ )); | |||
2371 | ||||
2372 | if (NumElements == VecTy->getNumElements()) | |||
2373 | return V; | |||
2374 | ||||
2375 | if (NumElements == 1) { | |||
2376 | V = IRB.CreateExtractElement(V, IRB.getInt32(BeginIndex), | |||
2377 | Name + ".extract"); | |||
2378 | LLVM_DEBUG(dbgs() << " extract: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " extract: " << *V << "\n"; } } while (false); | |||
2379 | return V; | |||
2380 | } | |||
2381 | ||||
2382 | auto Mask = llvm::to_vector<8>(llvm::seq<int>(BeginIndex, EndIndex)); | |||
2383 | V = IRB.CreateShuffleVector(V, Mask, Name + ".extract"); | |||
2384 | LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shuffle: " << *V << "\n"; } } while (false); | |||
2385 | return V; | |||
2386 | } | |||
2387 | ||||
2388 | static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V, | |||
2389 | unsigned BeginIndex, const Twine &Name) { | |||
2390 | VectorType *VecTy = cast<VectorType>(Old->getType()); | |||
2391 | assert(VecTy && "Can only insert a vector into a vector")(static_cast <bool> (VecTy && "Can only insert a vector into a vector" ) ? void (0) : __assert_fail ("VecTy && \"Can only insert a vector into a vector\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2391, __extension__ __PRETTY_FUNCTION__ )); | |||
2392 | ||||
2393 | VectorType *Ty = dyn_cast<VectorType>(V->getType()); | |||
2394 | if (!Ty) { | |||
2395 | // Single element to insert. | |||
2396 | V = IRB.CreateInsertElement(Old, V, IRB.getInt32(BeginIndex), | |||
2397 | Name + ".insert"); | |||
2398 | LLVM_DEBUG(dbgs() << " insert: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " insert: " << *V << "\n"; } } while (false); | |||
2399 | return V; | |||
2400 | } | |||
2401 | ||||
2402 | assert(cast<FixedVectorType>(Ty)->getNumElements() <=(static_cast <bool> (cast<FixedVectorType>(Ty)-> getNumElements() <= cast<FixedVectorType>(VecTy)-> getNumElements() && "Too many elements!") ? void (0) : __assert_fail ("cast<FixedVectorType>(Ty)->getNumElements() <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2404, __extension__ __PRETTY_FUNCTION__ )) | |||
2403 | cast<FixedVectorType>(VecTy)->getNumElements() &&(static_cast <bool> (cast<FixedVectorType>(Ty)-> getNumElements() <= cast<FixedVectorType>(VecTy)-> getNumElements() && "Too many elements!") ? void (0) : __assert_fail ("cast<FixedVectorType>(Ty)->getNumElements() <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2404, __extension__ __PRETTY_FUNCTION__ )) | |||
2404 | "Too many elements!")(static_cast <bool> (cast<FixedVectorType>(Ty)-> getNumElements() <= cast<FixedVectorType>(VecTy)-> getNumElements() && "Too many elements!") ? void (0) : __assert_fail ("cast<FixedVectorType>(Ty)->getNumElements() <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2404, __extension__ __PRETTY_FUNCTION__ )); | |||
2405 | if (cast<FixedVectorType>(Ty)->getNumElements() == | |||
2406 | cast<FixedVectorType>(VecTy)->getNumElements()) { | |||
2407 | assert(V->getType() == VecTy && "Vector type mismatch")(static_cast <bool> (V->getType() == VecTy && "Vector type mismatch") ? void (0) : __assert_fail ("V->getType() == VecTy && \"Vector type mismatch\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2407, __extension__ __PRETTY_FUNCTION__ )); | |||
2408 | return V; | |||
2409 | } | |||
2410 | unsigned EndIndex = BeginIndex + cast<FixedVectorType>(Ty)->getNumElements(); | |||
2411 | ||||
2412 | // When inserting a smaller vector into the larger to store, we first | |||
2413 | // use a shuffle vector to widen it with undef elements, and then | |||
2414 | // a second shuffle vector to select between the loaded vector and the | |||
2415 | // incoming vector. | |||
2416 | SmallVector<int, 8> Mask; | |||
2417 | Mask.reserve(cast<FixedVectorType>(VecTy)->getNumElements()); | |||
2418 | for (unsigned i = 0; i != cast<FixedVectorType>(VecTy)->getNumElements(); ++i) | |||
2419 | if (i >= BeginIndex && i < EndIndex) | |||
2420 | Mask.push_back(i - BeginIndex); | |||
2421 | else | |||
2422 | Mask.push_back(-1); | |||
2423 | V = IRB.CreateShuffleVector(V, Mask, Name + ".expand"); | |||
2424 | LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shuffle: " << *V << "\n"; } } while (false); | |||
2425 | ||||
2426 | SmallVector<Constant *, 8> Mask2; | |||
2427 | Mask2.reserve(cast<FixedVectorType>(VecTy)->getNumElements()); | |||
2428 | for (unsigned i = 0; i != cast<FixedVectorType>(VecTy)->getNumElements(); ++i) | |||
2429 | Mask2.push_back(IRB.getInt1(i >= BeginIndex && i < EndIndex)); | |||
2430 | ||||
2431 | V = IRB.CreateSelect(ConstantVector::get(Mask2), V, Old, Name + "blend"); | |||
2432 | ||||
2433 | LLVM_DEBUG(dbgs() << " blend: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " blend: " << *V << "\n"; } } while (false); | |||
2434 | return V; | |||
2435 | } | |||
2436 | ||||
2437 | /// Visitor to rewrite instructions using p particular slice of an alloca | |||
2438 | /// to use a new alloca. | |||
2439 | /// | |||
2440 | /// Also implements the rewriting to vector-based accesses when the partition | |||
2441 | /// passes the isVectorPromotionViable predicate. Most of the rewriting logic | |||
2442 | /// lives here. | |||
2443 | class llvm::sroa::AllocaSliceRewriter | |||
2444 | : public InstVisitor<AllocaSliceRewriter, bool> { | |||
2445 | // Befriend the base class so it can delegate to private visit methods. | |||
2446 | friend class InstVisitor<AllocaSliceRewriter, bool>; | |||
2447 | ||||
2448 | using Base = InstVisitor<AllocaSliceRewriter, bool>; | |||
2449 | ||||
2450 | const DataLayout &DL; | |||
2451 | AllocaSlices &AS; | |||
2452 | SROAPass &Pass; | |||
2453 | AllocaInst &OldAI, &NewAI; | |||
2454 | const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset; | |||
2455 | Type *NewAllocaTy; | |||
2456 | ||||
2457 | // This is a convenience and flag variable that will be null unless the new | |||
2458 | // alloca's integer operations should be widened to this integer type due to | |||
2459 | // passing isIntegerWideningViable above. If it is non-null, the desired | |||
2460 | // integer type will be stored here for easy access during rewriting. | |||
2461 | IntegerType *IntTy; | |||
2462 | ||||
2463 | // If we are rewriting an alloca partition which can be written as pure | |||
2464 | // vector operations, we stash extra information here. When VecTy is | |||
2465 | // non-null, we have some strict guarantees about the rewritten alloca: | |||
2466 | // - The new alloca is exactly the size of the vector type here. | |||
2467 | // - The accesses all either map to the entire vector or to a single | |||
2468 | // element. | |||
2469 | // - The set of accessing instructions is only one of those handled above | |||
2470 | // in isVectorPromotionViable. Generally these are the same access kinds | |||
2471 | // which are promotable via mem2reg. | |||
2472 | VectorType *VecTy; | |||
2473 | Type *ElementTy; | |||
2474 | uint64_t ElementSize; | |||
2475 | ||||
2476 | // The original offset of the slice currently being rewritten relative to | |||
2477 | // the original alloca. | |||
2478 | uint64_t BeginOffset = 0; | |||
2479 | uint64_t EndOffset = 0; | |||
2480 | ||||
2481 | // The new offsets of the slice currently being rewritten relative to the | |||
2482 | // original alloca. | |||
2483 | uint64_t NewBeginOffset = 0, NewEndOffset = 0; | |||
2484 | ||||
2485 | uint64_t SliceSize = 0; | |||
2486 | bool IsSplittable = false; | |||
2487 | bool IsSplit = false; | |||
2488 | Use *OldUse = nullptr; | |||
2489 | Instruction *OldPtr = nullptr; | |||
2490 | ||||
2491 | // Track post-rewrite users which are PHI nodes and Selects. | |||
2492 | SmallSetVector<PHINode *, 8> &PHIUsers; | |||
2493 | SmallSetVector<SelectInst *, 8> &SelectUsers; | |||
2494 | ||||
2495 | // Utility IR builder, whose name prefix is setup for each visited use, and | |||
2496 | // the insertion point is set to point to the user. | |||
2497 | IRBuilderTy IRB; | |||
2498 | ||||
2499 | // Return the new alloca, addrspacecasted if required to avoid changing the | |||
2500 | // addrspace of a volatile access. | |||
2501 | Value *getPtrToNewAI(unsigned AddrSpace, bool IsVolatile) { | |||
2502 | if (!IsVolatile || AddrSpace == NewAI.getType()->getPointerAddressSpace()) | |||
2503 | return &NewAI; | |||
2504 | ||||
2505 | Type *AccessTy = NewAI.getAllocatedType()->getPointerTo(AddrSpace); | |||
2506 | return IRB.CreateAddrSpaceCast(&NewAI, AccessTy); | |||
2507 | } | |||
2508 | ||||
2509 | public: | |||
2510 | AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROAPass &Pass, | |||
2511 | AllocaInst &OldAI, AllocaInst &NewAI, | |||
2512 | uint64_t NewAllocaBeginOffset, | |||
2513 | uint64_t NewAllocaEndOffset, bool IsIntegerPromotable, | |||
2514 | VectorType *PromotableVecTy, | |||
2515 | SmallSetVector<PHINode *, 8> &PHIUsers, | |||
2516 | SmallSetVector<SelectInst *, 8> &SelectUsers) | |||
2517 | : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI), | |||
2518 | NewAllocaBeginOffset(NewAllocaBeginOffset), | |||
2519 | NewAllocaEndOffset(NewAllocaEndOffset), | |||
2520 | NewAllocaTy(NewAI.getAllocatedType()), | |||
2521 | IntTy( | |||
2522 | IsIntegerPromotable | |||
2523 | ? Type::getIntNTy(NewAI.getContext(), | |||
2524 | DL.getTypeSizeInBits(NewAI.getAllocatedType()) | |||
2525 | .getFixedValue()) | |||
2526 | : nullptr), | |||
2527 | VecTy(PromotableVecTy), | |||
2528 | ElementTy(VecTy ? VecTy->getElementType() : nullptr), | |||
2529 | ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy).getFixedValue() / 8 | |||
2530 | : 0), | |||
2531 | PHIUsers(PHIUsers), SelectUsers(SelectUsers), | |||
2532 | IRB(NewAI.getContext(), ConstantFolder()) { | |||
2533 | if (VecTy) { | |||
2534 | assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 &&(static_cast <bool> ((DL.getTypeSizeInBits(ElementTy).getFixedValue () % 8) == 0 && "Only multiple-of-8 sized vector elements are viable" ) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 && \"Only multiple-of-8 sized vector elements are viable\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2535, __extension__ __PRETTY_FUNCTION__ )) | |||
2535 | "Only multiple-of-8 sized vector elements are viable")(static_cast <bool> ((DL.getTypeSizeInBits(ElementTy).getFixedValue () % 8) == 0 && "Only multiple-of-8 sized vector elements are viable" ) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 && \"Only multiple-of-8 sized vector elements are viable\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2535, __extension__ __PRETTY_FUNCTION__ )); | |||
2536 | ++NumVectorized; | |||
2537 | } | |||
2538 | assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy))(static_cast <bool> ((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy)) ? void (0) : __assert_fail ("(!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy)" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2538, __extension__ __PRETTY_FUNCTION__ )); | |||
2539 | } | |||
2540 | ||||
2541 | bool visit(AllocaSlices::const_iterator I) { | |||
2542 | bool CanSROA = true; | |||
2543 | BeginOffset = I->beginOffset(); | |||
2544 | EndOffset = I->endOffset(); | |||
2545 | IsSplittable = I->isSplittable(); | |||
2546 | IsSplit = | |||
2547 | BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset; | |||
2548 | LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " rewriting " << (IsSplit ? "split " : ""); } } while (false); | |||
2549 | LLVM_DEBUG(AS.printSlice(dbgs(), I, ""))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.printSlice(dbgs(), I, ""); } } while (false); | |||
2550 | LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n"; } } while (false); | |||
2551 | ||||
2552 | // Compute the intersecting offset range. | |||
2553 | assert(BeginOffset < NewAllocaEndOffset)(static_cast <bool> (BeginOffset < NewAllocaEndOffset ) ? void (0) : __assert_fail ("BeginOffset < NewAllocaEndOffset" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2553, __extension__ __PRETTY_FUNCTION__ )); | |||
2554 | assert(EndOffset > NewAllocaBeginOffset)(static_cast <bool> (EndOffset > NewAllocaBeginOffset ) ? void (0) : __assert_fail ("EndOffset > NewAllocaBeginOffset" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2554, __extension__ __PRETTY_FUNCTION__ )); | |||
2555 | NewBeginOffset = std::max(BeginOffset, NewAllocaBeginOffset); | |||
2556 | NewEndOffset = std::min(EndOffset, NewAllocaEndOffset); | |||
2557 | ||||
2558 | SliceSize = NewEndOffset - NewBeginOffset; | |||
2559 | LLVM_DEBUG(dbgs() << " Begin:(" << BeginOffset << ", " << EndOffsetdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset << ") NewBegin:(" << NewBeginOffset << ", " << NewEndOffset << ") NewAllocaBegin:(" << NewAllocaBeginOffset << ", " << NewAllocaEndOffset << ")\n"; } } while (false) | |||
2560 | << ") NewBegin:(" << NewBeginOffset << ", "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset << ") NewBegin:(" << NewBeginOffset << ", " << NewEndOffset << ") NewAllocaBegin:(" << NewAllocaBeginOffset << ", " << NewAllocaEndOffset << ")\n"; } } while (false) | |||
2561 | << NewEndOffset << ") NewAllocaBegin:("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset << ") NewBegin:(" << NewBeginOffset << ", " << NewEndOffset << ") NewAllocaBegin:(" << NewAllocaBeginOffset << ", " << NewAllocaEndOffset << ")\n"; } } while (false) | |||
2562 | << NewAllocaBeginOffset << ", " << NewAllocaEndOffsetdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset << ") NewBegin:(" << NewBeginOffset << ", " << NewEndOffset << ") NewAllocaBegin:(" << NewAllocaBeginOffset << ", " << NewAllocaEndOffset << ")\n"; } } while (false) | |||
2563 | << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset << ") NewBegin:(" << NewBeginOffset << ", " << NewEndOffset << ") NewAllocaBegin:(" << NewAllocaBeginOffset << ", " << NewAllocaEndOffset << ")\n"; } } while (false); | |||
2564 | assert(IsSplit || NewBeginOffset == BeginOffset)(static_cast <bool> (IsSplit || NewBeginOffset == BeginOffset ) ? void (0) : __assert_fail ("IsSplit || NewBeginOffset == BeginOffset" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2564, __extension__ __PRETTY_FUNCTION__ )); | |||
2565 | OldUse = I->getUse(); | |||
2566 | OldPtr = cast<Instruction>(OldUse->get()); | |||
2567 | ||||
2568 | Instruction *OldUserI = cast<Instruction>(OldUse->getUser()); | |||
2569 | IRB.SetInsertPoint(OldUserI); | |||
2570 | IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc()); | |||
2571 | IRB.getInserter().SetNamePrefix( | |||
2572 | Twine(NewAI.getName()) + "." + Twine(BeginOffset) + "."); | |||
2573 | ||||
2574 | CanSROA &= visit(cast<Instruction>(OldUse->getUser())); | |||
2575 | if (VecTy || IntTy) | |||
2576 | assert(CanSROA)(static_cast <bool> (CanSROA) ? void (0) : __assert_fail ("CanSROA", "llvm/lib/Transforms/Scalar/SROA.cpp", 2576, __extension__ __PRETTY_FUNCTION__)); | |||
2577 | return CanSROA; | |||
2578 | } | |||
2579 | ||||
2580 | private: | |||
2581 | // Make sure the other visit overloads are visible. | |||
2582 | using Base::visit; | |||
2583 | ||||
2584 | // Every instruction which can end up as a user must have a rewrite rule. | |||
2585 | bool visitInstruction(Instruction &I) { | |||
2586 | LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " !!!! Cannot rewrite: " << I << "\n"; } } while (false); | |||
2587 | llvm_unreachable("No rewrite rule for this instruction!")::llvm::llvm_unreachable_internal("No rewrite rule for this instruction!" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2587); | |||
2588 | } | |||
2589 | ||||
2590 | Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) { | |||
2591 | // Note that the offset computation can use BeginOffset or NewBeginOffset | |||
2592 | // interchangeably for unsplit slices. | |||
2593 | assert(IsSplit || BeginOffset == NewBeginOffset)(static_cast <bool> (IsSplit || BeginOffset == NewBeginOffset ) ? void (0) : __assert_fail ("IsSplit || BeginOffset == NewBeginOffset" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2593, __extension__ __PRETTY_FUNCTION__ )); | |||
2594 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
2595 | ||||
2596 | #ifndef NDEBUG | |||
2597 | StringRef OldName = OldPtr->getName(); | |||
2598 | // Skip through the last '.sroa.' component of the name. | |||
2599 | size_t LastSROAPrefix = OldName.rfind(".sroa."); | |||
2600 | if (LastSROAPrefix != StringRef::npos) { | |||
2601 | OldName = OldName.substr(LastSROAPrefix + strlen(".sroa.")); | |||
2602 | // Look for an SROA slice index. | |||
2603 | size_t IndexEnd = OldName.find_first_not_of("0123456789"); | |||
2604 | if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') { | |||
2605 | // Strip the index and look for the offset. | |||
2606 | OldName = OldName.substr(IndexEnd + 1); | |||
2607 | size_t OffsetEnd = OldName.find_first_not_of("0123456789"); | |||
2608 | if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.') | |||
2609 | // Strip the offset. | |||
2610 | OldName = OldName.substr(OffsetEnd + 1); | |||
2611 | } | |||
2612 | } | |||
2613 | // Strip any SROA suffixes as well. | |||
2614 | OldName = OldName.substr(0, OldName.find(".sroa_")); | |||
2615 | #endif | |||
2616 | ||||
2617 | return getAdjustedPtr(IRB, DL, &NewAI, | |||
2618 | APInt(DL.getIndexTypeSizeInBits(PointerTy), Offset), | |||
2619 | PointerTy, | |||
2620 | #ifndef NDEBUG | |||
2621 | Twine(OldName) + "." | |||
2622 | #else | |||
2623 | Twine() | |||
2624 | #endif | |||
2625 | ); | |||
2626 | } | |||
2627 | ||||
2628 | /// Compute suitable alignment to access this slice of the *new* | |||
2629 | /// alloca. | |||
2630 | /// | |||
2631 | /// You can optionally pass a type to this routine and if that type's ABI | |||
2632 | /// alignment is itself suitable, this will return zero. | |||
2633 | Align getSliceAlign() { | |||
2634 | return commonAlignment(NewAI.getAlign(), | |||
2635 | NewBeginOffset - NewAllocaBeginOffset); | |||
2636 | } | |||
2637 | ||||
2638 | unsigned getIndex(uint64_t Offset) { | |||
2639 | assert(VecTy && "Can only call getIndex when rewriting a vector")(static_cast <bool> (VecTy && "Can only call getIndex when rewriting a vector" ) ? void (0) : __assert_fail ("VecTy && \"Can only call getIndex when rewriting a vector\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2639, __extension__ __PRETTY_FUNCTION__ )); | |||
2640 | uint64_t RelOffset = Offset - NewAllocaBeginOffset; | |||
2641 | assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds")(static_cast <bool> (RelOffset / ElementSize < (4294967295U ) && "Index out of bounds") ? void (0) : __assert_fail ("RelOffset / ElementSize < UINT32_MAX && \"Index out of bounds\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2641, __extension__ __PRETTY_FUNCTION__ )); | |||
2642 | uint32_t Index = RelOffset / ElementSize; | |||
2643 | assert(Index * ElementSize == RelOffset)(static_cast <bool> (Index * ElementSize == RelOffset) ? void (0) : __assert_fail ("Index * ElementSize == RelOffset" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2643, __extension__ __PRETTY_FUNCTION__ )); | |||
2644 | return Index; | |||
2645 | } | |||
2646 | ||||
2647 | void deleteIfTriviallyDead(Value *V) { | |||
2648 | Instruction *I = cast<Instruction>(V); | |||
2649 | if (isInstructionTriviallyDead(I)) | |||
2650 | Pass.DeadInsts.push_back(I); | |||
2651 | } | |||
2652 | ||||
2653 | Value *rewriteVectorizedLoadInst(LoadInst &LI) { | |||
2654 | unsigned BeginIndex = getIndex(NewBeginOffset); | |||
2655 | unsigned EndIndex = getIndex(NewEndOffset); | |||
2656 | assert(EndIndex > BeginIndex && "Empty vector!")(static_cast <bool> (EndIndex > BeginIndex && "Empty vector!") ? void (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2656, __extension__ __PRETTY_FUNCTION__ )); | |||
2657 | ||||
2658 | LoadInst *Load = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2659 | NewAI.getAlign(), "load"); | |||
2660 | ||||
2661 | Load->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2662 | LLVMContext::MD_access_group}); | |||
2663 | return extractVector(IRB, Load, BeginIndex, EndIndex, "vec"); | |||
2664 | } | |||
2665 | ||||
2666 | Value *rewriteIntegerLoad(LoadInst &LI) { | |||
2667 | assert(IntTy && "We cannot insert an integer to the alloca")(static_cast <bool> (IntTy && "We cannot insert an integer to the alloca" ) ? void (0) : __assert_fail ("IntTy && \"We cannot insert an integer to the alloca\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2667, __extension__ __PRETTY_FUNCTION__ )); | |||
2668 | assert(!LI.isVolatile())(static_cast <bool> (!LI.isVolatile()) ? void (0) : __assert_fail ("!LI.isVolatile()", "llvm/lib/Transforms/Scalar/SROA.cpp", 2668 , __extension__ __PRETTY_FUNCTION__)); | |||
2669 | Value *V = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2670 | NewAI.getAlign(), "load"); | |||
2671 | V = convertValue(DL, IRB, V, IntTy); | |||
2672 | assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset")(static_cast <bool> (NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset") ? void (0) : __assert_fail ("NewBeginOffset >= NewAllocaBeginOffset && \"Out of bounds offset\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2672, __extension__ __PRETTY_FUNCTION__ )); | |||
2673 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
2674 | if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) { | |||
2675 | IntegerType *ExtractTy = Type::getIntNTy(LI.getContext(), SliceSize * 8); | |||
2676 | V = extractInteger(DL, IRB, V, ExtractTy, Offset, "extract"); | |||
2677 | } | |||
2678 | // It is possible that the extracted type is not the load type. This | |||
2679 | // happens if there is a load past the end of the alloca, and as | |||
2680 | // a consequence the slice is narrower but still a candidate for integer | |||
2681 | // lowering. To handle this case, we just zero extend the extracted | |||
2682 | // integer. | |||
2683 | assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&(static_cast <bool> (cast<IntegerType>(LI.getType ())->getBitWidth() >= SliceSize * 8 && "Can only handle an extract for an overly wide load" ) ? void (0) : __assert_fail ("cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && \"Can only handle an extract for an overly wide load\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2684, __extension__ __PRETTY_FUNCTION__ )) | |||
2684 | "Can only handle an extract for an overly wide load")(static_cast <bool> (cast<IntegerType>(LI.getType ())->getBitWidth() >= SliceSize * 8 && "Can only handle an extract for an overly wide load" ) ? void (0) : __assert_fail ("cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && \"Can only handle an extract for an overly wide load\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2684, __extension__ __PRETTY_FUNCTION__ )); | |||
2685 | if (cast<IntegerType>(LI.getType())->getBitWidth() > SliceSize * 8) | |||
2686 | V = IRB.CreateZExt(V, LI.getType()); | |||
2687 | return V; | |||
2688 | } | |||
2689 | ||||
2690 | bool visitLoadInst(LoadInst &LI) { | |||
2691 | LLVM_DEBUG(dbgs() << " original: " << LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << LI << "\n"; } } while (false); | |||
2692 | Value *OldOp = LI.getOperand(0); | |||
2693 | assert(OldOp == OldPtr)(static_cast <bool> (OldOp == OldPtr) ? void (0) : __assert_fail ("OldOp == OldPtr", "llvm/lib/Transforms/Scalar/SROA.cpp", 2693 , __extension__ __PRETTY_FUNCTION__)); | |||
2694 | ||||
2695 | AAMDNodes AATags = LI.getAAMetadata(); | |||
2696 | ||||
2697 | unsigned AS = LI.getPointerAddressSpace(); | |||
2698 | ||||
2699 | Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8) | |||
2700 | : LI.getType(); | |||
2701 | const bool IsLoadPastEnd = | |||
2702 | DL.getTypeStoreSize(TargetTy).getFixedValue() > SliceSize; | |||
2703 | bool IsPtrAdjusted = false; | |||
2704 | Value *V; | |||
2705 | if (VecTy) { | |||
2706 | V = rewriteVectorizedLoadInst(LI); | |||
2707 | } else if (IntTy && LI.getType()->isIntegerTy()) { | |||
2708 | V = rewriteIntegerLoad(LI); | |||
2709 | } else if (NewBeginOffset == NewAllocaBeginOffset && | |||
2710 | NewEndOffset == NewAllocaEndOffset && | |||
2711 | (canConvertValue(DL, NewAllocaTy, TargetTy) || | |||
2712 | (IsLoadPastEnd && NewAllocaTy->isIntegerTy() && | |||
2713 | TargetTy->isIntegerTy()))) { | |||
2714 | Value *NewPtr = | |||
2715 | getPtrToNewAI(LI.getPointerAddressSpace(), LI.isVolatile()); | |||
2716 | LoadInst *NewLI = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), NewPtr, | |||
2717 | NewAI.getAlign(), LI.isVolatile(), | |||
2718 | LI.getName()); | |||
2719 | if (LI.isVolatile()) | |||
2720 | NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); | |||
2721 | if (NewLI->isAtomic()) | |||
2722 | NewLI->setAlignment(LI.getAlign()); | |||
2723 | ||||
2724 | // Copy any metadata that is valid for the new load. This may require | |||
2725 | // conversion to a different kind of metadata, e.g. !nonnull might change | |||
2726 | // to !range or vice versa. | |||
2727 | copyMetadataForLoad(*NewLI, LI); | |||
2728 | ||||
2729 | // Do this after copyMetadataForLoad() to preserve the TBAA shift. | |||
2730 | if (AATags) | |||
2731 | NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2732 | ||||
2733 | // Try to preserve nonnull metadata | |||
2734 | V = NewLI; | |||
2735 | ||||
2736 | // If this is an integer load past the end of the slice (which means the | |||
2737 | // bytes outside the slice are undef or this load is dead) just forcibly | |||
2738 | // fix the integer size with correct handling of endianness. | |||
2739 | if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy)) | |||
2740 | if (auto *TITy = dyn_cast<IntegerType>(TargetTy)) | |||
2741 | if (AITy->getBitWidth() < TITy->getBitWidth()) { | |||
2742 | V = IRB.CreateZExt(V, TITy, "load.ext"); | |||
2743 | if (DL.isBigEndian()) | |||
2744 | V = IRB.CreateShl(V, TITy->getBitWidth() - AITy->getBitWidth(), | |||
2745 | "endian_shift"); | |||
2746 | } | |||
2747 | } else { | |||
2748 | Type *LTy = TargetTy->getPointerTo(AS); | |||
2749 | LoadInst *NewLI = | |||
2750 | IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy), | |||
2751 | getSliceAlign(), LI.isVolatile(), LI.getName()); | |||
2752 | if (AATags) | |||
2753 | NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2754 | if (LI.isVolatile()) | |||
2755 | NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); | |||
2756 | NewLI->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2757 | LLVMContext::MD_access_group}); | |||
2758 | ||||
2759 | V = NewLI; | |||
2760 | IsPtrAdjusted = true; | |||
2761 | } | |||
2762 | V = convertValue(DL, IRB, V, TargetTy); | |||
2763 | ||||
2764 | if (IsSplit) { | |||
2765 | assert(!LI.isVolatile())(static_cast <bool> (!LI.isVolatile()) ? void (0) : __assert_fail ("!LI.isVolatile()", "llvm/lib/Transforms/Scalar/SROA.cpp", 2765 , __extension__ __PRETTY_FUNCTION__)); | |||
2766 | assert(LI.getType()->isIntegerTy() &&(static_cast <bool> (LI.getType()->isIntegerTy() && "Only integer type loads and stores are split") ? void (0) : __assert_fail ("LI.getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2767, __extension__ __PRETTY_FUNCTION__ )) | |||
2767 | "Only integer type loads and stores are split")(static_cast <bool> (LI.getType()->isIntegerTy() && "Only integer type loads and stores are split") ? void (0) : __assert_fail ("LI.getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2767, __extension__ __PRETTY_FUNCTION__ )); | |||
2768 | assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() &&(static_cast <bool> (SliceSize < DL.getTypeStoreSize (LI.getType()).getFixedValue() && "Split load isn't smaller than original load" ) ? void (0) : __assert_fail ("SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() && \"Split load isn't smaller than original load\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2769, __extension__ __PRETTY_FUNCTION__ )) | |||
2769 | "Split load isn't smaller than original load")(static_cast <bool> (SliceSize < DL.getTypeStoreSize (LI.getType()).getFixedValue() && "Split load isn't smaller than original load" ) ? void (0) : __assert_fail ("SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() && \"Split load isn't smaller than original load\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2769, __extension__ __PRETTY_FUNCTION__ )); | |||
2770 | assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&(static_cast <bool> (DL.typeSizeEqualsStoreSize(LI.getType ()) && "Non-byte-multiple bit width") ? void (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(LI.getType()) && \"Non-byte-multiple bit width\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2771, __extension__ __PRETTY_FUNCTION__ )) | |||
2771 | "Non-byte-multiple bit width")(static_cast <bool> (DL.typeSizeEqualsStoreSize(LI.getType ()) && "Non-byte-multiple bit width") ? void (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(LI.getType()) && \"Non-byte-multiple bit width\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2771, __extension__ __PRETTY_FUNCTION__ )); | |||
2772 | // Move the insertion point just past the load so that we can refer to it. | |||
2773 | IRB.SetInsertPoint(&*std::next(BasicBlock::iterator(&LI))); | |||
2774 | // Create a placeholder value with the same type as LI to use as the | |||
2775 | // basis for the new value. This allows us to replace the uses of LI with | |||
2776 | // the computed value, and then replace the placeholder with LI, leaving | |||
2777 | // LI only used for this computation. | |||
2778 | Value *Placeholder = new LoadInst( | |||
2779 | LI.getType(), PoisonValue::get(LI.getType()->getPointerTo(AS)), "", | |||
2780 | false, Align(1)); | |||
2781 | V = insertInteger(DL, IRB, Placeholder, V, NewBeginOffset - BeginOffset, | |||
2782 | "insert"); | |||
2783 | LI.replaceAllUsesWith(V); | |||
2784 | Placeholder->replaceAllUsesWith(&LI); | |||
2785 | Placeholder->deleteValue(); | |||
2786 | } else { | |||
2787 | LI.replaceAllUsesWith(V); | |||
2788 | } | |||
2789 | ||||
2790 | Pass.DeadInsts.push_back(&LI); | |||
2791 | deleteIfTriviallyDead(OldOp); | |||
2792 | LLVM_DEBUG(dbgs() << " to: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *V << "\n"; } } while (false); | |||
2793 | return !LI.isVolatile() && !IsPtrAdjusted; | |||
2794 | } | |||
2795 | ||||
2796 | bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp, | |||
2797 | AAMDNodes AATags) { | |||
2798 | // Capture V for the purpose of debug-info accounting once it's converted | |||
2799 | // to a vector store. | |||
2800 | Value *OrigV = V; | |||
2801 | if (V->getType() != VecTy) { | |||
2802 | unsigned BeginIndex = getIndex(NewBeginOffset); | |||
2803 | unsigned EndIndex = getIndex(NewEndOffset); | |||
2804 | assert(EndIndex > BeginIndex && "Empty vector!")(static_cast <bool> (EndIndex > BeginIndex && "Empty vector!") ? void (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2804, __extension__ __PRETTY_FUNCTION__ )); | |||
2805 | unsigned NumElements = EndIndex - BeginIndex; | |||
2806 | assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&(static_cast <bool> (NumElements <= cast<FixedVectorType >(VecTy)->getNumElements() && "Too many elements!" ) ? void (0) : __assert_fail ("NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2807, __extension__ __PRETTY_FUNCTION__ )) | |||
2807 | "Too many elements!")(static_cast <bool> (NumElements <= cast<FixedVectorType >(VecTy)->getNumElements() && "Too many elements!" ) ? void (0) : __assert_fail ("NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2807, __extension__ __PRETTY_FUNCTION__ )); | |||
2808 | Type *SliceTy = (NumElements == 1) | |||
2809 | ? ElementTy | |||
2810 | : FixedVectorType::get(ElementTy, NumElements); | |||
2811 | if (V->getType() != SliceTy) | |||
2812 | V = convertValue(DL, IRB, V, SliceTy); | |||
2813 | ||||
2814 | // Mix in the existing elements. | |||
2815 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2816 | NewAI.getAlign(), "load"); | |||
2817 | V = insertVector(IRB, Old, V, BeginIndex, "vec"); | |||
2818 | } | |||
2819 | StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign()); | |||
2820 | Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2821 | LLVMContext::MD_access_group}); | |||
2822 | if (AATags) | |||
2823 | Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2824 | Pass.DeadInsts.push_back(&SI); | |||
2825 | ||||
2826 | // NOTE: Careful to use OrigV rather than V. | |||
2827 | migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI, | |||
2828 | Store, Store->getPointerOperand(), OrigV, DL); | |||
2829 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||
2830 | return true; | |||
2831 | } | |||
2832 | ||||
2833 | bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) { | |||
2834 | assert(IntTy && "We cannot extract an integer from the alloca")(static_cast <bool> (IntTy && "We cannot extract an integer from the alloca" ) ? void (0) : __assert_fail ("IntTy && \"We cannot extract an integer from the alloca\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2834, __extension__ __PRETTY_FUNCTION__ )); | |||
2835 | assert(!SI.isVolatile())(static_cast <bool> (!SI.isVolatile()) ? void (0) : __assert_fail ("!SI.isVolatile()", "llvm/lib/Transforms/Scalar/SROA.cpp", 2835 , __extension__ __PRETTY_FUNCTION__)); | |||
2836 | if (DL.getTypeSizeInBits(V->getType()).getFixedValue() != | |||
2837 | IntTy->getBitWidth()) { | |||
2838 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2839 | NewAI.getAlign(), "oldload"); | |||
2840 | Old = convertValue(DL, IRB, Old, IntTy); | |||
2841 | assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset")(static_cast <bool> (BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset") ? void (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"Out of bounds offset\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2841, __extension__ __PRETTY_FUNCTION__ )); | |||
2842 | uint64_t Offset = BeginOffset - NewAllocaBeginOffset; | |||
2843 | V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert"); | |||
2844 | } | |||
2845 | V = convertValue(DL, IRB, V, NewAllocaTy); | |||
2846 | StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign()); | |||
2847 | Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2848 | LLVMContext::MD_access_group}); | |||
2849 | if (AATags) | |||
2850 | Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2851 | ||||
2852 | migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI, | |||
2853 | Store, Store->getPointerOperand(), | |||
2854 | Store->getValueOperand(), DL); | |||
2855 | ||||
2856 | Pass.DeadInsts.push_back(&SI); | |||
2857 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||
2858 | return true; | |||
2859 | } | |||
2860 | ||||
2861 | bool visitStoreInst(StoreInst &SI) { | |||
2862 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||
2863 | Value *OldOp = SI.getOperand(1); | |||
2864 | assert(OldOp == OldPtr)(static_cast <bool> (OldOp == OldPtr) ? void (0) : __assert_fail ("OldOp == OldPtr", "llvm/lib/Transforms/Scalar/SROA.cpp", 2864 , __extension__ __PRETTY_FUNCTION__)); | |||
2865 | ||||
2866 | AAMDNodes AATags = SI.getAAMetadata(); | |||
2867 | Value *V = SI.getValueOperand(); | |||
2868 | ||||
2869 | // Strip all inbounds GEPs and pointer casts to try to dig out any root | |||
2870 | // alloca that should be re-examined after promoting this alloca. | |||
2871 | if (V->getType()->isPointerTy()) | |||
2872 | if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets())) | |||
2873 | Pass.PostPromotionWorklist.insert(AI); | |||
2874 | ||||
2875 | if (SliceSize < DL.getTypeStoreSize(V->getType()).getFixedValue()) { | |||
2876 | assert(!SI.isVolatile())(static_cast <bool> (!SI.isVolatile()) ? void (0) : __assert_fail ("!SI.isVolatile()", "llvm/lib/Transforms/Scalar/SROA.cpp", 2876 , __extension__ __PRETTY_FUNCTION__)); | |||
2877 | assert(V->getType()->isIntegerTy() &&(static_cast <bool> (V->getType()->isIntegerTy() && "Only integer type loads and stores are split") ? void (0) : __assert_fail ("V->getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2878, __extension__ __PRETTY_FUNCTION__ )) | |||
2878 | "Only integer type loads and stores are split")(static_cast <bool> (V->getType()->isIntegerTy() && "Only integer type loads and stores are split") ? void (0) : __assert_fail ("V->getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2878, __extension__ __PRETTY_FUNCTION__ )); | |||
2879 | assert(DL.typeSizeEqualsStoreSize(V->getType()) &&(static_cast <bool> (DL.typeSizeEqualsStoreSize(V->getType ()) && "Non-byte-multiple bit width") ? void (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(V->getType()) && \"Non-byte-multiple bit width\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2880, __extension__ __PRETTY_FUNCTION__ )) | |||
2880 | "Non-byte-multiple bit width")(static_cast <bool> (DL.typeSizeEqualsStoreSize(V->getType ()) && "Non-byte-multiple bit width") ? void (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(V->getType()) && \"Non-byte-multiple bit width\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2880, __extension__ __PRETTY_FUNCTION__ )); | |||
2881 | IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8); | |||
2882 | V = extractInteger(DL, IRB, V, NarrowTy, NewBeginOffset - BeginOffset, | |||
2883 | "extract"); | |||
2884 | } | |||
2885 | ||||
2886 | if (VecTy) | |||
2887 | return rewriteVectorizedStoreInst(V, SI, OldOp, AATags); | |||
2888 | if (IntTy && V->getType()->isIntegerTy()) | |||
2889 | return rewriteIntegerStore(V, SI, AATags); | |||
2890 | ||||
2891 | const bool IsStorePastEnd = | |||
2892 | DL.getTypeStoreSize(V->getType()).getFixedValue() > SliceSize; | |||
2893 | StoreInst *NewSI; | |||
2894 | if (NewBeginOffset == NewAllocaBeginOffset && | |||
2895 | NewEndOffset == NewAllocaEndOffset && | |||
2896 | (canConvertValue(DL, V->getType(), NewAllocaTy) || | |||
2897 | (IsStorePastEnd && NewAllocaTy->isIntegerTy() && | |||
2898 | V->getType()->isIntegerTy()))) { | |||
2899 | // If this is an integer store past the end of slice (and thus the bytes | |||
2900 | // past that point are irrelevant or this is unreachable), truncate the | |||
2901 | // value prior to storing. | |||
2902 | if (auto *VITy = dyn_cast<IntegerType>(V->getType())) | |||
2903 | if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy)) | |||
2904 | if (VITy->getBitWidth() > AITy->getBitWidth()) { | |||
2905 | if (DL.isBigEndian()) | |||
2906 | V = IRB.CreateLShr(V, VITy->getBitWidth() - AITy->getBitWidth(), | |||
2907 | "endian_shift"); | |||
2908 | V = IRB.CreateTrunc(V, AITy, "load.trunc"); | |||
2909 | } | |||
2910 | ||||
2911 | V = convertValue(DL, IRB, V, NewAllocaTy); | |||
2912 | Value *NewPtr = | |||
2913 | getPtrToNewAI(SI.getPointerAddressSpace(), SI.isVolatile()); | |||
2914 | ||||
2915 | NewSI = | |||
2916 | IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), SI.isVolatile()); | |||
2917 | } else { | |||
2918 | unsigned AS = SI.getPointerAddressSpace(); | |||
2919 | Value *NewPtr = getNewAllocaSlicePtr(IRB, V->getType()->getPointerTo(AS)); | |||
2920 | NewSI = | |||
2921 | IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(), SI.isVolatile()); | |||
2922 | } | |||
2923 | NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2924 | LLVMContext::MD_access_group}); | |||
2925 | if (AATags) | |||
2926 | NewSI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2927 | if (SI.isVolatile()) | |||
2928 | NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID()); | |||
2929 | if (NewSI->isAtomic()) | |||
2930 | NewSI->setAlignment(SI.getAlign()); | |||
2931 | ||||
2932 | migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI, | |||
2933 | NewSI, NewSI->getPointerOperand(), | |||
2934 | NewSI->getValueOperand(), DL); | |||
2935 | ||||
2936 | Pass.DeadInsts.push_back(&SI); | |||
2937 | deleteIfTriviallyDead(OldOp); | |||
2938 | ||||
2939 | LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *NewSI << "\n"; } } while (false); | |||
2940 | return NewSI->getPointerOperand() == &NewAI && | |||
2941 | NewSI->getValueOperand()->getType() == NewAllocaTy && | |||
2942 | !SI.isVolatile(); | |||
2943 | } | |||
2944 | ||||
2945 | /// Compute an integer value from splatting an i8 across the given | |||
2946 | /// number of bytes. | |||
2947 | /// | |||
2948 | /// Note that this routine assumes an i8 is a byte. If that isn't true, don't | |||
2949 | /// call this routine. | |||
2950 | /// FIXME: Heed the advice above. | |||
2951 | /// | |||
2952 | /// \param V The i8 value to splat. | |||
2953 | /// \param Size The number of bytes in the output (assuming i8 is one byte) | |||
2954 | Value *getIntegerSplat(Value *V, unsigned Size) { | |||
2955 | assert(Size > 0 && "Expected a positive number of bytes.")(static_cast <bool> (Size > 0 && "Expected a positive number of bytes." ) ? void (0) : __assert_fail ("Size > 0 && \"Expected a positive number of bytes.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2955, __extension__ __PRETTY_FUNCTION__ )); | |||
2956 | IntegerType *VTy = cast<IntegerType>(V->getType()); | |||
2957 | assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte")(static_cast <bool> (VTy->getBitWidth() == 8 && "Expected an i8 value for the byte") ? void (0) : __assert_fail ("VTy->getBitWidth() == 8 && \"Expected an i8 value for the byte\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2957, __extension__ __PRETTY_FUNCTION__ )); | |||
2958 | if (Size == 1) | |||
2959 | return V; | |||
2960 | ||||
2961 | Type *SplatIntTy = Type::getIntNTy(VTy->getContext(), Size * 8); | |||
2962 | V = IRB.CreateMul( | |||
2963 | IRB.CreateZExt(V, SplatIntTy, "zext"), | |||
2964 | IRB.CreateUDiv(Constant::getAllOnesValue(SplatIntTy), | |||
2965 | IRB.CreateZExt(Constant::getAllOnesValue(V->getType()), | |||
2966 | SplatIntTy)), | |||
2967 | "isplat"); | |||
2968 | return V; | |||
2969 | } | |||
2970 | ||||
2971 | /// Compute a vector splat for a given element value. | |||
2972 | Value *getVectorSplat(Value *V, unsigned NumElements) { | |||
2973 | V = IRB.CreateVectorSplat(NumElements, V, "vsplat"); | |||
2974 | LLVM_DEBUG(dbgs() << " splat: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " splat: " << *V << "\n"; } } while (false); | |||
2975 | return V; | |||
2976 | } | |||
2977 | ||||
2978 | bool visitMemSetInst(MemSetInst &II) { | |||
2979 | LLVM_DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | |||
2980 | assert(II.getRawDest() == OldPtr)(static_cast <bool> (II.getRawDest() == OldPtr) ? void ( 0) : __assert_fail ("II.getRawDest() == OldPtr", "llvm/lib/Transforms/Scalar/SROA.cpp" , 2980, __extension__ __PRETTY_FUNCTION__)); | |||
2981 | ||||
2982 | AAMDNodes AATags = II.getAAMetadata(); | |||
2983 | ||||
2984 | // If the memset has a variable size, it cannot be split, just adjust the | |||
2985 | // pointer to the new alloca. | |||
2986 | if (!isa<ConstantInt>(II.getLength())) { | |||
2987 | assert(!IsSplit)(static_cast <bool> (!IsSplit) ? void (0) : __assert_fail ("!IsSplit", "llvm/lib/Transforms/Scalar/SROA.cpp", 2987, __extension__ __PRETTY_FUNCTION__)); | |||
2988 | assert(NewBeginOffset == BeginOffset)(static_cast <bool> (NewBeginOffset == BeginOffset) ? void (0) : __assert_fail ("NewBeginOffset == BeginOffset", "llvm/lib/Transforms/Scalar/SROA.cpp" , 2988, __extension__ __PRETTY_FUNCTION__)); | |||
2989 | II.setDest(getNewAllocaSlicePtr(IRB, OldPtr->getType())); | |||
2990 | II.setDestAlignment(getSliceAlign()); | |||
2991 | // In theory we should call migrateDebugInfo here. However, we do not | |||
2992 | // emit dbg.assign intrinsics for mem intrinsics storing through non- | |||
2993 | // constant geps, or storing a variable number of bytes. | |||
2994 | assert(at::getAssignmentMarkers(&II).empty() &&(static_cast <bool> (at::getAssignmentMarkers(&II). empty() && "AT: Unexpected link to non-const GEP") ? void (0) : __assert_fail ("at::getAssignmentMarkers(&II).empty() && \"AT: Unexpected link to non-const GEP\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2995, __extension__ __PRETTY_FUNCTION__ )) | |||
2995 | "AT: Unexpected link to non-const GEP")(static_cast <bool> (at::getAssignmentMarkers(&II). empty() && "AT: Unexpected link to non-const GEP") ? void (0) : __assert_fail ("at::getAssignmentMarkers(&II).empty() && \"AT: Unexpected link to non-const GEP\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 2995, __extension__ __PRETTY_FUNCTION__ )); | |||
2996 | deleteIfTriviallyDead(OldPtr); | |||
2997 | return false; | |||
2998 | } | |||
2999 | ||||
3000 | // Record this instruction for deletion. | |||
3001 | Pass.DeadInsts.push_back(&II); | |||
3002 | ||||
3003 | Type *AllocaTy = NewAI.getAllocatedType(); | |||
3004 | Type *ScalarTy = AllocaTy->getScalarType(); | |||
3005 | ||||
3006 | const bool CanContinue = [&]() { | |||
3007 | if (VecTy || IntTy) | |||
3008 | return true; | |||
3009 | if (BeginOffset > NewAllocaBeginOffset || | |||
3010 | EndOffset < NewAllocaEndOffset) | |||
3011 | return false; | |||
3012 | // Length must be in range for FixedVectorType. | |||
3013 | auto *C = cast<ConstantInt>(II.getLength()); | |||
3014 | const uint64_t Len = C->getLimitedValue(); | |||
3015 | if (Len > std::numeric_limits<unsigned>::max()) | |||
3016 | return false; | |||
3017 | auto *Int8Ty = IntegerType::getInt8Ty(NewAI.getContext()); | |||
3018 | auto *SrcTy = FixedVectorType::get(Int8Ty, Len); | |||
3019 | return canConvertValue(DL, SrcTy, AllocaTy) && | |||
3020 | DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy).getFixedValue()); | |||
3021 | }(); | |||
3022 | ||||
3023 | // If this doesn't map cleanly onto the alloca type, and that type isn't | |||
3024 | // a single value type, just emit a memset. | |||
3025 | if (!CanContinue) { | |||
3026 | Type *SizeTy = II.getLength()->getType(); | |||
3027 | Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset); | |||
3028 | MemIntrinsic *New = cast<MemIntrinsic>(IRB.CreateMemSet( | |||
3029 | getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size, | |||
3030 | MaybeAlign(getSliceAlign()), II.isVolatile())); | |||
3031 | if (AATags) | |||
3032 | New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
3033 | ||||
3034 | migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II, | |||
3035 | New, New->getRawDest(), nullptr, DL); | |||
3036 | ||||
3037 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||
3038 | return false; | |||
3039 | } | |||
3040 | ||||
3041 | // If we can represent this as a simple value, we have to build the actual | |||
3042 | // value to store, which requires expanding the byte present in memset to | |||
3043 | // a sensible representation for the alloca type. This is essentially | |||
3044 | // splatting the byte to a sufficiently wide integer, splatting it across | |||
3045 | // any desired vector width, and bitcasting to the final type. | |||
3046 | Value *V; | |||
3047 | ||||
3048 | if (VecTy) { | |||
3049 | // If this is a memset of a vectorized alloca, insert it. | |||
3050 | assert(ElementTy == ScalarTy)(static_cast <bool> (ElementTy == ScalarTy) ? void (0) : __assert_fail ("ElementTy == ScalarTy", "llvm/lib/Transforms/Scalar/SROA.cpp" , 3050, __extension__ __PRETTY_FUNCTION__)); | |||
3051 | ||||
3052 | unsigned BeginIndex = getIndex(NewBeginOffset); | |||
3053 | unsigned EndIndex = getIndex(NewEndOffset); | |||
3054 | assert(EndIndex > BeginIndex && "Empty vector!")(static_cast <bool> (EndIndex > BeginIndex && "Empty vector!") ? void (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3054, __extension__ __PRETTY_FUNCTION__ )); | |||
3055 | unsigned NumElements = EndIndex - BeginIndex; | |||
3056 | assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&(static_cast <bool> (NumElements <= cast<FixedVectorType >(VecTy)->getNumElements() && "Too many elements!" ) ? void (0) : __assert_fail ("NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3057, __extension__ __PRETTY_FUNCTION__ )) | |||
3057 | "Too many elements!")(static_cast <bool> (NumElements <= cast<FixedVectorType >(VecTy)->getNumElements() && "Too many elements!" ) ? void (0) : __assert_fail ("NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3057, __extension__ __PRETTY_FUNCTION__ )); | |||
3058 | ||||
3059 | Value *Splat = getIntegerSplat( | |||
3060 | II.getValue(), DL.getTypeSizeInBits(ElementTy).getFixedValue() / 8); | |||
3061 | Splat = convertValue(DL, IRB, Splat, ElementTy); | |||
3062 | if (NumElements > 1) | |||
3063 | Splat = getVectorSplat(Splat, NumElements); | |||
3064 | ||||
3065 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3066 | NewAI.getAlign(), "oldload"); | |||
3067 | V = insertVector(IRB, Old, Splat, BeginIndex, "vec"); | |||
3068 | } else if (IntTy) { | |||
3069 | // If this is a memset on an alloca where we can widen stores, insert the | |||
3070 | // set integer. | |||
3071 | assert(!II.isVolatile())(static_cast <bool> (!II.isVolatile()) ? void (0) : __assert_fail ("!II.isVolatile()", "llvm/lib/Transforms/Scalar/SROA.cpp", 3071 , __extension__ __PRETTY_FUNCTION__)); | |||
3072 | ||||
3073 | uint64_t Size = NewEndOffset - NewBeginOffset; | |||
3074 | V = getIntegerSplat(II.getValue(), Size); | |||
3075 | ||||
3076 | if (IntTy && (BeginOffset != NewAllocaBeginOffset || | |||
3077 | EndOffset != NewAllocaBeginOffset)) { | |||
3078 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3079 | NewAI.getAlign(), "oldload"); | |||
3080 | Old = convertValue(DL, IRB, Old, IntTy); | |||
3081 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
3082 | V = insertInteger(DL, IRB, Old, V, Offset, "insert"); | |||
3083 | } else { | |||
3084 | assert(V->getType() == IntTy &&(static_cast <bool> (V->getType() == IntTy && "Wrong type for an alloca wide integer!") ? void (0) : __assert_fail ("V->getType() == IntTy && \"Wrong type for an alloca wide integer!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3085, __extension__ __PRETTY_FUNCTION__ )) | |||
3085 | "Wrong type for an alloca wide integer!")(static_cast <bool> (V->getType() == IntTy && "Wrong type for an alloca wide integer!") ? void (0) : __assert_fail ("V->getType() == IntTy && \"Wrong type for an alloca wide integer!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3085, __extension__ __PRETTY_FUNCTION__ )); | |||
3086 | } | |||
3087 | V = convertValue(DL, IRB, V, AllocaTy); | |||
3088 | } else { | |||
3089 | // Established these invariants above. | |||
3090 | assert(NewBeginOffset == NewAllocaBeginOffset)(static_cast <bool> (NewBeginOffset == NewAllocaBeginOffset ) ? void (0) : __assert_fail ("NewBeginOffset == NewAllocaBeginOffset" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3090, __extension__ __PRETTY_FUNCTION__ )); | |||
3091 | assert(NewEndOffset == NewAllocaEndOffset)(static_cast <bool> (NewEndOffset == NewAllocaEndOffset ) ? void (0) : __assert_fail ("NewEndOffset == NewAllocaEndOffset" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3091, __extension__ __PRETTY_FUNCTION__ )); | |||
3092 | ||||
3093 | V = getIntegerSplat(II.getValue(), | |||
3094 | DL.getTypeSizeInBits(ScalarTy).getFixedValue() / 8); | |||
3095 | if (VectorType *AllocaVecTy = dyn_cast<VectorType>(AllocaTy)) | |||
3096 | V = getVectorSplat( | |||
3097 | V, cast<FixedVectorType>(AllocaVecTy)->getNumElements()); | |||
3098 | ||||
3099 | V = convertValue(DL, IRB, V, AllocaTy); | |||
3100 | } | |||
3101 | ||||
3102 | Value *NewPtr = getPtrToNewAI(II.getDestAddressSpace(), II.isVolatile()); | |||
3103 | StoreInst *New = | |||
3104 | IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), II.isVolatile()); | |||
3105 | New->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access, | |||
3106 | LLVMContext::MD_access_group}); | |||
3107 | if (AATags) | |||
3108 | New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
3109 | ||||
3110 | migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II, | |||
3111 | New, New->getPointerOperand(), V, DL); | |||
3112 | ||||
3113 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||
3114 | return !II.isVolatile(); | |||
3115 | } | |||
3116 | ||||
3117 | bool visitMemTransferInst(MemTransferInst &II) { | |||
3118 | // Rewriting of memory transfer instructions can be a bit tricky. We break | |||
3119 | // them into two categories: split intrinsics and unsplit intrinsics. | |||
3120 | ||||
3121 | LLVM_DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | |||
| ||||
3122 | ||||
3123 | AAMDNodes AATags = II.getAAMetadata(); | |||
3124 | ||||
3125 | bool IsDest = &II.getRawDestUse() == OldUse; | |||
3126 | assert((IsDest && II.getRawDest() == OldPtr) ||(static_cast <bool> ((IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr )) ? void (0) : __assert_fail ("(IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3127, __extension__ __PRETTY_FUNCTION__ )) | |||
3127 | (!IsDest && II.getRawSource() == OldPtr))(static_cast <bool> ((IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr )) ? void (0) : __assert_fail ("(IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3127, __extension__ __PRETTY_FUNCTION__ )); | |||
3128 | ||||
3129 | Align SliceAlign = getSliceAlign(); | |||
3130 | // For unsplit intrinsics, we simply modify the source and destination | |||
3131 | // pointers in place. This isn't just an optimization, it is a matter of | |||
3132 | // correctness. With unsplit intrinsics we may be dealing with transfers | |||
3133 | // within a single alloca before SROA ran, or with transfers that have | |||
3134 | // a variable length. We may also be dealing with memmove instead of | |||
3135 | // memcpy, and so simply updating the pointers is the necessary for us to | |||
3136 | // update both source and dest of a single call. | |||
3137 | if (!IsSplittable) { | |||
3138 | Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||
| ||||
3139 | if (IsDest) { | |||
3140 | // Update the address component of linked dbg.assigns. | |||
3141 | for (auto *DAI : at::getAssignmentMarkers(&II)) { | |||
3142 | if (any_of(DAI->location_ops(), | |||
3143 | [&](Value *V) { return V == II.getDest(); }) || | |||
3144 | DAI->getAddress() == II.getDest()) | |||
3145 | DAI->replaceVariableLocationOp(II.getDest(), AdjustedPtr); | |||
3146 | } | |||
3147 | II.setDest(AdjustedPtr); | |||
3148 | II.setDestAlignment(SliceAlign); | |||
3149 | } else { | |||
3150 | II.setSource(AdjustedPtr); | |||
3151 | II.setSourceAlignment(SliceAlign); | |||
3152 | } | |||
3153 | ||||
3154 | LLVM_DEBUG(dbgs() << " to: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << II << "\n"; } } while (false); | |||
3155 | deleteIfTriviallyDead(OldPtr); | |||
3156 | return false; | |||
3157 | } | |||
3158 | // For split transfer intrinsics we have an incredibly useful assurance: | |||
3159 | // the source and destination do not reside within the same alloca, and at | |||
3160 | // least one of them does not escape. This means that we can replace | |||
3161 | // memmove with memcpy, and we don't need to worry about all manner of | |||
3162 | // downsides to splitting and transforming the operations. | |||
3163 | ||||
3164 | // If this doesn't map cleanly onto the alloca type, and that type isn't | |||
3165 | // a single value type, just emit a memcpy. | |||
3166 | bool EmitMemCpy = | |||
3167 | !VecTy && !IntTy && | |||
3168 | (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset || | |||
3169 | SliceSize != | |||
3170 | DL.getTypeStoreSize(NewAI.getAllocatedType()).getFixedValue() || | |||
3171 | !NewAI.getAllocatedType()->isSingleValueType()); | |||
3172 | ||||
3173 | // If we're just going to emit a memcpy, the alloca hasn't changed, and the | |||
3174 | // size hasn't been shrunk based on analysis of the viable range, this is | |||
3175 | // a no-op. | |||
3176 | if (EmitMemCpy && &OldAI == &NewAI) { | |||
3177 | // Ensure the start lines up. | |||
3178 | assert(NewBeginOffset == BeginOffset)(static_cast <bool> (NewBeginOffset == BeginOffset) ? void (0) : __assert_fail ("NewBeginOffset == BeginOffset", "llvm/lib/Transforms/Scalar/SROA.cpp" , 3178, __extension__ __PRETTY_FUNCTION__)); | |||
3179 | ||||
3180 | // Rewrite the size as needed. | |||
3181 | if (NewEndOffset != EndOffset) | |||
3182 | II.setLength(ConstantInt::get(II.getLength()->getType(), | |||
3183 | NewEndOffset - NewBeginOffset)); | |||
3184 | return false; | |||
3185 | } | |||
3186 | // Record this instruction for deletion. | |||
3187 | Pass.DeadInsts.push_back(&II); | |||
3188 | ||||
3189 | // Strip all inbounds GEPs and pointer casts to try to dig out any root | |||
3190 | // alloca that should be re-examined after rewriting this instruction. | |||
3191 | Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest(); | |||
3192 | if (AllocaInst *AI = | |||
3193 | dyn_cast<AllocaInst>(OtherPtr->stripInBoundsOffsets())) { | |||
3194 | assert(AI != &OldAI && AI != &NewAI &&(static_cast <bool> (AI != &OldAI && AI != & NewAI && "Splittable transfers cannot reach the same alloca on both ends." ) ? void (0) : __assert_fail ("AI != &OldAI && AI != &NewAI && \"Splittable transfers cannot reach the same alloca on both ends.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3195, __extension__ __PRETTY_FUNCTION__ )) | |||
3195 | "Splittable transfers cannot reach the same alloca on both ends.")(static_cast <bool> (AI != &OldAI && AI != & NewAI && "Splittable transfers cannot reach the same alloca on both ends." ) ? void (0) : __assert_fail ("AI != &OldAI && AI != &NewAI && \"Splittable transfers cannot reach the same alloca on both ends.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3195, __extension__ __PRETTY_FUNCTION__ )); | |||
3196 | Pass.Worklist.insert(AI); | |||
3197 | } | |||
3198 | ||||
3199 | Type *OtherPtrTy = OtherPtr->getType(); | |||
3200 | unsigned OtherAS = OtherPtrTy->getPointerAddressSpace(); | |||
3201 | ||||
3202 | // Compute the relative offset for the other pointer within the transfer. | |||
3203 | unsigned OffsetWidth = DL.getIndexSizeInBits(OtherAS); | |||
3204 | APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset); | |||
3205 | Align OtherAlign = | |||
3206 | (IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne(); | |||
3207 | OtherAlign = | |||
3208 | commonAlignment(OtherAlign, OtherOffset.zextOrTrunc(64).getZExtValue()); | |||
3209 | ||||
3210 | if (EmitMemCpy) { | |||
3211 | // Compute the other pointer, folding as much as possible to produce | |||
3212 | // a single, simple GEP in most cases. | |||
3213 | OtherPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy, | |||
3214 | OtherPtr->getName() + "."); | |||
3215 | ||||
3216 | Value *OurPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||
3217 | Type *SizeTy = II.getLength()->getType(); | |||
3218 | Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset); | |||
3219 | ||||
3220 | Value *DestPtr, *SrcPtr; | |||
3221 | MaybeAlign DestAlign, SrcAlign; | |||
3222 | // Note: IsDest is true iff we're copying into the new alloca slice | |||
3223 | if (IsDest) { | |||
3224 | DestPtr = OurPtr; | |||
3225 | DestAlign = SliceAlign; | |||
3226 | SrcPtr = OtherPtr; | |||
3227 | SrcAlign = OtherAlign; | |||
3228 | } else { | |||
3229 | DestPtr = OtherPtr; | |||
3230 | DestAlign = OtherAlign; | |||
3231 | SrcPtr = OurPtr; | |||
3232 | SrcAlign = SliceAlign; | |||
3233 | } | |||
3234 | CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign, | |||
3235 | Size, II.isVolatile()); | |||
3236 | if (AATags) | |||
3237 | New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
3238 | ||||
3239 | APInt Offset(DL.getIndexTypeSizeInBits(DestPtr->getType()), 0); | |||
3240 | if (IsDest) { | |||
3241 | migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, | |||
3242 | &II, New, DestPtr, nullptr, DL); | |||
3243 | } else if (AllocaInst *Base = dyn_cast<AllocaInst>( | |||
3244 | DestPtr->stripAndAccumulateConstantOffsets( | |||
3245 | DL, Offset, /*AllowNonInbounds*/ true))) { | |||
3246 | migrateDebugInfo(Base, IsSplit, Offset.getZExtValue() * 8, | |||
3247 | SliceSize * 8, &II, New, DestPtr, nullptr, DL); | |||
3248 | } | |||
3249 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||
3250 | return false; | |||
3251 | } | |||
3252 | ||||
3253 | bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset && | |||
3254 | NewEndOffset == NewAllocaEndOffset; | |||
3255 | uint64_t Size = NewEndOffset - NewBeginOffset; | |||
3256 | unsigned BeginIndex = VecTy ? getIndex(NewBeginOffset) : 0; | |||
3257 | unsigned EndIndex = VecTy ? getIndex(NewEndOffset) : 0; | |||
3258 | unsigned NumElements = EndIndex - BeginIndex; | |||
3259 | IntegerType *SubIntTy = | |||
3260 | IntTy ? Type::getIntNTy(IntTy->getContext(), Size * 8) : nullptr; | |||
3261 | ||||
3262 | // Reset the other pointer type to match the register type we're going to | |||
3263 | // use, but using the address space of the original other pointer. | |||
3264 | Type *OtherTy; | |||
3265 | if (VecTy && !IsWholeAlloca) { | |||
3266 | if (NumElements == 1) | |||
3267 | OtherTy = VecTy->getElementType(); | |||
3268 | else | |||
3269 | OtherTy = FixedVectorType::get(VecTy->getElementType(), NumElements); | |||
3270 | } else if (IntTy && !IsWholeAlloca) { | |||
3271 | OtherTy = SubIntTy; | |||
3272 | } else { | |||
3273 | OtherTy = NewAllocaTy; | |||
3274 | } | |||
3275 | OtherPtrTy = OtherTy->getPointerTo(OtherAS); | |||
3276 | ||||
3277 | Value *AdjPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy, | |||
3278 | OtherPtr->getName() + "."); | |||
3279 | MaybeAlign SrcAlign = OtherAlign; | |||
3280 | MaybeAlign DstAlign = SliceAlign; | |||
3281 | if (!IsDest) | |||
3282 | std::swap(SrcAlign, DstAlign); | |||
3283 | ||||
3284 | Value *SrcPtr; | |||
3285 | Value *DstPtr; | |||
3286 | ||||
3287 | if (IsDest) { | |||
3288 | DstPtr = getPtrToNewAI(II.getDestAddressSpace(), II.isVolatile()); | |||
3289 | SrcPtr = AdjPtr; | |||
3290 | } else { | |||
3291 | DstPtr = AdjPtr; | |||
3292 | SrcPtr = getPtrToNewAI(II.getSourceAddressSpace(), II.isVolatile()); | |||
3293 | } | |||
3294 | ||||
3295 | Value *Src; | |||
3296 | if (VecTy && !IsWholeAlloca && !IsDest) { | |||
3297 | Src = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3298 | NewAI.getAlign(), "load"); | |||
3299 | Src = extractVector(IRB, Src, BeginIndex, EndIndex, "vec"); | |||
3300 | } else if (IntTy && !IsWholeAlloca && !IsDest) { | |||
3301 | Src = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3302 | NewAI.getAlign(), "load"); | |||
3303 | Src = convertValue(DL, IRB, Src, IntTy); | |||
3304 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
3305 | Src = extractInteger(DL, IRB, Src, SubIntTy, Offset, "extract"); | |||
3306 | } else { | |||
3307 | LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign, | |||
3308 | II.isVolatile(), "copyload"); | |||
3309 | Load->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access, | |||
3310 | LLVMContext::MD_access_group}); | |||
3311 | if (AATags) | |||
3312 | Load->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
3313 | Src = Load; | |||
3314 | } | |||
3315 | ||||
3316 | if (VecTy && !IsWholeAlloca && IsDest) { | |||
3317 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3318 | NewAI.getAlign(), "oldload"); | |||
3319 | Src = insertVector(IRB, Old, Src, BeginIndex, "vec"); | |||
3320 | } else if (IntTy && !IsWholeAlloca && IsDest) { | |||
3321 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3322 | NewAI.getAlign(), "oldload"); | |||
3323 | Old = convertValue(DL, IRB, Old, IntTy); | |||
3324 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
3325 | Src = insertInteger(DL, IRB, Old, Src, Offset, "insert"); | |||
3326 | Src = convertValue(DL, IRB, Src, NewAllocaTy); | |||
3327 | } | |||
3328 | ||||
3329 | StoreInst *Store = cast<StoreInst>( | |||
3330 | IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile())); | |||
3331 | Store->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access, | |||
3332 | LLVMContext::MD_access_group}); | |||
3333 | if (AATags) | |||
3334 | Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
3335 | ||||
3336 | APInt Offset(DL.getIndexTypeSizeInBits(DstPtr->getType()), 0); | |||
3337 | if (IsDest) { | |||
3338 | ||||
3339 | migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II, | |||
3340 | Store, DstPtr, Src, DL); | |||
3341 | } else if (AllocaInst *Base = dyn_cast<AllocaInst>( | |||
3342 | DstPtr->stripAndAccumulateConstantOffsets( | |||
3343 | DL, Offset, /*AllowNonInbounds*/ true))) { | |||
3344 | migrateDebugInfo(Base, IsSplit, Offset.getZExtValue() * 8, SliceSize * 8, | |||
3345 | &II, Store, DstPtr, Src, DL); | |||
3346 | } | |||
3347 | ||||
3348 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||
3349 | return !II.isVolatile(); | |||
3350 | } | |||
3351 | ||||
3352 | bool visitIntrinsicInst(IntrinsicInst &II) { | |||
3353 | assert((II.isLifetimeStartOrEnd() || II.isDroppable()) &&(static_cast <bool> ((II.isLifetimeStartOrEnd() || II.isDroppable ()) && "Unexpected intrinsic!") ? void (0) : __assert_fail ("(II.isLifetimeStartOrEnd() || II.isDroppable()) && \"Unexpected intrinsic!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3354, __extension__ __PRETTY_FUNCTION__ )) | |||
3354 | "Unexpected intrinsic!")(static_cast <bool> ((II.isLifetimeStartOrEnd() || II.isDroppable ()) && "Unexpected intrinsic!") ? void (0) : __assert_fail ("(II.isLifetimeStartOrEnd() || II.isDroppable()) && \"Unexpected intrinsic!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3354, __extension__ __PRETTY_FUNCTION__ )); | |||
3355 | LLVM_DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | |||
3356 | ||||
3357 | // Record this instruction for deletion. | |||
3358 | Pass.DeadInsts.push_back(&II); | |||
3359 | ||||
3360 | if (II.isDroppable()) { | |||
3361 | assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume")(static_cast <bool> (II.getIntrinsicID() == Intrinsic:: assume && "Expected assume") ? void (0) : __assert_fail ("II.getIntrinsicID() == Intrinsic::assume && \"Expected assume\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3361, __extension__ __PRETTY_FUNCTION__ )); | |||
3362 | // TODO For now we forget assumed information, this can be improved. | |||
3363 | OldPtr->dropDroppableUsesIn(II); | |||
3364 | return true; | |||
3365 | } | |||
3366 | ||||
3367 | assert(II.getArgOperand(1) == OldPtr)(static_cast <bool> (II.getArgOperand(1) == OldPtr) ? void (0) : __assert_fail ("II.getArgOperand(1) == OldPtr", "llvm/lib/Transforms/Scalar/SROA.cpp" , 3367, __extension__ __PRETTY_FUNCTION__)); | |||
3368 | // Lifetime intrinsics are only promotable if they cover the whole alloca. | |||
3369 | // Therefore, we drop lifetime intrinsics which don't cover the whole | |||
3370 | // alloca. | |||
3371 | // (In theory, intrinsics which partially cover an alloca could be | |||
3372 | // promoted, but PromoteMemToReg doesn't handle that case.) | |||
3373 | // FIXME: Check whether the alloca is promotable before dropping the | |||
3374 | // lifetime intrinsics? | |||
3375 | if (NewBeginOffset != NewAllocaBeginOffset || | |||
3376 | NewEndOffset != NewAllocaEndOffset) | |||
3377 | return true; | |||
3378 | ||||
3379 | ConstantInt *Size = | |||
3380 | ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()), | |||
3381 | NewEndOffset - NewBeginOffset); | |||
3382 | // Lifetime intrinsics always expect an i8* so directly get such a pointer | |||
3383 | // for the new alloca slice. | |||
3384 | Type *PointerTy = IRB.getInt8PtrTy(OldPtr->getType()->getPointerAddressSpace()); | |||
3385 | Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy); | |||
3386 | Value *New; | |||
3387 | if (II.getIntrinsicID() == Intrinsic::lifetime_start) | |||
3388 | New = IRB.CreateLifetimeStart(Ptr, Size); | |||
3389 | else | |||
3390 | New = IRB.CreateLifetimeEnd(Ptr, Size); | |||
3391 | ||||
3392 | (void)New; | |||
3393 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||
3394 | ||||
3395 | return true; | |||
3396 | } | |||
3397 | ||||
3398 | void fixLoadStoreAlign(Instruction &Root) { | |||
3399 | // This algorithm implements the same visitor loop as | |||
3400 | // hasUnsafePHIOrSelectUse, and fixes the alignment of each load | |||
3401 | // or store found. | |||
3402 | SmallPtrSet<Instruction *, 4> Visited; | |||
3403 | SmallVector<Instruction *, 4> Uses; | |||
3404 | Visited.insert(&Root); | |||
3405 | Uses.push_back(&Root); | |||
3406 | do { | |||
3407 | Instruction *I = Uses.pop_back_val(); | |||
3408 | ||||
3409 | if (LoadInst *LI = dyn_cast<LoadInst>(I)) { | |||
3410 | LI->setAlignment(std::min(LI->getAlign(), getSliceAlign())); | |||
3411 | continue; | |||
3412 | } | |||
3413 | if (StoreInst *SI = dyn_cast<StoreInst>(I)) { | |||
3414 | SI->setAlignment(std::min(SI->getAlign(), getSliceAlign())); | |||
3415 | continue; | |||
3416 | } | |||
3417 | ||||
3418 | assert(isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) ||(static_cast <bool> (isa<BitCastInst>(I) || isa< AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst >(I) || isa<GetElementPtrInst>(I)) ? void (0) : __assert_fail ("isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst>(I) || isa<GetElementPtrInst>(I)" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3420, __extension__ __PRETTY_FUNCTION__ )) | |||
3419 | isa<PHINode>(I) || isa<SelectInst>(I) ||(static_cast <bool> (isa<BitCastInst>(I) || isa< AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst >(I) || isa<GetElementPtrInst>(I)) ? void (0) : __assert_fail ("isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst>(I) || isa<GetElementPtrInst>(I)" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3420, __extension__ __PRETTY_FUNCTION__ )) | |||
3420 | isa<GetElementPtrInst>(I))(static_cast <bool> (isa<BitCastInst>(I) || isa< AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst >(I) || isa<GetElementPtrInst>(I)) ? void (0) : __assert_fail ("isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst>(I) || isa<GetElementPtrInst>(I)" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3420, __extension__ __PRETTY_FUNCTION__ )); | |||
3421 | for (User *U : I->users()) | |||
3422 | if (Visited.insert(cast<Instruction>(U)).second) | |||
3423 | Uses.push_back(cast<Instruction>(U)); | |||
3424 | } while (!Uses.empty()); | |||
3425 | } | |||
3426 | ||||
3427 | bool visitPHINode(PHINode &PN) { | |||
3428 | LLVM_DEBUG(dbgs() << " original: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << PN << "\n"; } } while (false); | |||
3429 | assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable")(static_cast <bool> (BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable") ? void (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"PHIs are unsplittable\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3429, __extension__ __PRETTY_FUNCTION__ )); | |||
3430 | assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable")(static_cast <bool> (EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable") ? void (0) : __assert_fail ("EndOffset <= NewAllocaEndOffset && \"PHIs are unsplittable\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3430, __extension__ __PRETTY_FUNCTION__ )); | |||
3431 | ||||
3432 | // We would like to compute a new pointer in only one place, but have it be | |||
3433 | // as local as possible to the PHI. To do that, we re-use the location of | |||
3434 | // the old pointer, which necessarily must be in the right position to | |||
3435 | // dominate the PHI. | |||
3436 | IRBuilderBase::InsertPointGuard Guard(IRB); | |||
3437 | if (isa<PHINode>(OldPtr)) | |||
3438 | IRB.SetInsertPoint(&*OldPtr->getParent()->getFirstInsertionPt()); | |||
3439 | else | |||
3440 | IRB.SetInsertPoint(OldPtr); | |||
3441 | IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc()); | |||
3442 | ||||
3443 | Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||
3444 | // Replace the operands which were using the old pointer. | |||
3445 | std::replace(PN.op_begin(), PN.op_end(), cast<Value>(OldPtr), NewPtr); | |||
3446 | ||||
3447 | LLVM_DEBUG(dbgs() << " to: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << PN << "\n"; } } while (false); | |||
3448 | deleteIfTriviallyDead(OldPtr); | |||
3449 | ||||
3450 | // Fix the alignment of any loads or stores using this PHI node. | |||
3451 | fixLoadStoreAlign(PN); | |||
3452 | ||||
3453 | // PHIs can't be promoted on their own, but often can be speculated. We | |||
3454 | // check the speculation outside of the rewriter so that we see the | |||
3455 | // fully-rewritten alloca. | |||
3456 | PHIUsers.insert(&PN); | |||
3457 | return true; | |||
3458 | } | |||
3459 | ||||
3460 | bool visitSelectInst(SelectInst &SI) { | |||
3461 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||
3462 | assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&(static_cast <bool> ((SI.getTrueValue() == OldPtr || SI .getFalseValue() == OldPtr) && "Pointer isn't an operand!" ) ? void (0) : __assert_fail ("(SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && \"Pointer isn't an operand!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3463, __extension__ __PRETTY_FUNCTION__ )) | |||
3463 | "Pointer isn't an operand!")(static_cast <bool> ((SI.getTrueValue() == OldPtr || SI .getFalseValue() == OldPtr) && "Pointer isn't an operand!" ) ? void (0) : __assert_fail ("(SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && \"Pointer isn't an operand!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3463, __extension__ __PRETTY_FUNCTION__ )); | |||
3464 | assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable")(static_cast <bool> (BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable") ? void (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"Selects are unsplittable\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3464, __extension__ __PRETTY_FUNCTION__ )); | |||
3465 | assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable")(static_cast <bool> (EndOffset <= NewAllocaEndOffset && "Selects are unsplittable") ? void (0) : __assert_fail ("EndOffset <= NewAllocaEndOffset && \"Selects are unsplittable\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3465, __extension__ __PRETTY_FUNCTION__ )); | |||
3466 | ||||
3467 | Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||
3468 | // Replace the operands which were using the old pointer. | |||
3469 | if (SI.getOperand(1) == OldPtr) | |||
3470 | SI.setOperand(1, NewPtr); | |||
3471 | if (SI.getOperand(2) == OldPtr) | |||
3472 | SI.setOperand(2, NewPtr); | |||
3473 | ||||
3474 | LLVM_DEBUG(dbgs() << " to: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << SI << "\n"; } } while (false); | |||
3475 | deleteIfTriviallyDead(OldPtr); | |||
3476 | ||||
3477 | // Fix the alignment of any loads or stores using this select. | |||
3478 | fixLoadStoreAlign(SI); | |||
3479 | ||||
3480 | // Selects can't be promoted on their own, but often can be speculated. We | |||
3481 | // check the speculation outside of the rewriter so that we see the | |||
3482 | // fully-rewritten alloca. | |||
3483 | SelectUsers.insert(&SI); | |||
3484 | return true; | |||
3485 | } | |||
3486 | }; | |||
3487 | ||||
3488 | namespace { | |||
3489 | ||||
3490 | /// Visitor to rewrite aggregate loads and stores as scalar. | |||
3491 | /// | |||
3492 | /// This pass aggressively rewrites all aggregate loads and stores on | |||
3493 | /// a particular pointer (or any pointer derived from it which we can identify) | |||
3494 | /// with scalar loads and stores. | |||
3495 | class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> { | |||
3496 | // Befriend the base class so it can delegate to private visit methods. | |||
3497 | friend class InstVisitor<AggLoadStoreRewriter, bool>; | |||
3498 | ||||
3499 | /// Queue of pointer uses to analyze and potentially rewrite. | |||
3500 | SmallVector<Use *, 8> Queue; | |||
3501 | ||||
3502 | /// Set to prevent us from cycling with phi nodes and loops. | |||
3503 | SmallPtrSet<User *, 8> Visited; | |||
3504 | ||||
3505 | /// The current pointer use being rewritten. This is used to dig up the used | |||
3506 | /// value (as opposed to the user). | |||
3507 | Use *U = nullptr; | |||
3508 | ||||
3509 | /// Used to calculate offsets, and hence alignment, of subobjects. | |||
3510 | const DataLayout &DL; | |||
3511 | ||||
3512 | IRBuilderTy &IRB; | |||
3513 | ||||
3514 | public: | |||
3515 | AggLoadStoreRewriter(const DataLayout &DL, IRBuilderTy &IRB) | |||
3516 | : DL(DL), IRB(IRB) {} | |||
3517 | ||||
3518 | /// Rewrite loads and stores through a pointer and all pointers derived from | |||
3519 | /// it. | |||
3520 | bool rewrite(Instruction &I) { | |||
3521 | LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting FCA loads and stores...\n" ; } } while (false); | |||
3522 | enqueueUsers(I); | |||
3523 | bool Changed = false; | |||
3524 | while (!Queue.empty()) { | |||
3525 | U = Queue.pop_back_val(); | |||
3526 | Changed |= visit(cast<Instruction>(U->getUser())); | |||
3527 | } | |||
3528 | return Changed; | |||
3529 | } | |||
3530 | ||||
3531 | private: | |||
3532 | /// Enqueue all the users of the given instruction for further processing. | |||
3533 | /// This uses a set to de-duplicate users. | |||
3534 | void enqueueUsers(Instruction &I) { | |||
3535 | for (Use &U : I.uses()) | |||
3536 | if (Visited.insert(U.getUser()).second) | |||
3537 | Queue.push_back(&U); | |||
3538 | } | |||
3539 | ||||
3540 | // Conservative default is to not rewrite anything. | |||
3541 | bool visitInstruction(Instruction &I) { return false; } | |||
3542 | ||||
3543 | /// Generic recursive split emission class. | |||
3544 | template <typename Derived> class OpSplitter { | |||
3545 | protected: | |||
3546 | /// The builder used to form new instructions. | |||
3547 | IRBuilderTy &IRB; | |||
3548 | ||||
3549 | /// The indices which to be used with insert- or extractvalue to select the | |||
3550 | /// appropriate value within the aggregate. | |||
3551 | SmallVector<unsigned, 4> Indices; | |||
3552 | ||||
3553 | /// The indices to a GEP instruction which will move Ptr to the correct slot | |||
3554 | /// within the aggregate. | |||
3555 | SmallVector<Value *, 4> GEPIndices; | |||
3556 | ||||
3557 | /// The base pointer of the original op, used as a base for GEPing the | |||
3558 | /// split operations. | |||
3559 | Value *Ptr; | |||
3560 | ||||
3561 | /// The base pointee type being GEPed into. | |||
3562 | Type *BaseTy; | |||
3563 | ||||
3564 | /// Known alignment of the base pointer. | |||
3565 | Align BaseAlign; | |||
3566 | ||||
3567 | /// To calculate offset of each component so we can correctly deduce | |||
3568 | /// alignments. | |||
3569 | const DataLayout &DL; | |||
3570 | ||||
3571 | /// Initialize the splitter with an insertion point, Ptr and start with a | |||
3572 | /// single zero GEP index. | |||
3573 | OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy, | |||
3574 | Align BaseAlign, const DataLayout &DL, IRBuilderTy &IRB) | |||
3575 | : IRB(IRB), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr), BaseTy(BaseTy), | |||
3576 | BaseAlign(BaseAlign), DL(DL) { | |||
3577 | IRB.SetInsertPoint(InsertionPoint); | |||
3578 | } | |||
3579 | ||||
3580 | public: | |||
3581 | /// Generic recursive split emission routine. | |||
3582 | /// | |||
3583 | /// This method recursively splits an aggregate op (load or store) into | |||
3584 | /// scalar or vector ops. It splits recursively until it hits a single value | |||
3585 | /// and emits that single value operation via the template argument. | |||
3586 | /// | |||
3587 | /// The logic of this routine relies on GEPs and insertvalue and | |||
3588 | /// extractvalue all operating with the same fundamental index list, merely | |||
3589 | /// formatted differently (GEPs need actual values). | |||
3590 | /// | |||
3591 | /// \param Ty The type being split recursively into smaller ops. | |||
3592 | /// \param Agg The aggregate value being built up or stored, depending on | |||
3593 | /// whether this is splitting a load or a store respectively. | |||
3594 | void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) { | |||
3595 | if (Ty->isSingleValueType()) { | |||
3596 | unsigned Offset = DL.getIndexedOffsetInType(BaseTy, GEPIndices); | |||
3597 | return static_cast<Derived *>(this)->emitFunc( | |||
3598 | Ty, Agg, commonAlignment(BaseAlign, Offset), Name); | |||
3599 | } | |||
3600 | ||||
3601 | if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { | |||
3602 | unsigned OldSize = Indices.size(); | |||
3603 | (void)OldSize; | |||
3604 | for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size; | |||
3605 | ++Idx) { | |||
3606 | assert(Indices.size() == OldSize && "Did not return to the old size")(static_cast <bool> (Indices.size() == OldSize && "Did not return to the old size") ? void (0) : __assert_fail ("Indices.size() == OldSize && \"Did not return to the old size\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3606, __extension__ __PRETTY_FUNCTION__ )); | |||
3607 | Indices.push_back(Idx); | |||
3608 | GEPIndices.push_back(IRB.getInt32(Idx)); | |||
3609 | emitSplitOps(ATy->getElementType(), Agg, Name + "." + Twine(Idx)); | |||
3610 | GEPIndices.pop_back(); | |||
3611 | Indices.pop_back(); | |||
3612 | } | |||
3613 | return; | |||
3614 | } | |||
3615 | ||||
3616 | if (StructType *STy = dyn_cast<StructType>(Ty)) { | |||
3617 | unsigned OldSize = Indices.size(); | |||
3618 | (void)OldSize; | |||
3619 | for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size; | |||
3620 | ++Idx) { | |||
3621 | assert(Indices.size() == OldSize && "Did not return to the old size")(static_cast <bool> (Indices.size() == OldSize && "Did not return to the old size") ? void (0) : __assert_fail ("Indices.size() == OldSize && \"Did not return to the old size\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3621, __extension__ __PRETTY_FUNCTION__ )); | |||
3622 | Indices.push_back(Idx); | |||
3623 | GEPIndices.push_back(IRB.getInt32(Idx)); | |||
3624 | emitSplitOps(STy->getElementType(Idx), Agg, Name + "." + Twine(Idx)); | |||
3625 | GEPIndices.pop_back(); | |||
3626 | Indices.pop_back(); | |||
3627 | } | |||
3628 | return; | |||
3629 | } | |||
3630 | ||||
3631 | llvm_unreachable("Only arrays and structs are aggregate loadable types")::llvm::llvm_unreachable_internal("Only arrays and structs are aggregate loadable types" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3631); | |||
3632 | } | |||
3633 | }; | |||
3634 | ||||
3635 | struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> { | |||
3636 | AAMDNodes AATags; | |||
3637 | ||||
3638 | LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy, | |||
3639 | AAMDNodes AATags, Align BaseAlign, const DataLayout &DL, | |||
3640 | IRBuilderTy &IRB) | |||
3641 | : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL, | |||
3642 | IRB), | |||
3643 | AATags(AATags) {} | |||
3644 | ||||
3645 | /// Emit a leaf load of a single value. This is called at the leaves of the | |||
3646 | /// recursive emission to actually load values. | |||
3647 | void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) { | |||
3648 | assert(Ty->isSingleValueType())(static_cast <bool> (Ty->isSingleValueType()) ? void (0) : __assert_fail ("Ty->isSingleValueType()", "llvm/lib/Transforms/Scalar/SROA.cpp" , 3648, __extension__ __PRETTY_FUNCTION__)); | |||
3649 | // Load the single value and insert it using the indices. | |||
3650 | Value *GEP = | |||
3651 | IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep"); | |||
3652 | LoadInst *Load = | |||
3653 | IRB.CreateAlignedLoad(Ty, GEP, Alignment, Name + ".load"); | |||
3654 | ||||
3655 | APInt Offset( | |||
3656 | DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0); | |||
3657 | if (AATags && | |||
3658 | GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset)) | |||
3659 | Load->setAAMetadata(AATags.shift(Offset.getZExtValue())); | |||
3660 | ||||
3661 | Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert"); | |||
3662 | LLVM_DEBUG(dbgs() << " to: " << *Load << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Load << "\n"; } } while (false); | |||
3663 | } | |||
3664 | }; | |||
3665 | ||||
3666 | bool visitLoadInst(LoadInst &LI) { | |||
3667 | assert(LI.getPointerOperand() == *U)(static_cast <bool> (LI.getPointerOperand() == *U) ? void (0) : __assert_fail ("LI.getPointerOperand() == *U", "llvm/lib/Transforms/Scalar/SROA.cpp" , 3667, __extension__ __PRETTY_FUNCTION__)); | |||
3668 | if (!LI.isSimple() || LI.getType()->isSingleValueType()) | |||
3669 | return false; | |||
3670 | ||||
3671 | // We have an aggregate being loaded, split it apart. | |||
3672 | LLVM_DEBUG(dbgs() << " original: " << LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << LI << "\n"; } } while (false); | |||
3673 | LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(), | |||
3674 | getAdjustedAlignment(&LI, 0), DL, IRB); | |||
3675 | Value *V = PoisonValue::get(LI.getType()); | |||
3676 | Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca"); | |||
3677 | Visited.erase(&LI); | |||
3678 | LI.replaceAllUsesWith(V); | |||
3679 | LI.eraseFromParent(); | |||
3680 | return true; | |||
3681 | } | |||
3682 | ||||
3683 | struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> { | |||
3684 | StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy, | |||
3685 | AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign, | |||
3686 | const DataLayout &DL, IRBuilderTy &IRB) | |||
3687 | : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, | |||
3688 | DL, IRB), | |||
3689 | AATags(AATags), AggStore(AggStore) {} | |||
3690 | AAMDNodes AATags; | |||
3691 | StoreInst *AggStore; | |||
3692 | /// Emit a leaf store of a single value. This is called at the leaves of the | |||
3693 | /// recursive emission to actually produce stores. | |||
3694 | void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) { | |||
3695 | assert(Ty->isSingleValueType())(static_cast <bool> (Ty->isSingleValueType()) ? void (0) : __assert_fail ("Ty->isSingleValueType()", "llvm/lib/Transforms/Scalar/SROA.cpp" , 3695, __extension__ __PRETTY_FUNCTION__)); | |||
3696 | // Extract the single value and store it using the indices. | |||
3697 | // | |||
3698 | // The gep and extractvalue values are factored out of the CreateStore | |||
3699 | // call to make the output independent of the argument evaluation order. | |||
3700 | Value *ExtractValue = | |||
3701 | IRB.CreateExtractValue(Agg, Indices, Name + ".extract"); | |||
3702 | Value *InBoundsGEP = | |||
3703 | IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep"); | |||
3704 | StoreInst *Store = | |||
3705 | IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Alignment); | |||
3706 | ||||
3707 | APInt Offset( | |||
3708 | DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0); | |||
3709 | GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset); | |||
3710 | if (AATags) | |||
3711 | Store->setAAMetadata(AATags.shift(Offset.getZExtValue())); | |||
3712 | ||||
3713 | // migrateDebugInfo requires the base Alloca. Walk to it from this gep. | |||
3714 | // If we cannot (because there's an intervening non-const or unbounded | |||
3715 | // gep) then we wouldn't expect to see dbg.assign intrinsics linked to | |||
3716 | // this instruction. | |||
3717 | Value *Base = AggStore->getPointerOperand()->stripInBoundsOffsets(); | |||
3718 | if (auto *OldAI = dyn_cast<AllocaInst>(Base)) { | |||
3719 | uint64_t SizeInBits = | |||
3720 | DL.getTypeSizeInBits(Store->getValueOperand()->getType()); | |||
3721 | migrateDebugInfo(OldAI, /*IsSplit*/ true, Offset.getZExtValue() * 8, | |||
3722 | SizeInBits, AggStore, Store, | |||
3723 | Store->getPointerOperand(), Store->getValueOperand(), | |||
3724 | DL); | |||
3725 | } else { | |||
3726 | assert(at::getAssignmentMarkers(Store).empty() &&(static_cast <bool> (at::getAssignmentMarkers(Store).empty () && "AT: unexpected debug.assign linked to store through " "unbounded GEP") ? void (0) : __assert_fail ("at::getAssignmentMarkers(Store).empty() && \"AT: unexpected debug.assign linked to store through \" \"unbounded GEP\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3728, __extension__ __PRETTY_FUNCTION__ )) | |||
3727 | "AT: unexpected debug.assign linked to store through "(static_cast <bool> (at::getAssignmentMarkers(Store).empty () && "AT: unexpected debug.assign linked to store through " "unbounded GEP") ? void (0) : __assert_fail ("at::getAssignmentMarkers(Store).empty() && \"AT: unexpected debug.assign linked to store through \" \"unbounded GEP\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3728, __extension__ __PRETTY_FUNCTION__ )) | |||
3728 | "unbounded GEP")(static_cast <bool> (at::getAssignmentMarkers(Store).empty () && "AT: unexpected debug.assign linked to store through " "unbounded GEP") ? void (0) : __assert_fail ("at::getAssignmentMarkers(Store).empty() && \"AT: unexpected debug.assign linked to store through \" \"unbounded GEP\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 3728, __extension__ __PRETTY_FUNCTION__ )); | |||
3729 | } | |||
3730 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||
3731 | } | |||
3732 | }; | |||
3733 | ||||
3734 | bool visitStoreInst(StoreInst &SI) { | |||
3735 | if (!SI.isSimple() || SI.getPointerOperand() != *U) | |||
3736 | return false; | |||
3737 | Value *V = SI.getValueOperand(); | |||
3738 | if (V->getType()->isSingleValueType()) | |||
3739 | return false; | |||
3740 | ||||
3741 | // We have an aggregate being stored, split it apart. | |||
3742 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||
3743 | StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI, | |||
3744 | getAdjustedAlignment(&SI, 0), DL, IRB); | |||
3745 | Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca"); | |||
3746 | Visited.erase(&SI); | |||
3747 | // The stores replacing SI each have markers describing fragments of the | |||
3748 | // assignment so delete the assignment markers linked to SI. | |||
3749 | at::deleteAssignmentMarkers(&SI); | |||
3750 | SI.eraseFromParent(); | |||
3751 | return true; | |||
3752 | } | |||
3753 | ||||
3754 | bool visitBitCastInst(BitCastInst &BC) { | |||
3755 | enqueueUsers(BC); | |||
3756 | return false; | |||
3757 | } | |||
3758 | ||||
3759 | bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) { | |||
3760 | enqueueUsers(ASC); | |||
3761 | return false; | |||
3762 | } | |||
3763 | ||||
3764 | // Fold gep (select cond, ptr1, ptr2) => select cond, gep(ptr1), gep(ptr2) | |||
3765 | bool foldGEPSelect(GetElementPtrInst &GEPI) { | |||
3766 | if (!GEPI.hasAllConstantIndices()) | |||
3767 | return false; | |||
3768 | ||||
3769 | SelectInst *Sel = cast<SelectInst>(GEPI.getPointerOperand()); | |||
3770 | ||||
3771 | LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(select) -> select(gep):" << "\n original: " << *Sel << "\n " << GEPI; } } while (false) | |||
3772 | << "\n original: " << *Seldo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(select) -> select(gep):" << "\n original: " << *Sel << "\n " << GEPI; } } while (false) | |||
3773 | << "\n " << GEPI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(select) -> select(gep):" << "\n original: " << *Sel << "\n " << GEPI; } } while (false); | |||
3774 | ||||
3775 | IRB.SetInsertPoint(&GEPI); | |||
3776 | SmallVector<Value *, 4> Index(GEPI.indices()); | |||
3777 | bool IsInBounds = GEPI.isInBounds(); | |||
3778 | ||||
3779 | Type *Ty = GEPI.getSourceElementType(); | |||
3780 | Value *True = Sel->getTrueValue(); | |||
3781 | Value *NTrue = IRB.CreateGEP(Ty, True, Index, True->getName() + ".sroa.gep", | |||
3782 | IsInBounds); | |||
3783 | ||||
3784 | Value *False = Sel->getFalseValue(); | |||
3785 | ||||
3786 | Value *NFalse = IRB.CreateGEP(Ty, False, Index, | |||
3787 | False->getName() + ".sroa.gep", IsInBounds); | |||
3788 | ||||
3789 | Value *NSel = IRB.CreateSelect(Sel->getCondition(), NTrue, NFalse, | |||
3790 | Sel->getName() + ".sroa.sel"); | |||
3791 | Visited.erase(&GEPI); | |||
3792 | GEPI.replaceAllUsesWith(NSel); | |||
3793 | GEPI.eraseFromParent(); | |||
3794 | Instruction *NSelI = cast<Instruction>(NSel); | |||
3795 | Visited.insert(NSelI); | |||
3796 | enqueueUsers(*NSelI); | |||
3797 | ||||
3798 | LLVM_DEBUG(dbgs() << "\n to: " << *NTruedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n to: " << *NTrue << "\n " << *NFalse << "\n " << *NSel << '\n'; } } while (false) | |||
3799 | << "\n " << *NFalsedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n to: " << *NTrue << "\n " << *NFalse << "\n " << *NSel << '\n'; } } while (false) | |||
3800 | << "\n " << *NSel << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n to: " << *NTrue << "\n " << *NFalse << "\n " << *NSel << '\n'; } } while (false); | |||
3801 | ||||
3802 | return true; | |||
3803 | } | |||
3804 | ||||
3805 | // Fold gep (phi ptr1, ptr2) => phi gep(ptr1), gep(ptr2) | |||
3806 | bool foldGEPPhi(GetElementPtrInst &GEPI) { | |||
3807 | if (!GEPI.hasAllConstantIndices()) | |||
3808 | return false; | |||
3809 | ||||
3810 | PHINode *PHI = cast<PHINode>(GEPI.getPointerOperand()); | |||
3811 | if (GEPI.getParent() != PHI->getParent() || | |||
3812 | llvm::any_of(PHI->incoming_values(), [](Value *In) | |||
3813 | { Instruction *I = dyn_cast<Instruction>(In); | |||
3814 | return !I || isa<GetElementPtrInst>(I) || isa<PHINode>(I) || | |||
3815 | succ_empty(I->getParent()) || | |||
3816 | !I->getParent()->isLegalToHoistInto(); | |||
3817 | })) | |||
3818 | return false; | |||
3819 | ||||
3820 | LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(phi) -> phi(gep):" << "\n original: " << *PHI << "\n " << GEPI << "\n to: "; } } while (false) | |||
3821 | << "\n original: " << *PHIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(phi) -> phi(gep):" << "\n original: " << *PHI << "\n " << GEPI << "\n to: "; } } while (false) | |||
3822 | << "\n " << GEPIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(phi) -> phi(gep):" << "\n original: " << *PHI << "\n " << GEPI << "\n to: "; } } while (false) | |||
3823 | << "\n to: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(phi) -> phi(gep):" << "\n original: " << *PHI << "\n " << GEPI << "\n to: "; } } while (false); | |||
3824 | ||||
3825 | SmallVector<Value *, 4> Index(GEPI.indices()); | |||
3826 | bool IsInBounds = GEPI.isInBounds(); | |||
3827 | IRB.SetInsertPoint(GEPI.getParent()->getFirstNonPHI()); | |||
3828 | PHINode *NewPN = IRB.CreatePHI(GEPI.getType(), PHI->getNumIncomingValues(), | |||
3829 | PHI->getName() + ".sroa.phi"); | |||
3830 | for (unsigned I = 0, E = PHI->getNumIncomingValues(); I != E; ++I) { | |||
3831 | BasicBlock *B = PHI->getIncomingBlock(I); | |||
3832 | Value *NewVal = nullptr; | |||
3833 | int Idx = NewPN->getBasicBlockIndex(B); | |||
3834 | if (Idx >= 0) { | |||
3835 | NewVal = NewPN->getIncomingValue(Idx); | |||
3836 | } else { | |||
3837 | Instruction *In = cast<Instruction>(PHI->getIncomingValue(I)); | |||
3838 | ||||
3839 | IRB.SetInsertPoint(In->getParent(), std::next(In->getIterator())); | |||
3840 | Type *Ty = GEPI.getSourceElementType(); | |||
3841 | NewVal = IRB.CreateGEP(Ty, In, Index, In->getName() + ".sroa.gep", | |||
3842 | IsInBounds); | |||
3843 | } | |||
3844 | NewPN->addIncoming(NewVal, B); | |||
3845 | } | |||
3846 | ||||
3847 | Visited.erase(&GEPI); | |||
3848 | GEPI.replaceAllUsesWith(NewPN); | |||
3849 | GEPI.eraseFromParent(); | |||
3850 | Visited.insert(NewPN); | |||
3851 | enqueueUsers(*NewPN); | |||
3852 | ||||
3853 | LLVM_DEBUG(for (Value *In : NewPN->incoming_values())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { for (Value *In : NewPN->incoming_values()) dbgs () << "\n " << *In; dbgs() << "\n " << *NewPN << '\n'; } } while (false) | |||
3854 | dbgs() << "\n " << *In;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { for (Value *In : NewPN->incoming_values()) dbgs () << "\n " << *In; dbgs() << "\n " << *NewPN << '\n'; } } while (false) | |||
3855 | dbgs() << "\n " << *NewPN << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { for (Value *In : NewPN->incoming_values()) dbgs () << "\n " << *In; dbgs() << "\n " << *NewPN << '\n'; } } while (false); | |||
3856 | ||||
3857 | return true; | |||
3858 | } | |||
3859 | ||||
3860 | bool visitGetElementPtrInst(GetElementPtrInst &GEPI) { | |||
3861 | if (isa<SelectInst>(GEPI.getPointerOperand()) && | |||
3862 | foldGEPSelect(GEPI)) | |||
3863 | return true; | |||
3864 | ||||
3865 | if (isa<PHINode>(GEPI.getPointerOperand()) && | |||
3866 | foldGEPPhi(GEPI)) | |||
3867 | return true; | |||
3868 | ||||
3869 | enqueueUsers(GEPI); | |||
3870 | return false; | |||
3871 | } | |||
3872 | ||||
3873 | bool visitPHINode(PHINode &PN) { | |||
3874 | enqueueUsers(PN); | |||
3875 | return false; | |||
3876 | } | |||
3877 | ||||
3878 | bool visitSelectInst(SelectInst &SI) { | |||
3879 | enqueueUsers(SI); | |||
3880 | return false; | |||
3881 | } | |||
3882 | }; | |||
3883 | ||||
3884 | } // end anonymous namespace | |||
3885 | ||||
3886 | /// Strip aggregate type wrapping. | |||
3887 | /// | |||
3888 | /// This removes no-op aggregate types wrapping an underlying type. It will | |||
3889 | /// strip as many layers of types as it can without changing either the type | |||
3890 | /// size or the allocated size. | |||
3891 | static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) { | |||
3892 | if (Ty->isSingleValueType()) | |||
3893 | return Ty; | |||
3894 | ||||
3895 | uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedValue(); | |||
3896 | uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue(); | |||
3897 | ||||
3898 | Type *InnerTy; | |||
3899 | if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) { | |||
3900 | InnerTy = ArrTy->getElementType(); | |||
3901 | } else if (StructType *STy = dyn_cast<StructType>(Ty)) { | |||
3902 | const StructLayout *SL = DL.getStructLayout(STy); | |||
3903 | unsigned Index = SL->getElementContainingOffset(0); | |||
3904 | InnerTy = STy->getElementType(Index); | |||
3905 | } else { | |||
3906 | return Ty; | |||
3907 | } | |||
3908 | ||||
3909 | if (AllocSize > DL.getTypeAllocSize(InnerTy).getFixedValue() || | |||
3910 | TypeSize > DL.getTypeSizeInBits(InnerTy).getFixedValue()) | |||
3911 | return Ty; | |||
3912 | ||||
3913 | return stripAggregateTypeWrapping(DL, InnerTy); | |||
3914 | } | |||
3915 | ||||
3916 | /// Try to find a partition of the aggregate type passed in for a given | |||
3917 | /// offset and size. | |||
3918 | /// | |||
3919 | /// This recurses through the aggregate type and tries to compute a subtype | |||
3920 | /// based on the offset and size. When the offset and size span a sub-section | |||
3921 | /// of an array, it will even compute a new array type for that sub-section, | |||
3922 | /// and the same for structs. | |||
3923 | /// | |||
3924 | /// Note that this routine is very strict and tries to find a partition of the | |||
3925 | /// type which produces the *exact* right offset and size. It is not forgiving | |||
3926 | /// when the size or offset cause either end of type-based partition to be off. | |||
3927 | /// Also, this is a best-effort routine. It is reasonable to give up and not | |||
3928 | /// return a type if necessary. | |||
3929 | static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset, | |||
3930 | uint64_t Size) { | |||
3931 | if (Offset == 0 && DL.getTypeAllocSize(Ty).getFixedValue() == Size) | |||
3932 | return stripAggregateTypeWrapping(DL, Ty); | |||
3933 | if (Offset > DL.getTypeAllocSize(Ty).getFixedValue() || | |||
3934 | (DL.getTypeAllocSize(Ty).getFixedValue() - Offset) < Size) | |||
3935 | return nullptr; | |||
3936 | ||||
3937 | if (isa<ArrayType>(Ty) || isa<VectorType>(Ty)) { | |||
3938 | Type *ElementTy; | |||
3939 | uint64_t TyNumElements; | |||
3940 | if (auto *AT = dyn_cast<ArrayType>(Ty)) { | |||
3941 | ElementTy = AT->getElementType(); | |||
3942 | TyNumElements = AT->getNumElements(); | |||
3943 | } else { | |||
3944 | // FIXME: This isn't right for vectors with non-byte-sized or | |||
3945 | // non-power-of-two sized elements. | |||
3946 | auto *VT = cast<FixedVectorType>(Ty); | |||
3947 | ElementTy = VT->getElementType(); | |||
3948 | TyNumElements = VT->getNumElements(); | |||
3949 | } | |||
3950 | uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedValue(); | |||
3951 | uint64_t NumSkippedElements = Offset / ElementSize; | |||
3952 | if (NumSkippedElements >= TyNumElements) | |||
3953 | return nullptr; | |||
3954 | Offset -= NumSkippedElements * ElementSize; | |||
3955 | ||||
3956 | // First check if we need to recurse. | |||
3957 | if (Offset > 0 || Size < ElementSize) { | |||
3958 | // Bail if the partition ends in a different array element. | |||
3959 | if ((Offset + Size) > ElementSize) | |||
3960 | return nullptr; | |||
3961 | // Recurse through the element type trying to peel off offset bytes. | |||
3962 | return getTypePartition(DL, ElementTy, Offset, Size); | |||
3963 | } | |||
3964 | assert(Offset == 0)(static_cast <bool> (Offset == 0) ? void (0) : __assert_fail ("Offset == 0", "llvm/lib/Transforms/Scalar/SROA.cpp", 3964, __extension__ __PRETTY_FUNCTION__)); | |||
3965 | ||||
3966 | if (Size == ElementSize) | |||
3967 | return stripAggregateTypeWrapping(DL, ElementTy); | |||
3968 | assert(Size > ElementSize)(static_cast <bool> (Size > ElementSize) ? void (0) : __assert_fail ("Size > ElementSize", "llvm/lib/Transforms/Scalar/SROA.cpp" , 3968, __extension__ __PRETTY_FUNCTION__)); | |||
3969 | uint64_t NumElements = Size / ElementSize; | |||
3970 | if (NumElements * ElementSize != Size) | |||
3971 | return nullptr; | |||
3972 | return ArrayType::get(ElementTy, NumElements); | |||
3973 | } | |||
3974 | ||||
3975 | StructType *STy = dyn_cast<StructType>(Ty); | |||
3976 | if (!STy) | |||
3977 | return nullptr; | |||
3978 | ||||
3979 | const StructLayout *SL = DL.getStructLayout(STy); | |||
3980 | if (Offset >= SL->getSizeInBytes()) | |||
3981 | return nullptr; | |||
3982 | uint64_t EndOffset = Offset + Size; | |||
3983 | if (EndOffset > SL->getSizeInBytes()) | |||
3984 | return nullptr; | |||
3985 | ||||
3986 | unsigned Index = SL->getElementContainingOffset(Offset); | |||
3987 | Offset -= SL->getElementOffset(Index); | |||
3988 | ||||
3989 | Type *ElementTy = STy->getElementType(Index); | |||
3990 | uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedValue(); | |||
3991 | if (Offset >= ElementSize) | |||
3992 | return nullptr; // The offset points into alignment padding. | |||
3993 | ||||
3994 | // See if any partition must be contained by the element. | |||
3995 | if (Offset > 0 || Size < ElementSize) { | |||
3996 | if ((Offset + Size) > ElementSize) | |||
3997 | return nullptr; | |||
3998 | return getTypePartition(DL, ElementTy, Offset, Size); | |||
3999 | } | |||
4000 | assert(Offset == 0)(static_cast <bool> (Offset == 0) ? void (0) : __assert_fail ("Offset == 0", "llvm/lib/Transforms/Scalar/SROA.cpp", 4000, __extension__ __PRETTY_FUNCTION__)); | |||
4001 | ||||
4002 | if (Size == ElementSize) | |||
4003 | return stripAggregateTypeWrapping(DL, ElementTy); | |||
4004 | ||||
4005 | StructType::element_iterator EI = STy->element_begin() + Index, | |||
4006 | EE = STy->element_end(); | |||
4007 | if (EndOffset < SL->getSizeInBytes()) { | |||
4008 | unsigned EndIndex = SL->getElementContainingOffset(EndOffset); | |||
4009 | if (Index == EndIndex) | |||
4010 | return nullptr; // Within a single element and its padding. | |||
4011 | ||||
4012 | // Don't try to form "natural" types if the elements don't line up with the | |||
4013 | // expected size. | |||
4014 | // FIXME: We could potentially recurse down through the last element in the | |||
4015 | // sub-struct to find a natural end point. | |||
4016 | if (SL->getElementOffset(EndIndex) != EndOffset) | |||
4017 | return nullptr; | |||
4018 | ||||
4019 | assert(Index < EndIndex)(static_cast <bool> (Index < EndIndex) ? void (0) : __assert_fail ("Index < EndIndex", "llvm/lib/Transforms/Scalar/SROA.cpp" , 4019, __extension__ __PRETTY_FUNCTION__)); | |||
4020 | EE = STy->element_begin() + EndIndex; | |||
4021 | } | |||
4022 | ||||
4023 | // Try to build up a sub-structure. | |||
4024 | StructType *SubTy = | |||
4025 | StructType::get(STy->getContext(), ArrayRef(EI, EE), STy->isPacked()); | |||
4026 | const StructLayout *SubSL = DL.getStructLayout(SubTy); | |||
4027 | if (Size != SubSL->getSizeInBytes()) | |||
4028 | return nullptr; // The sub-struct doesn't have quite the size needed. | |||
4029 | ||||
4030 | return SubTy; | |||
4031 | } | |||
4032 | ||||
4033 | /// Pre-split loads and stores to simplify rewriting. | |||
4034 | /// | |||
4035 | /// We want to break up the splittable load+store pairs as much as | |||
4036 | /// possible. This is important to do as a preprocessing step, as once we | |||
4037 | /// start rewriting the accesses to partitions of the alloca we lose the | |||
4038 | /// necessary information to correctly split apart paired loads and stores | |||
4039 | /// which both point into this alloca. The case to consider is something like | |||
4040 | /// the following: | |||
4041 | /// | |||
4042 | /// %a = alloca [12 x i8] | |||
4043 | /// %gep1 = getelementptr i8, ptr %a, i32 0 | |||
4044 | /// %gep2 = getelementptr i8, ptr %a, i32 4 | |||
4045 | /// %gep3 = getelementptr i8, ptr %a, i32 8 | |||
4046 | /// store float 0.0, ptr %gep1 | |||
4047 | /// store float 1.0, ptr %gep2 | |||
4048 | /// %v = load i64, ptr %gep1 | |||
4049 | /// store i64 %v, ptr %gep2 | |||
4050 | /// %f1 = load float, ptr %gep2 | |||
4051 | /// %f2 = load float, ptr %gep3 | |||
4052 | /// | |||
4053 | /// Here we want to form 3 partitions of the alloca, each 4 bytes large, and | |||
4054 | /// promote everything so we recover the 2 SSA values that should have been | |||
4055 | /// there all along. | |||
4056 | /// | |||
4057 | /// \returns true if any changes are made. | |||
4058 | bool SROAPass::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { | |||
4059 | LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Pre-splitting loads and stores\n" ; } } while (false); | |||
4060 | ||||
4061 | // Track the loads and stores which are candidates for pre-splitting here, in | |||
4062 | // the order they first appear during the partition scan. These give stable | |||
4063 | // iteration order and a basis for tracking which loads and stores we | |||
4064 | // actually split. | |||
4065 | SmallVector<LoadInst *, 4> Loads; | |||
4066 | SmallVector<StoreInst *, 4> Stores; | |||
4067 | ||||
4068 | // We need to accumulate the splits required of each load or store where we | |||
4069 | // can find them via a direct lookup. This is important to cross-check loads | |||
4070 | // and stores against each other. We also track the slice so that we can kill | |||
4071 | // all the slices that end up split. | |||
4072 | struct SplitOffsets { | |||
4073 | Slice *S; | |||
4074 | std::vector<uint64_t> Splits; | |||
4075 | }; | |||
4076 | SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap; | |||
4077 | ||||
4078 | // Track loads out of this alloca which cannot, for any reason, be pre-split. | |||
4079 | // This is important as we also cannot pre-split stores of those loads! | |||
4080 | // FIXME: This is all pretty gross. It means that we can be more aggressive | |||
4081 | // in pre-splitting when the load feeding the store happens to come from | |||
4082 | // a separate alloca. Put another way, the effectiveness of SROA would be | |||
4083 | // decreased by a frontend which just concatenated all of its local allocas | |||
4084 | // into one big flat alloca. But defeating such patterns is exactly the job | |||
4085 | // SROA is tasked with! Sadly, to not have this discrepancy we would have | |||
4086 | // change store pre-splitting to actually force pre-splitting of the load | |||
4087 | // that feeds it *and all stores*. That makes pre-splitting much harder, but | |||
4088 | // maybe it would make it more principled? | |||
4089 | SmallPtrSet<LoadInst *, 8> UnsplittableLoads; | |||
4090 | ||||
4091 | LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Searching for candidate loads and stores\n" ; } } while (false); | |||
4092 | for (auto &P : AS.partitions()) { | |||
4093 | for (Slice &S : P) { | |||
4094 | Instruction *I = cast<Instruction>(S.getUse()->getUser()); | |||
4095 | if (!S.isSplittable() || S.endOffset() <= P.endOffset()) { | |||
4096 | // If this is a load we have to track that it can't participate in any | |||
4097 | // pre-splitting. If this is a store of a load we have to track that | |||
4098 | // that load also can't participate in any pre-splitting. | |||
4099 | if (auto *LI = dyn_cast<LoadInst>(I)) | |||
4100 | UnsplittableLoads.insert(LI); | |||
4101 | else if (auto *SI = dyn_cast<StoreInst>(I)) | |||
4102 | if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand())) | |||
4103 | UnsplittableLoads.insert(LI); | |||
4104 | continue; | |||
4105 | } | |||
4106 | assert(P.endOffset() > S.beginOffset() &&(static_cast <bool> (P.endOffset() > S.beginOffset() && "Empty or backwards partition!") ? void (0) : __assert_fail ("P.endOffset() > S.beginOffset() && \"Empty or backwards partition!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4107, __extension__ __PRETTY_FUNCTION__ )) | |||
4107 | "Empty or backwards partition!")(static_cast <bool> (P.endOffset() > S.beginOffset() && "Empty or backwards partition!") ? void (0) : __assert_fail ("P.endOffset() > S.beginOffset() && \"Empty or backwards partition!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4107, __extension__ __PRETTY_FUNCTION__ )); | |||
4108 | ||||
4109 | // Determine if this is a pre-splittable slice. | |||
4110 | if (auto *LI = dyn_cast<LoadInst>(I)) { | |||
4111 | assert(!LI->isVolatile() && "Cannot split volatile loads!")(static_cast <bool> (!LI->isVolatile() && "Cannot split volatile loads!" ) ? void (0) : __assert_fail ("!LI->isVolatile() && \"Cannot split volatile loads!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4111, __extension__ __PRETTY_FUNCTION__ )); | |||
4112 | ||||
4113 | // The load must be used exclusively to store into other pointers for | |||
4114 | // us to be able to arbitrarily pre-split it. The stores must also be | |||
4115 | // simple to avoid changing semantics. | |||
4116 | auto IsLoadSimplyStored = [](LoadInst *LI) { | |||
4117 | for (User *LU : LI->users()) { | |||
4118 | auto *SI = dyn_cast<StoreInst>(LU); | |||
4119 | if (!SI || !SI->isSimple()) | |||
4120 | return false; | |||
4121 | } | |||
4122 | return true; | |||
4123 | }; | |||
4124 | if (!IsLoadSimplyStored(LI)) { | |||
4125 | UnsplittableLoads.insert(LI); | |||
4126 | continue; | |||
4127 | } | |||
4128 | ||||
4129 | Loads.push_back(LI); | |||
4130 | } else if (auto *SI = dyn_cast<StoreInst>(I)) { | |||
4131 | if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex())) | |||
4132 | // Skip stores *of* pointers. FIXME: This shouldn't even be possible! | |||
4133 | continue; | |||
4134 | auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand()); | |||
4135 | if (!StoredLoad || !StoredLoad->isSimple()) | |||
4136 | continue; | |||
4137 | assert(!SI->isVolatile() && "Cannot split volatile stores!")(static_cast <bool> (!SI->isVolatile() && "Cannot split volatile stores!" ) ? void (0) : __assert_fail ("!SI->isVolatile() && \"Cannot split volatile stores!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4137, __extension__ __PRETTY_FUNCTION__ )); | |||
4138 | ||||
4139 | Stores.push_back(SI); | |||
4140 | } else { | |||
4141 | // Other uses cannot be pre-split. | |||
4142 | continue; | |||
4143 | } | |||
4144 | ||||
4145 | // Record the initial split. | |||
4146 | LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Candidate: " << *I << "\n"; } } while (false); | |||
4147 | auto &Offsets = SplitOffsetsMap[I]; | |||
4148 | assert(Offsets.Splits.empty() &&(static_cast <bool> (Offsets.Splits.empty() && "Should not have splits the first time we see an instruction!" ) ? void (0) : __assert_fail ("Offsets.Splits.empty() && \"Should not have splits the first time we see an instruction!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4149, __extension__ __PRETTY_FUNCTION__ )) | |||
4149 | "Should not have splits the first time we see an instruction!")(static_cast <bool> (Offsets.Splits.empty() && "Should not have splits the first time we see an instruction!" ) ? void (0) : __assert_fail ("Offsets.Splits.empty() && \"Should not have splits the first time we see an instruction!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4149, __extension__ __PRETTY_FUNCTION__ )); | |||
4150 | Offsets.S = &S; | |||
4151 | Offsets.Splits.push_back(P.endOffset() - S.beginOffset()); | |||
4152 | } | |||
4153 | ||||
4154 | // Now scan the already split slices, and add a split for any of them which | |||
4155 | // we're going to pre-split. | |||
4156 | for (Slice *S : P.splitSliceTails()) { | |||
4157 | auto SplitOffsetsMapI = | |||
4158 | SplitOffsetsMap.find(cast<Instruction>(S->getUse()->getUser())); | |||
4159 | if (SplitOffsetsMapI == SplitOffsetsMap.end()) | |||
4160 | continue; | |||
4161 | auto &Offsets = SplitOffsetsMapI->second; | |||
4162 | ||||
4163 | assert(Offsets.S == S && "Found a mismatched slice!")(static_cast <bool> (Offsets.S == S && "Found a mismatched slice!" ) ? void (0) : __assert_fail ("Offsets.S == S && \"Found a mismatched slice!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4163, __extension__ __PRETTY_FUNCTION__ )); | |||
4164 | assert(!Offsets.Splits.empty() &&(static_cast <bool> (!Offsets.Splits.empty() && "Cannot have an empty set of splits on the second partition!" ) ? void (0) : __assert_fail ("!Offsets.Splits.empty() && \"Cannot have an empty set of splits on the second partition!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4165, __extension__ __PRETTY_FUNCTION__ )) | |||
4165 | "Cannot have an empty set of splits on the second partition!")(static_cast <bool> (!Offsets.Splits.empty() && "Cannot have an empty set of splits on the second partition!" ) ? void (0) : __assert_fail ("!Offsets.Splits.empty() && \"Cannot have an empty set of splits on the second partition!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4165, __extension__ __PRETTY_FUNCTION__ )); | |||
4166 | assert(Offsets.Splits.back() ==(static_cast <bool> (Offsets.Splits.back() == P.beginOffset () - Offsets.S->beginOffset() && "Previous split does not end where this one begins!" ) ? void (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4168, __extension__ __PRETTY_FUNCTION__ )) | |||
4167 | P.beginOffset() - Offsets.S->beginOffset() &&(static_cast <bool> (Offsets.Splits.back() == P.beginOffset () - Offsets.S->beginOffset() && "Previous split does not end where this one begins!" ) ? void (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4168, __extension__ __PRETTY_FUNCTION__ )) | |||
4168 | "Previous split does not end where this one begins!")(static_cast <bool> (Offsets.Splits.back() == P.beginOffset () - Offsets.S->beginOffset() && "Previous split does not end where this one begins!" ) ? void (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4168, __extension__ __PRETTY_FUNCTION__ )); | |||
4169 | ||||
4170 | // Record each split. The last partition's end isn't needed as the size | |||
4171 | // of the slice dictates that. | |||
4172 | if (S->endOffset() > P.endOffset()) | |||
4173 | Offsets.Splits.push_back(P.endOffset() - Offsets.S->beginOffset()); | |||
4174 | } | |||
4175 | } | |||
4176 | ||||
4177 | // We may have split loads where some of their stores are split stores. For | |||
4178 | // such loads and stores, we can only pre-split them if their splits exactly | |||
4179 | // match relative to their starting offset. We have to verify this prior to | |||
4180 | // any rewriting. | |||
4181 | llvm::erase_if(Stores, [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) { | |||
4182 | // Lookup the load we are storing in our map of split | |||
4183 | // offsets. | |||
4184 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | |||
4185 | // If it was completely unsplittable, then we're done, | |||
4186 | // and this store can't be pre-split. | |||
4187 | if (UnsplittableLoads.count(LI)) | |||
4188 | return true; | |||
4189 | ||||
4190 | auto LoadOffsetsI = SplitOffsetsMap.find(LI); | |||
4191 | if (LoadOffsetsI == SplitOffsetsMap.end()) | |||
4192 | return false; // Unrelated loads are definitely safe. | |||
4193 | auto &LoadOffsets = LoadOffsetsI->second; | |||
4194 | ||||
4195 | // Now lookup the store's offsets. | |||
4196 | auto &StoreOffsets = SplitOffsetsMap[SI]; | |||
4197 | ||||
4198 | // If the relative offsets of each split in the load and | |||
4199 | // store match exactly, then we can split them and we | |||
4200 | // don't need to remove them here. | |||
4201 | if (LoadOffsets.Splits == StoreOffsets.Splits) | |||
4202 | return false; | |||
4203 | ||||
4204 | LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | |||
4205 | << " " << *LI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | |||
4206 | << " " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false); | |||
4207 | ||||
4208 | // We've found a store and load that we need to split | |||
4209 | // with mismatched relative splits. Just give up on them | |||
4210 | // and remove both instructions from our list of | |||
4211 | // candidates. | |||
4212 | UnsplittableLoads.insert(LI); | |||
4213 | return true; | |||
4214 | }); | |||
4215 | // Now we have to go *back* through all the stores, because a later store may | |||
4216 | // have caused an earlier store's load to become unsplittable and if it is | |||
4217 | // unsplittable for the later store, then we can't rely on it being split in | |||
4218 | // the earlier store either. | |||
4219 | llvm::erase_if(Stores, [&UnsplittableLoads](StoreInst *SI) { | |||
4220 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | |||
4221 | return UnsplittableLoads.count(LI); | |||
4222 | }); | |||
4223 | // Once we've established all the loads that can't be split for some reason, | |||
4224 | // filter any that made it into our list out. | |||
4225 | llvm::erase_if(Loads, [&UnsplittableLoads](LoadInst *LI) { | |||
4226 | return UnsplittableLoads.count(LI); | |||
4227 | }); | |||
4228 | ||||
4229 | // If no loads or stores are left, there is no pre-splitting to be done for | |||
4230 | // this alloca. | |||
4231 | if (Loads.empty() && Stores.empty()) | |||
4232 | return false; | |||
4233 | ||||
4234 | // From here on, we can't fail and will be building new accesses, so rig up | |||
4235 | // an IR builder. | |||
4236 | IRBuilderTy IRB(&AI); | |||
4237 | ||||
4238 | // Collect the new slices which we will merge into the alloca slices. | |||
4239 | SmallVector<Slice, 4> NewSlices; | |||
4240 | ||||
4241 | // Track any allocas we end up splitting loads and stores for so we iterate | |||
4242 | // on them. | |||
4243 | SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas; | |||
4244 | ||||
4245 | // At this point, we have collected all of the loads and stores we can | |||
4246 | // pre-split, and the specific splits needed for them. We actually do the | |||
4247 | // splitting in a specific order in order to handle when one of the loads in | |||
4248 | // the value operand to one of the stores. | |||
4249 | // | |||
4250 | // First, we rewrite all of the split loads, and just accumulate each split | |||
4251 | // load in a parallel structure. We also build the slices for them and append | |||
4252 | // them to the alloca slices. | |||
4253 | SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap; | |||
4254 | std::vector<LoadInst *> SplitLoads; | |||
4255 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||
4256 | for (LoadInst *LI : Loads) { | |||
4257 | SplitLoads.clear(); | |||
4258 | ||||
4259 | auto &Offsets = SplitOffsetsMap[LI]; | |||
4260 | unsigned SliceSize = Offsets.S->endOffset() - Offsets.S->beginOffset(); | |||
4261 | assert(LI->getType()->getIntegerBitWidth() % 8 == 0 &&(static_cast <bool> (LI->getType()->getIntegerBitWidth () % 8 == 0 && "Load must have type size equal to store size" ) ? void (0) : __assert_fail ("LI->getType()->getIntegerBitWidth() % 8 == 0 && \"Load must have type size equal to store size\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4262, __extension__ __PRETTY_FUNCTION__ )) | |||
4262 | "Load must have type size equal to store size")(static_cast <bool> (LI->getType()->getIntegerBitWidth () % 8 == 0 && "Load must have type size equal to store size" ) ? void (0) : __assert_fail ("LI->getType()->getIntegerBitWidth() % 8 == 0 && \"Load must have type size equal to store size\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4262, __extension__ __PRETTY_FUNCTION__ )); | |||
4263 | assert(LI->getType()->getIntegerBitWidth() / 8 >= SliceSize &&(static_cast <bool> (LI->getType()->getIntegerBitWidth () / 8 >= SliceSize && "Load must be >= slice size" ) ? void (0) : __assert_fail ("LI->getType()->getIntegerBitWidth() / 8 >= SliceSize && \"Load must be >= slice size\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4264, __extension__ __PRETTY_FUNCTION__ )) | |||
4264 | "Load must be >= slice size")(static_cast <bool> (LI->getType()->getIntegerBitWidth () / 8 >= SliceSize && "Load must be >= slice size" ) ? void (0) : __assert_fail ("LI->getType()->getIntegerBitWidth() / 8 >= SliceSize && \"Load must be >= slice size\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4264, __extension__ __PRETTY_FUNCTION__ )); | |||
4265 | ||||
4266 | uint64_t BaseOffset = Offsets.S->beginOffset(); | |||
4267 | assert(BaseOffset + SliceSize > BaseOffset &&(static_cast <bool> (BaseOffset + SliceSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? void (0) : __assert_fail ("BaseOffset + SliceSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4268, __extension__ __PRETTY_FUNCTION__ )) | |||
4268 | "Cannot represent alloca access size using 64-bit integers!")(static_cast <bool> (BaseOffset + SliceSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? void (0) : __assert_fail ("BaseOffset + SliceSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4268, __extension__ __PRETTY_FUNCTION__ )); | |||
4269 | ||||
4270 | Instruction *BasePtr = cast<Instruction>(LI->getPointerOperand()); | |||
4271 | IRB.SetInsertPoint(LI); | |||
4272 | ||||
4273 | LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting load: " << *LI << "\n"; } } while (false); | |||
4274 | ||||
4275 | uint64_t PartOffset = 0, PartSize = Offsets.Splits.front(); | |||
4276 | int Idx = 0, Size = Offsets.Splits.size(); | |||
4277 | for (;;) { | |||
4278 | auto *PartTy = Type::getIntNTy(LI->getContext(), PartSize * 8); | |||
4279 | auto AS = LI->getPointerAddressSpace(); | |||
4280 | auto *PartPtrTy = PartTy->getPointerTo(AS); | |||
4281 | LoadInst *PLoad = IRB.CreateAlignedLoad( | |||
4282 | PartTy, | |||
4283 | getAdjustedPtr(IRB, DL, BasePtr, | |||
4284 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | |||
4285 | PartPtrTy, BasePtr->getName() + "."), | |||
4286 | getAdjustedAlignment(LI, PartOffset), | |||
4287 | /*IsVolatile*/ false, LI->getName()); | |||
4288 | PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access, | |||
4289 | LLVMContext::MD_access_group}); | |||
4290 | ||||
4291 | // Append this load onto the list of split loads so we can find it later | |||
4292 | // to rewrite the stores. | |||
4293 | SplitLoads.push_back(PLoad); | |||
4294 | ||||
4295 | // Now build a new slice for the alloca. | |||
4296 | NewSlices.push_back( | |||
4297 | Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, | |||
4298 | &PLoad->getOperandUse(PLoad->getPointerOperandIndex()), | |||
4299 | /*IsSplittable*/ false)); | |||
4300 | LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false) | |||
4301 | << ", " << NewSlices.back().endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false) | |||
4302 | << "): " << *PLoad << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false); | |||
4303 | ||||
4304 | // See if we've handled all the splits. | |||
4305 | if (Idx >= Size) | |||
4306 | break; | |||
4307 | ||||
4308 | // Setup the next partition. | |||
4309 | PartOffset = Offsets.Splits[Idx]; | |||
4310 | ++Idx; | |||
4311 | PartSize = (Idx < Size ? Offsets.Splits[Idx] : SliceSize) - PartOffset; | |||
4312 | } | |||
4313 | ||||
4314 | // Now that we have the split loads, do the slow walk over all uses of the | |||
4315 | // load and rewrite them as split stores, or save the split loads to use | |||
4316 | // below if the store is going to be split there anyways. | |||
4317 | bool DeferredStores = false; | |||
4318 | for (User *LU : LI->users()) { | |||
4319 | StoreInst *SI = cast<StoreInst>(LU); | |||
4320 | if (!Stores.empty() && SplitOffsetsMap.count(SI)) { | |||
4321 | DeferredStores = true; | |||
4322 | LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Deferred splitting of store: " << *SI << "\n"; } } while (false) | |||
4323 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Deferred splitting of store: " << *SI << "\n"; } } while (false); | |||
4324 | continue; | |||
4325 | } | |||
4326 | ||||
4327 | Value *StoreBasePtr = SI->getPointerOperand(); | |||
4328 | IRB.SetInsertPoint(SI); | |||
4329 | ||||
4330 | LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting store of load: " << *SI << "\n"; } } while (false); | |||
4331 | ||||
4332 | for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) { | |||
4333 | LoadInst *PLoad = SplitLoads[Idx]; | |||
4334 | uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1]; | |||
4335 | auto *PartPtrTy = | |||
4336 | PLoad->getType()->getPointerTo(SI->getPointerAddressSpace()); | |||
4337 | ||||
4338 | auto AS = SI->getPointerAddressSpace(); | |||
4339 | StoreInst *PStore = IRB.CreateAlignedStore( | |||
4340 | PLoad, | |||
4341 | getAdjustedPtr(IRB, DL, StoreBasePtr, | |||
4342 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | |||
4343 | PartPtrTy, StoreBasePtr->getName() + "."), | |||
4344 | getAdjustedAlignment(SI, PartOffset), | |||
4345 | /*IsVolatile*/ false); | |||
4346 | PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
4347 | LLVMContext::MD_access_group, | |||
4348 | LLVMContext::MD_DIAssignID}); | |||
4349 | LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " +" << PartOffset << ":" << *PStore << "\n"; } } while (false); | |||
4350 | } | |||
4351 | ||||
4352 | // We want to immediately iterate on any allocas impacted by splitting | |||
4353 | // this store, and we have to track any promotable alloca (indicated by | |||
4354 | // a direct store) as needing to be resplit because it is no longer | |||
4355 | // promotable. | |||
4356 | if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(StoreBasePtr)) { | |||
4357 | ResplitPromotableAllocas.insert(OtherAI); | |||
4358 | Worklist.insert(OtherAI); | |||
4359 | } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>( | |||
4360 | StoreBasePtr->stripInBoundsOffsets())) { | |||
4361 | Worklist.insert(OtherAI); | |||
4362 | } | |||
4363 | ||||
4364 | // Mark the original store as dead. | |||
4365 | DeadInsts.push_back(SI); | |||
4366 | } | |||
4367 | ||||
4368 | // Save the split loads if there are deferred stores among the users. | |||
4369 | if (DeferredStores) | |||
4370 | SplitLoadsMap.insert(std::make_pair(LI, std::move(SplitLoads))); | |||
4371 | ||||
4372 | // Mark the original load as dead and kill the original slice. | |||
4373 | DeadInsts.push_back(LI); | |||
4374 | Offsets.S->kill(); | |||
4375 | } | |||
4376 | ||||
4377 | // Second, we rewrite all of the split stores. At this point, we know that | |||
4378 | // all loads from this alloca have been split already. For stores of such | |||
4379 | // loads, we can simply look up the pre-existing split loads. For stores of | |||
4380 | // other loads, we split those loads first and then write split stores of | |||
4381 | // them. | |||
4382 | for (StoreInst *SI : Stores) { | |||
4383 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | |||
4384 | IntegerType *Ty = cast<IntegerType>(LI->getType()); | |||
4385 | assert(Ty->getBitWidth() % 8 == 0)(static_cast <bool> (Ty->getBitWidth() % 8 == 0) ? void (0) : __assert_fail ("Ty->getBitWidth() % 8 == 0", "llvm/lib/Transforms/Scalar/SROA.cpp" , 4385, __extension__ __PRETTY_FUNCTION__)); | |||
4386 | uint64_t StoreSize = Ty->getBitWidth() / 8; | |||
4387 | assert(StoreSize > 0 && "Cannot have a zero-sized integer store!")(static_cast <bool> (StoreSize > 0 && "Cannot have a zero-sized integer store!" ) ? void (0) : __assert_fail ("StoreSize > 0 && \"Cannot have a zero-sized integer store!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4387, __extension__ __PRETTY_FUNCTION__ )); | |||
4388 | ||||
4389 | auto &Offsets = SplitOffsetsMap[SI]; | |||
4390 | assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&(static_cast <bool> (StoreSize == Offsets.S->endOffset () - Offsets.S->beginOffset() && "Slice size should always match load size exactly!" ) ? void (0) : __assert_fail ("StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4391, __extension__ __PRETTY_FUNCTION__ )) | |||
4391 | "Slice size should always match load size exactly!")(static_cast <bool> (StoreSize == Offsets.S->endOffset () - Offsets.S->beginOffset() && "Slice size should always match load size exactly!" ) ? void (0) : __assert_fail ("StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4391, __extension__ __PRETTY_FUNCTION__ )); | |||
4392 | uint64_t BaseOffset = Offsets.S->beginOffset(); | |||
4393 | assert(BaseOffset + StoreSize > BaseOffset &&(static_cast <bool> (BaseOffset + StoreSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? void (0) : __assert_fail ("BaseOffset + StoreSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4394, __extension__ __PRETTY_FUNCTION__ )) | |||
4394 | "Cannot represent alloca access size using 64-bit integers!")(static_cast <bool> (BaseOffset + StoreSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? void (0) : __assert_fail ("BaseOffset + StoreSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4394, __extension__ __PRETTY_FUNCTION__ )); | |||
4395 | ||||
4396 | Value *LoadBasePtr = LI->getPointerOperand(); | |||
4397 | Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand()); | |||
4398 | ||||
4399 | LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting store: " << *SI << "\n"; } } while (false); | |||
4400 | ||||
4401 | // Check whether we have an already split load. | |||
4402 | auto SplitLoadsMapI = SplitLoadsMap.find(LI); | |||
4403 | std::vector<LoadInst *> *SplitLoads = nullptr; | |||
4404 | if (SplitLoadsMapI != SplitLoadsMap.end()) { | |||
4405 | SplitLoads = &SplitLoadsMapI->second; | |||
4406 | assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&(static_cast <bool> (SplitLoads->size() == Offsets.Splits .size() + 1 && "Too few split loads for the number of splits in the store!" ) ? void (0) : __assert_fail ("SplitLoads->size() == Offsets.Splits.size() + 1 && \"Too few split loads for the number of splits in the store!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4407, __extension__ __PRETTY_FUNCTION__ )) | |||
4407 | "Too few split loads for the number of splits in the store!")(static_cast <bool> (SplitLoads->size() == Offsets.Splits .size() + 1 && "Too few split loads for the number of splits in the store!" ) ? void (0) : __assert_fail ("SplitLoads->size() == Offsets.Splits.size() + 1 && \"Too few split loads for the number of splits in the store!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4407, __extension__ __PRETTY_FUNCTION__ )); | |||
4408 | } else { | |||
4409 | LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " of load: " << *LI << "\n"; } } while (false); | |||
4410 | } | |||
4411 | ||||
4412 | uint64_t PartOffset = 0, PartSize = Offsets.Splits.front(); | |||
4413 | int Idx = 0, Size = Offsets.Splits.size(); | |||
4414 | for (;;) { | |||
4415 | auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8); | |||
4416 | auto *LoadPartPtrTy = PartTy->getPointerTo(LI->getPointerAddressSpace()); | |||
4417 | auto *StorePartPtrTy = PartTy->getPointerTo(SI->getPointerAddressSpace()); | |||
4418 | ||||
4419 | // Either lookup a split load or create one. | |||
4420 | LoadInst *PLoad; | |||
4421 | if (SplitLoads) { | |||
4422 | PLoad = (*SplitLoads)[Idx]; | |||
4423 | } else { | |||
4424 | IRB.SetInsertPoint(LI); | |||
4425 | auto AS = LI->getPointerAddressSpace(); | |||
4426 | PLoad = IRB.CreateAlignedLoad( | |||
4427 | PartTy, | |||
4428 | getAdjustedPtr(IRB, DL, LoadBasePtr, | |||
4429 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | |||
4430 | LoadPartPtrTy, LoadBasePtr->getName() + "."), | |||
4431 | getAdjustedAlignment(LI, PartOffset), | |||
4432 | /*IsVolatile*/ false, LI->getName()); | |||
4433 | PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access, | |||
4434 | LLVMContext::MD_access_group}); | |||
4435 | } | |||
4436 | ||||
4437 | // And store this partition. | |||
4438 | IRB.SetInsertPoint(SI); | |||
4439 | auto AS = SI->getPointerAddressSpace(); | |||
4440 | StoreInst *PStore = IRB.CreateAlignedStore( | |||
4441 | PLoad, | |||
4442 | getAdjustedPtr(IRB, DL, StoreBasePtr, | |||
4443 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | |||
4444 | StorePartPtrTy, StoreBasePtr->getName() + "."), | |||
4445 | getAdjustedAlignment(SI, PartOffset), | |||
4446 | /*IsVolatile*/ false); | |||
4447 | PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
4448 | LLVMContext::MD_access_group}); | |||
4449 | ||||
4450 | // Now build a new slice for the alloca. | |||
4451 | NewSlices.push_back( | |||
4452 | Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, | |||
4453 | &PStore->getOperandUse(PStore->getPointerOperandIndex()), | |||
4454 | /*IsSplittable*/ false)); | |||
4455 | LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false) | |||
4456 | << ", " << NewSlices.back().endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false) | |||
4457 | << "): " << *PStore << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false); | |||
4458 | if (!SplitLoads) { | |||
4459 | LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " of split load: " << * PLoad << "\n"; } } while (false); | |||
4460 | } | |||
4461 | ||||
4462 | // See if we've finished all the splits. | |||
4463 | if (Idx >= Size) | |||
4464 | break; | |||
4465 | ||||
4466 | // Setup the next partition. | |||
4467 | PartOffset = Offsets.Splits[Idx]; | |||
4468 | ++Idx; | |||
4469 | PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset; | |||
4470 | } | |||
4471 | ||||
4472 | // We want to immediately iterate on any allocas impacted by splitting | |||
4473 | // this load, which is only relevant if it isn't a load of this alloca and | |||
4474 | // thus we didn't already split the loads above. We also have to keep track | |||
4475 | // of any promotable allocas we split loads on as they can no longer be | |||
4476 | // promoted. | |||
4477 | if (!SplitLoads) { | |||
4478 | if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(LoadBasePtr)) { | |||
4479 | assert(OtherAI != &AI && "We can't re-split our own alloca!")(static_cast <bool> (OtherAI != &AI && "We can't re-split our own alloca!" ) ? void (0) : __assert_fail ("OtherAI != &AI && \"We can't re-split our own alloca!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4479, __extension__ __PRETTY_FUNCTION__ )); | |||
4480 | ResplitPromotableAllocas.insert(OtherAI); | |||
4481 | Worklist.insert(OtherAI); | |||
4482 | } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>( | |||
4483 | LoadBasePtr->stripInBoundsOffsets())) { | |||
4484 | assert(OtherAI != &AI && "We can't re-split our own alloca!")(static_cast <bool> (OtherAI != &AI && "We can't re-split our own alloca!" ) ? void (0) : __assert_fail ("OtherAI != &AI && \"We can't re-split our own alloca!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4484, __extension__ __PRETTY_FUNCTION__ )); | |||
4485 | Worklist.insert(OtherAI); | |||
4486 | } | |||
4487 | } | |||
4488 | ||||
4489 | // Mark the original store as dead now that we've split it up and kill its | |||
4490 | // slice. Note that we leave the original load in place unless this store | |||
4491 | // was its only use. It may in turn be split up if it is an alloca load | |||
4492 | // for some other alloca, but it may be a normal load. This may introduce | |||
4493 | // redundant loads, but where those can be merged the rest of the optimizer | |||
4494 | // should handle the merging, and this uncovers SSA splits which is more | |||
4495 | // important. In practice, the original loads will almost always be fully | |||
4496 | // split and removed eventually, and the splits will be merged by any | |||
4497 | // trivial CSE, including instcombine. | |||
4498 | if (LI->hasOneUse()) { | |||
4499 | assert(*LI->user_begin() == SI && "Single use isn't this store!")(static_cast <bool> (*LI->user_begin() == SI && "Single use isn't this store!") ? void (0) : __assert_fail ( "*LI->user_begin() == SI && \"Single use isn't this store!\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4499, __extension__ __PRETTY_FUNCTION__ )); | |||
4500 | DeadInsts.push_back(LI); | |||
4501 | } | |||
4502 | DeadInsts.push_back(SI); | |||
4503 | Offsets.S->kill(); | |||
4504 | } | |||
4505 | ||||
4506 | // Remove the killed slices that have ben pre-split. | |||
4507 | llvm::erase_if(AS, [](const Slice &S) { return S.isDead(); }); | |||
4508 | ||||
4509 | // Insert our new slices. This will sort and merge them into the sorted | |||
4510 | // sequence. | |||
4511 | AS.insert(NewSlices); | |||
4512 | ||||
4513 | LLVM_DEBUG(dbgs() << " Pre-split slices:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Pre-split slices:\n"; } } while (false); | |||
4514 | #ifndef NDEBUG | |||
4515 | for (auto I = AS.begin(), E = AS.end(); I != E; ++I) | |||
4516 | LLVM_DEBUG(AS.print(dbgs(), I, " "))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.print(dbgs(), I, " "); } } while (false); | |||
4517 | #endif | |||
4518 | ||||
4519 | // Finally, don't try to promote any allocas that new require re-splitting. | |||
4520 | // They have already been added to the worklist above. | |||
4521 | llvm::erase_if(PromotableAllocas, [&](AllocaInst *AI) { | |||
4522 | return ResplitPromotableAllocas.count(AI); | |||
4523 | }); | |||
4524 | ||||
4525 | return true; | |||
4526 | } | |||
4527 | ||||
4528 | /// Rewrite an alloca partition's users. | |||
4529 | /// | |||
4530 | /// This routine drives both of the rewriting goals of the SROA pass. It tries | |||
4531 | /// to rewrite uses of an alloca partition to be conducive for SSA value | |||
4532 | /// promotion. If the partition needs a new, more refined alloca, this will | |||
4533 | /// build that new alloca, preserving as much type information as possible, and | |||
4534 | /// rewrite the uses of the old alloca to point at the new one and have the | |||
4535 | /// appropriate new offsets. It also evaluates how successful the rewrite was | |||
4536 | /// at enabling promotion and if it was successful queues the alloca to be | |||
4537 | /// promoted. | |||
4538 | AllocaInst *SROAPass::rewritePartition(AllocaInst &AI, AllocaSlices &AS, | |||
4539 | Partition &P) { | |||
4540 | // Try to compute a friendly type for this partition of the alloca. This | |||
4541 | // won't always succeed, in which case we fall back to a legal integer type | |||
4542 | // or an i8 array of an appropriate size. | |||
4543 | Type *SliceTy = nullptr; | |||
4544 | VectorType *SliceVecTy = nullptr; | |||
4545 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||
4546 | std::pair<Type *, IntegerType *> CommonUseTy = | |||
4547 | findCommonType(P.begin(), P.end(), P.endOffset()); | |||
4548 | // Do all uses operate on the same type? | |||
4549 | if (CommonUseTy.first) | |||
4550 | if (DL.getTypeAllocSize(CommonUseTy.first).getFixedValue() >= P.size()) { | |||
4551 | SliceTy = CommonUseTy.first; | |||
4552 | SliceVecTy = dyn_cast<VectorType>(SliceTy); | |||
4553 | } | |||
4554 | // If not, can we find an appropriate subtype in the original allocated type? | |||
4555 | if (!SliceTy) | |||
4556 | if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(), | |||
4557 | P.beginOffset(), P.size())) | |||
4558 | SliceTy = TypePartitionTy; | |||
4559 | ||||
4560 | // If still not, can we use the largest bitwidth integer type used? | |||
4561 | if (!SliceTy && CommonUseTy.second) | |||
4562 | if (DL.getTypeAllocSize(CommonUseTy.second).getFixedValue() >= P.size()) { | |||
4563 | SliceTy = CommonUseTy.second; | |||
4564 | SliceVecTy = dyn_cast<VectorType>(SliceTy); | |||
4565 | } | |||
4566 | if ((!SliceTy || (SliceTy->isArrayTy() && | |||
4567 | SliceTy->getArrayElementType()->isIntegerTy())) && | |||
4568 | DL.isLegalInteger(P.size() * 8)) { | |||
4569 | SliceTy = Type::getIntNTy(*C, P.size() * 8); | |||
4570 | } | |||
4571 | ||||
4572 | // If the common use types are not viable for promotion then attempt to find | |||
4573 | // another type that is viable. | |||
4574 | if (SliceVecTy && !checkVectorTypeForPromotion(P, SliceVecTy, DL)) | |||
4575 | if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(), | |||
4576 | P.beginOffset(), P.size())) { | |||
4577 | VectorType *TypePartitionVecTy = dyn_cast<VectorType>(TypePartitionTy); | |||
4578 | if (TypePartitionVecTy && | |||
4579 | checkVectorTypeForPromotion(P, TypePartitionVecTy, DL)) | |||
4580 | SliceTy = TypePartitionTy; | |||
4581 | } | |||
4582 | ||||
4583 | if (!SliceTy) | |||
4584 | SliceTy = ArrayType::get(Type::getInt8Ty(*C), P.size()); | |||
4585 | assert(DL.getTypeAllocSize(SliceTy).getFixedValue() >= P.size())(static_cast <bool> (DL.getTypeAllocSize(SliceTy).getFixedValue () >= P.size()) ? void (0) : __assert_fail ("DL.getTypeAllocSize(SliceTy).getFixedValue() >= P.size()" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4585, __extension__ __PRETTY_FUNCTION__ )); | |||
4586 | ||||
4587 | bool IsIntegerPromotable = isIntegerWideningViable(P, SliceTy, DL); | |||
4588 | ||||
4589 | VectorType *VecTy = | |||
4590 | IsIntegerPromotable ? nullptr : isVectorPromotionViable(P, DL); | |||
4591 | if (VecTy) | |||
4592 | SliceTy = VecTy; | |||
4593 | ||||
4594 | // Check for the case where we're going to rewrite to a new alloca of the | |||
4595 | // exact same type as the original, and with the same access offsets. In that | |||
4596 | // case, re-use the existing alloca, but still run through the rewriter to | |||
4597 | // perform phi and select speculation. | |||
4598 | // P.beginOffset() can be non-zero even with the same type in a case with | |||
4599 | // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll). | |||
4600 | AllocaInst *NewAI; | |||
4601 | if (SliceTy == AI.getAllocatedType() && P.beginOffset() == 0) { | |||
4602 | NewAI = &AI; | |||
4603 | // FIXME: We should be able to bail at this point with "nothing changed". | |||
4604 | // FIXME: We might want to defer PHI speculation until after here. | |||
4605 | // FIXME: return nullptr; | |||
4606 | } else { | |||
4607 | // Make sure the alignment is compatible with P.beginOffset(). | |||
4608 | const Align Alignment = commonAlignment(AI.getAlign(), P.beginOffset()); | |||
4609 | // If we will get at least this much alignment from the type alone, leave | |||
4610 | // the alloca's alignment unconstrained. | |||
4611 | const bool IsUnconstrained = Alignment <= DL.getABITypeAlign(SliceTy); | |||
4612 | NewAI = new AllocaInst( | |||
4613 | SliceTy, AI.getAddressSpace(), nullptr, | |||
4614 | IsUnconstrained ? DL.getPrefTypeAlign(SliceTy) : Alignment, | |||
4615 | AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()), &AI); | |||
4616 | // Copy the old AI debug location over to the new one. | |||
4617 | NewAI->setDebugLoc(AI.getDebugLoc()); | |||
4618 | ++NumNewAllocas; | |||
4619 | } | |||
4620 | ||||
4621 | LLVM_DEBUG(dbgs() << "Rewriting alloca partition "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false) | |||
4622 | << "[" << P.beginOffset() << "," << P.endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false) | |||
4623 | << ") to: " << *NewAI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false); | |||
4624 | ||||
4625 | // Track the high watermark on the worklist as it is only relevant for | |||
4626 | // promoted allocas. We will reset it to this point if the alloca is not in | |||
4627 | // fact scheduled for promotion. | |||
4628 | unsigned PPWOldSize = PostPromotionWorklist.size(); | |||
4629 | unsigned NumUses = 0; | |||
4630 | SmallSetVector<PHINode *, 8> PHIUsers; | |||
4631 | SmallSetVector<SelectInst *, 8> SelectUsers; | |||
4632 | ||||
4633 | AllocaSliceRewriter Rewriter(DL, AS, *this, AI, *NewAI, P.beginOffset(), | |||
4634 | P.endOffset(), IsIntegerPromotable, VecTy, | |||
4635 | PHIUsers, SelectUsers); | |||
4636 | bool Promotable = true; | |||
4637 | for (Slice *S : P.splitSliceTails()) { | |||
4638 | Promotable &= Rewriter.visit(S); | |||
4639 | ++NumUses; | |||
4640 | } | |||
4641 | for (Slice &S : P) { | |||
4642 | Promotable &= Rewriter.visit(&S); | |||
4643 | ++NumUses; | |||
4644 | } | |||
4645 | ||||
4646 | NumAllocaPartitionUses += NumUses; | |||
4647 | MaxUsesPerAllocaPartition.updateMax(NumUses); | |||
4648 | ||||
4649 | // Now that we've processed all the slices in the new partition, check if any | |||
4650 | // PHIs or Selects would block promotion. | |||
4651 | for (PHINode *PHI : PHIUsers) | |||
4652 | if (!isSafePHIToSpeculate(*PHI)) { | |||
4653 | Promotable = false; | |||
4654 | PHIUsers.clear(); | |||
4655 | SelectUsers.clear(); | |||
4656 | break; | |||
4657 | } | |||
4658 | ||||
4659 | SmallVector<std::pair<SelectInst *, RewriteableMemOps>, 2> | |||
4660 | NewSelectsToRewrite; | |||
4661 | NewSelectsToRewrite.reserve(SelectUsers.size()); | |||
4662 | for (SelectInst *Sel : SelectUsers) { | |||
4663 | std::optional<RewriteableMemOps> Ops = | |||
4664 | isSafeSelectToSpeculate(*Sel, PreserveCFG); | |||
4665 | if (!Ops) { | |||
4666 | Promotable = false; | |||
4667 | PHIUsers.clear(); | |||
4668 | SelectUsers.clear(); | |||
4669 | NewSelectsToRewrite.clear(); | |||
4670 | break; | |||
4671 | } | |||
4672 | NewSelectsToRewrite.emplace_back(std::make_pair(Sel, *Ops)); | |||
4673 | } | |||
4674 | ||||
4675 | if (Promotable) { | |||
4676 | for (Use *U : AS.getDeadUsesIfPromotable()) { | |||
4677 | auto *OldInst = dyn_cast<Instruction>(U->get()); | |||
4678 | Value::dropDroppableUse(*U); | |||
4679 | if (OldInst) | |||
4680 | if (isInstructionTriviallyDead(OldInst)) | |||
4681 | DeadInsts.push_back(OldInst); | |||
4682 | } | |||
4683 | if (PHIUsers.empty() && SelectUsers.empty()) { | |||
4684 | // Promote the alloca. | |||
4685 | PromotableAllocas.push_back(NewAI); | |||
4686 | } else { | |||
4687 | // If we have either PHIs or Selects to speculate, add them to those | |||
4688 | // worklists and re-queue the new alloca so that we promote in on the | |||
4689 | // next iteration. | |||
4690 | for (PHINode *PHIUser : PHIUsers) | |||
4691 | SpeculatablePHIs.insert(PHIUser); | |||
4692 | SelectsToRewrite.reserve(SelectsToRewrite.size() + | |||
4693 | NewSelectsToRewrite.size()); | |||
4694 | for (auto &&KV : llvm::make_range( | |||
4695 | std::make_move_iterator(NewSelectsToRewrite.begin()), | |||
4696 | std::make_move_iterator(NewSelectsToRewrite.end()))) | |||
4697 | SelectsToRewrite.insert(std::move(KV)); | |||
4698 | Worklist.insert(NewAI); | |||
4699 | } | |||
4700 | } else { | |||
4701 | // Drop any post-promotion work items if promotion didn't happen. | |||
4702 | while (PostPromotionWorklist.size() > PPWOldSize) | |||
4703 | PostPromotionWorklist.pop_back(); | |||
4704 | ||||
4705 | // We couldn't promote and we didn't create a new partition, nothing | |||
4706 | // happened. | |||
4707 | if (NewAI == &AI) | |||
4708 | return nullptr; | |||
4709 | ||||
4710 | // If we can't promote the alloca, iterate on it to check for new | |||
4711 | // refinements exposed by splitting the current alloca. Don't iterate on an | |||
4712 | // alloca which didn't actually change and didn't get promoted. | |||
4713 | Worklist.insert(NewAI); | |||
4714 | } | |||
4715 | ||||
4716 | return NewAI; | |||
4717 | } | |||
4718 | ||||
4719 | /// Walks the slices of an alloca and form partitions based on them, | |||
4720 | /// rewriting each of their uses. | |||
4721 | bool SROAPass::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { | |||
4722 | if (AS.begin() == AS.end()) | |||
4723 | return false; | |||
4724 | ||||
4725 | unsigned NumPartitions = 0; | |||
4726 | bool Changed = false; | |||
4727 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||
4728 | ||||
4729 | // First try to pre-split loads and stores. | |||
4730 | Changed |= presplitLoadsAndStores(AI, AS); | |||
4731 | ||||
4732 | // Now that we have identified any pre-splitting opportunities, | |||
4733 | // mark loads and stores unsplittable except for the following case. | |||
4734 | // We leave a slice splittable if all other slices are disjoint or fully | |||
4735 | // included in the slice, such as whole-alloca loads and stores. | |||
4736 | // If we fail to split these during pre-splitting, we want to force them | |||
4737 | // to be rewritten into a partition. | |||
4738 | bool IsSorted = true; | |||
4739 | ||||
4740 | uint64_t AllocaSize = | |||
4741 | DL.getTypeAllocSize(AI.getAllocatedType()).getFixedValue(); | |||
4742 | const uint64_t MaxBitVectorSize = 1024; | |||
4743 | if (AllocaSize <= MaxBitVectorSize) { | |||
4744 | // If a byte boundary is included in any load or store, a slice starting or | |||
4745 | // ending at the boundary is not splittable. | |||
4746 | SmallBitVector SplittableOffset(AllocaSize + 1, true); | |||
4747 | for (Slice &S : AS) | |||
4748 | for (unsigned O = S.beginOffset() + 1; | |||
4749 | O < S.endOffset() && O < AllocaSize; O++) | |||
4750 | SplittableOffset.reset(O); | |||
4751 | ||||
4752 | for (Slice &S : AS) { | |||
4753 | if (!S.isSplittable()) | |||
4754 | continue; | |||
4755 | ||||
4756 | if ((S.beginOffset() > AllocaSize || SplittableOffset[S.beginOffset()]) && | |||
4757 | (S.endOffset() > AllocaSize || SplittableOffset[S.endOffset()])) | |||
4758 | continue; | |||
4759 | ||||
4760 | if (isa<LoadInst>(S.getUse()->getUser()) || | |||
4761 | isa<StoreInst>(S.getUse()->getUser())) { | |||
4762 | S.makeUnsplittable(); | |||
4763 | IsSorted = false; | |||
4764 | } | |||
4765 | } | |||
4766 | } | |||
4767 | else { | |||
4768 | // We only allow whole-alloca splittable loads and stores | |||
4769 | // for a large alloca to avoid creating too large BitVector. | |||
4770 | for (Slice &S : AS) { | |||
4771 | if (!S.isSplittable()) | |||
4772 | continue; | |||
4773 | ||||
4774 | if (S.beginOffset() == 0 && S.endOffset() >= AllocaSize) | |||
4775 | continue; | |||
4776 | ||||
4777 | if (isa<LoadInst>(S.getUse()->getUser()) || | |||
4778 | isa<StoreInst>(S.getUse()->getUser())) { | |||
4779 | S.makeUnsplittable(); | |||
4780 | IsSorted = false; | |||
4781 | } | |||
4782 | } | |||
4783 | } | |||
4784 | ||||
4785 | if (!IsSorted) | |||
4786 | llvm::sort(AS); | |||
4787 | ||||
4788 | /// Describes the allocas introduced by rewritePartition in order to migrate | |||
4789 | /// the debug info. | |||
4790 | struct Fragment { | |||
4791 | AllocaInst *Alloca; | |||
4792 | uint64_t Offset; | |||
4793 | uint64_t Size; | |||
4794 | Fragment(AllocaInst *AI, uint64_t O, uint64_t S) | |||
4795 | : Alloca(AI), Offset(O), Size(S) {} | |||
4796 | }; | |||
4797 | SmallVector<Fragment, 4> Fragments; | |||
4798 | ||||
4799 | // Rewrite each partition. | |||
4800 | for (auto &P : AS.partitions()) { | |||
4801 | if (AllocaInst *NewAI = rewritePartition(AI, AS, P)) { | |||
4802 | Changed = true; | |||
4803 | if (NewAI != &AI) { | |||
4804 | uint64_t SizeOfByte = 8; | |||
4805 | uint64_t AllocaSize = | |||
4806 | DL.getTypeSizeInBits(NewAI->getAllocatedType()).getFixedValue(); | |||
4807 | // Don't include any padding. | |||
4808 | uint64_t Size = std::min(AllocaSize, P.size() * SizeOfByte); | |||
4809 | Fragments.push_back(Fragment(NewAI, P.beginOffset() * SizeOfByte, Size)); | |||
4810 | } | |||
4811 | } | |||
4812 | ++NumPartitions; | |||
4813 | } | |||
4814 | ||||
4815 | NumAllocaPartitions += NumPartitions; | |||
4816 | MaxPartitionsPerAlloca.updateMax(NumPartitions); | |||
4817 | ||||
4818 | // Migrate debug information from the old alloca to the new alloca(s) | |||
4819 | // and the individual partitions. | |||
4820 | TinyPtrVector<DbgVariableIntrinsic *> DbgVariables; | |||
4821 | for (auto *DbgDeclare : FindDbgDeclareUses(&AI)) | |||
4822 | DbgVariables.push_back(DbgDeclare); | |||
4823 | for (auto *DbgAssign : at::getAssignmentMarkers(&AI)) | |||
4824 | DbgVariables.push_back(DbgAssign); | |||
4825 | for (DbgVariableIntrinsic *DbgVariable : DbgVariables) { | |||
4826 | auto *Expr = DbgVariable->getExpression(); | |||
4827 | DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false); | |||
4828 | uint64_t AllocaSize = | |||
4829 | DL.getTypeSizeInBits(AI.getAllocatedType()).getFixedValue(); | |||
4830 | for (auto Fragment : Fragments) { | |||
4831 | // Create a fragment expression describing the new partition or reuse AI's | |||
4832 | // expression if there is only one partition. | |||
4833 | auto *FragmentExpr = Expr; | |||
4834 | if (Fragment.Size < AllocaSize || Expr->isFragment()) { | |||
4835 | // If this alloca is already a scalar replacement of a larger aggregate, | |||
4836 | // Fragment.Offset describes the offset inside the scalar. | |||
4837 | auto ExprFragment = Expr->getFragmentInfo(); | |||
4838 | uint64_t Offset = ExprFragment ? ExprFragment->OffsetInBits : 0; | |||
4839 | uint64_t Start = Offset + Fragment.Offset; | |||
4840 | uint64_t Size = Fragment.Size; | |||
4841 | if (ExprFragment) { | |||
4842 | uint64_t AbsEnd = | |||
4843 | ExprFragment->OffsetInBits + ExprFragment->SizeInBits; | |||
4844 | if (Start >= AbsEnd) { | |||
4845 | // No need to describe a SROAed padding. | |||
4846 | continue; | |||
4847 | } | |||
4848 | Size = std::min(Size, AbsEnd - Start); | |||
4849 | } | |||
4850 | // The new, smaller fragment is stenciled out from the old fragment. | |||
4851 | if (auto OrigFragment = FragmentExpr->getFragmentInfo()) { | |||
4852 | assert(Start >= OrigFragment->OffsetInBits &&(static_cast <bool> (Start >= OrigFragment->OffsetInBits && "new fragment is outside of original fragment") ? void (0) : __assert_fail ("Start >= OrigFragment->OffsetInBits && \"new fragment is outside of original fragment\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4853, __extension__ __PRETTY_FUNCTION__ )) | |||
4853 | "new fragment is outside of original fragment")(static_cast <bool> (Start >= OrigFragment->OffsetInBits && "new fragment is outside of original fragment") ? void (0) : __assert_fail ("Start >= OrigFragment->OffsetInBits && \"new fragment is outside of original fragment\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 4853, __extension__ __PRETTY_FUNCTION__ )); | |||
4854 | Start -= OrigFragment->OffsetInBits; | |||
4855 | } | |||
4856 | ||||
4857 | // The alloca may be larger than the variable. | |||
4858 | auto VarSize = DbgVariable->getVariable()->getSizeInBits(); | |||
4859 | if (VarSize) { | |||
4860 | if (Size > *VarSize) | |||
4861 | Size = *VarSize; | |||
4862 | if (Size == 0 || Start + Size > *VarSize) | |||
4863 | continue; | |||
4864 | } | |||
4865 | ||||
4866 | // Avoid creating a fragment expression that covers the entire variable. | |||
4867 | if (!VarSize || *VarSize != Size) { | |||
4868 | if (auto E = | |||
4869 | DIExpression::createFragmentExpression(Expr, Start, Size)) | |||
4870 | FragmentExpr = *E; | |||
4871 | else | |||
4872 | continue; | |||
4873 | } | |||
4874 | } | |||
4875 | ||||
4876 | // Remove any existing intrinsics on the new alloca describing | |||
4877 | // the variable fragment. | |||
4878 | for (DbgDeclareInst *OldDII : FindDbgDeclareUses(Fragment.Alloca)) { | |||
4879 | auto SameVariableFragment = [](const DbgVariableIntrinsic *LHS, | |||
4880 | const DbgVariableIntrinsic *RHS) { | |||
4881 | return LHS->getVariable() == RHS->getVariable() && | |||
4882 | LHS->getDebugLoc()->getInlinedAt() == | |||
4883 | RHS->getDebugLoc()->getInlinedAt(); | |||
4884 | }; | |||
4885 | if (SameVariableFragment(OldDII, DbgVariable)) | |||
4886 | OldDII->eraseFromParent(); | |||
4887 | } | |||
4888 | ||||
4889 | if (auto *DbgAssign = dyn_cast<DbgAssignIntrinsic>(DbgVariable)) { | |||
4890 | if (!Fragment.Alloca->hasMetadata(LLVMContext::MD_DIAssignID)) { | |||
4891 | Fragment.Alloca->setMetadata( | |||
4892 | LLVMContext::MD_DIAssignID, | |||
4893 | DIAssignID::getDistinct(AI.getContext())); | |||
4894 | } | |||
4895 | auto *NewAssign = DIB.insertDbgAssign( | |||
4896 | Fragment.Alloca, DbgAssign->getValue(), DbgAssign->getVariable(), | |||
4897 | FragmentExpr, Fragment.Alloca, DbgAssign->getAddressExpression(), | |||
4898 | DbgAssign->getDebugLoc()); | |||
4899 | NewAssign->setDebugLoc(DbgAssign->getDebugLoc()); | |||
4900 | LLVM_DEBUG(dbgs() << "Created new assign intrinsic: " << *NewAssigndo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Created new assign intrinsic: " << *NewAssign << "\n"; } } while (false) | |||
4901 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Created new assign intrinsic: " << *NewAssign << "\n"; } } while (false); | |||
4902 | } else { | |||
4903 | DIB.insertDeclare(Fragment.Alloca, DbgVariable->getVariable(), | |||
4904 | FragmentExpr, DbgVariable->getDebugLoc(), &AI); | |||
4905 | } | |||
4906 | } | |||
4907 | } | |||
4908 | return Changed; | |||
4909 | } | |||
4910 | ||||
4911 | /// Clobber a use with poison, deleting the used value if it becomes dead. | |||
4912 | void SROAPass::clobberUse(Use &U) { | |||
4913 | Value *OldV = U; | |||
4914 | // Replace the use with an poison value. | |||
4915 | U = PoisonValue::get(OldV->getType()); | |||
4916 | ||||
4917 | // Check for this making an instruction dead. We have to garbage collect | |||
4918 | // all the dead instructions to ensure the uses of any alloca end up being | |||
4919 | // minimal. | |||
4920 | if (Instruction *OldI = dyn_cast<Instruction>(OldV)) | |||
4921 | if (isInstructionTriviallyDead(OldI)) { | |||
4922 | DeadInsts.push_back(OldI); | |||
4923 | } | |||
4924 | } | |||
4925 | ||||
4926 | /// Analyze an alloca for SROA. | |||
4927 | /// | |||
4928 | /// This analyzes the alloca to ensure we can reason about it, builds | |||
4929 | /// the slices of the alloca, and then hands it off to be split and | |||
4930 | /// rewritten as needed. | |||
4931 | std::pair<bool /*Changed*/, bool /*CFGChanged*/> | |||
4932 | SROAPass::runOnAlloca(AllocaInst &AI) { | |||
4933 | bool Changed = false; | |||
4934 | bool CFGChanged = false; | |||
4935 | ||||
4936 | LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "SROA alloca: " << AI << "\n"; } } while (false); | |||
4937 | ++NumAllocasAnalyzed; | |||
4938 | ||||
4939 | // Special case dead allocas, as they're trivial. | |||
4940 | if (AI.use_empty()) { | |||
4941 | AI.eraseFromParent(); | |||
4942 | Changed = true; | |||
4943 | return {Changed, CFGChanged}; | |||
4944 | } | |||
4945 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||
4946 | ||||
4947 | // Skip alloca forms that this analysis can't handle. | |||
4948 | auto *AT = AI.getAllocatedType(); | |||
4949 | TypeSize Size = DL.getTypeAllocSize(AT); | |||
4950 | if (AI.isArrayAllocation() || !AT->isSized() || Size.isScalable() || | |||
4951 | Size.getFixedValue() == 0) | |||
4952 | return {Changed, CFGChanged}; | |||
4953 | ||||
4954 | // First, split any FCA loads and stores touching this alloca to promote | |||
4955 | // better splitting and promotion opportunities. | |||
4956 | IRBuilderTy IRB(&AI); | |||
4957 | AggLoadStoreRewriter AggRewriter(DL, IRB); | |||
4958 | Changed |= AggRewriter.rewrite(AI); | |||
4959 | ||||
4960 | // Build the slices using a recursive instruction-visiting builder. | |||
4961 | AllocaSlices AS(DL, AI); | |||
4962 | LLVM_DEBUG(AS.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.print(dbgs()); } } while (false); | |||
4963 | if (AS.isEscaped()) | |||
4964 | return {Changed, CFGChanged}; | |||
4965 | ||||
4966 | // Delete all the dead users of this alloca before splitting and rewriting it. | |||
4967 | for (Instruction *DeadUser : AS.getDeadUsers()) { | |||
4968 | // Free up everything used by this instruction. | |||
4969 | for (Use &DeadOp : DeadUser->operands()) | |||
4970 | clobberUse(DeadOp); | |||
4971 | ||||
4972 | // Now replace the uses of this instruction. | |||
4973 | DeadUser->replaceAllUsesWith(PoisonValue::get(DeadUser->getType())); | |||
4974 | ||||
4975 | // And mark it for deletion. | |||
4976 | DeadInsts.push_back(DeadUser); | |||
4977 | Changed = true; | |||
4978 | } | |||
4979 | for (Use *DeadOp : AS.getDeadOperands()) { | |||
4980 | clobberUse(*DeadOp); | |||
4981 | Changed = true; | |||
4982 | } | |||
4983 | ||||
4984 | // No slices to split. Leave the dead alloca for a later pass to clean up. | |||
4985 | if (AS.begin() == AS.end()) | |||
4986 | return {Changed, CFGChanged}; | |||
4987 | ||||
4988 | Changed |= splitAlloca(AI, AS); | |||
4989 | ||||
4990 | LLVM_DEBUG(dbgs() << " Speculating PHIs\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Speculating PHIs\n"; } } while (false); | |||
4991 | while (!SpeculatablePHIs.empty()) | |||
4992 | speculatePHINodeLoads(IRB, *SpeculatablePHIs.pop_back_val()); | |||
4993 | ||||
4994 | LLVM_DEBUG(dbgs() << " Rewriting Selects\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting Selects\n"; } } while (false); | |||
4995 | auto RemainingSelectsToRewrite = SelectsToRewrite.takeVector(); | |||
4996 | while (!RemainingSelectsToRewrite.empty()) { | |||
4997 | const auto [K, V] = RemainingSelectsToRewrite.pop_back_val(); | |||
4998 | CFGChanged |= | |||
4999 | rewriteSelectInstMemOps(*K, V, IRB, PreserveCFG ? nullptr : DTU); | |||
5000 | } | |||
5001 | ||||
5002 | return {Changed, CFGChanged}; | |||
5003 | } | |||
5004 | ||||
5005 | /// Delete the dead instructions accumulated in this run. | |||
5006 | /// | |||
5007 | /// Recursively deletes the dead instructions we've accumulated. This is done | |||
5008 | /// at the very end to maximize locality of the recursive delete and to | |||
5009 | /// minimize the problems of invalidated instruction pointers as such pointers | |||
5010 | /// are used heavily in the intermediate stages of the algorithm. | |||
5011 | /// | |||
5012 | /// We also record the alloca instructions deleted here so that they aren't | |||
5013 | /// subsequently handed to mem2reg to promote. | |||
5014 | bool SROAPass::deleteDeadInstructions( | |||
5015 | SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) { | |||
5016 | bool Changed = false; | |||
5017 | while (!DeadInsts.empty()) { | |||
5018 | Instruction *I = dyn_cast_or_null<Instruction>(DeadInsts.pop_back_val()); | |||
5019 | if (!I) | |||
5020 | continue; | |||
5021 | LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Deleting dead instruction: " << *I << "\n"; } } while (false); | |||
5022 | ||||
5023 | // If the instruction is an alloca, find the possible dbg.declare connected | |||
5024 | // to it, and remove it too. We must do this before calling RAUW or we will | |||
5025 | // not be able to find it. | |||
5026 | if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) { | |||
5027 | DeletedAllocas.insert(AI); | |||
5028 | for (DbgDeclareInst *OldDII : FindDbgDeclareUses(AI)) | |||
5029 | OldDII->eraseFromParent(); | |||
5030 | } | |||
5031 | ||||
5032 | at::deleteAssignmentMarkers(I); | |||
5033 | I->replaceAllUsesWith(UndefValue::get(I->getType())); | |||
5034 | ||||
5035 | for (Use &Operand : I->operands()) | |||
5036 | if (Instruction *U = dyn_cast<Instruction>(Operand)) { | |||
5037 | // Zero out the operand and see if it becomes trivially dead. | |||
5038 | Operand = nullptr; | |||
5039 | if (isInstructionTriviallyDead(U)) | |||
5040 | DeadInsts.push_back(U); | |||
5041 | } | |||
5042 | ||||
5043 | ++NumDeleted; | |||
5044 | I->eraseFromParent(); | |||
5045 | Changed = true; | |||
5046 | } | |||
5047 | return Changed; | |||
5048 | } | |||
5049 | ||||
5050 | /// Promote the allocas, using the best available technique. | |||
5051 | /// | |||
5052 | /// This attempts to promote whatever allocas have been identified as viable in | |||
5053 | /// the PromotableAllocas list. If that list is empty, there is nothing to do. | |||
5054 | /// This function returns whether any promotion occurred. | |||
5055 | bool SROAPass::promoteAllocas(Function &F) { | |||
5056 | if (PromotableAllocas.empty()) | |||
5057 | return false; | |||
5058 | ||||
5059 | NumPromoted += PromotableAllocas.size(); | |||
5060 | ||||
5061 | if (SROASkipMem2Reg) { | |||
5062 | LLVM_DEBUG(dbgs() << "Not promoting allocas with mem2reg!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Not promoting allocas with mem2reg!\n" ; } } while (false); | |||
5063 | } else { | |||
5064 | LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Promoting allocas with mem2reg...\n" ; } } while (false); | |||
5065 | PromoteMemToReg(PromotableAllocas, DTU->getDomTree(), AC); | |||
5066 | } | |||
5067 | ||||
5068 | PromotableAllocas.clear(); | |||
5069 | return true; | |||
5070 | } | |||
5071 | ||||
5072 | PreservedAnalyses SROAPass::runImpl(Function &F, DomTreeUpdater &RunDTU, | |||
5073 | AssumptionCache &RunAC) { | |||
5074 | LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "SROA function: " << F.getName () << "\n"; } } while (false); | |||
5075 | C = &F.getContext(); | |||
5076 | DTU = &RunDTU; | |||
5077 | AC = &RunAC; | |||
5078 | ||||
5079 | const DataLayout &DL = F.getParent()->getDataLayout(); | |||
5080 | BasicBlock &EntryBB = F.getEntryBlock(); | |||
5081 | for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(EntryBB.end()); | |||
5082 | I != E; ++I) { | |||
5083 | if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) { | |||
5084 | if (DL.getTypeAllocSize(AI->getAllocatedType()).isScalable() && | |||
5085 | isAllocaPromotable(AI)) | |||
5086 | PromotableAllocas.push_back(AI); | |||
5087 | else | |||
5088 | Worklist.insert(AI); | |||
5089 | } | |||
5090 | } | |||
5091 | ||||
5092 | bool Changed = false; | |||
5093 | bool CFGChanged = false; | |||
5094 | // A set of deleted alloca instruction pointers which should be removed from | |||
5095 | // the list of promotable allocas. | |||
5096 | SmallPtrSet<AllocaInst *, 4> DeletedAllocas; | |||
5097 | ||||
5098 | do { | |||
5099 | while (!Worklist.empty()) { | |||
5100 | auto [IterationChanged, IterationCFGChanged] = | |||
5101 | runOnAlloca(*Worklist.pop_back_val()); | |||
5102 | Changed |= IterationChanged; | |||
5103 | CFGChanged |= IterationCFGChanged; | |||
5104 | ||||
5105 | Changed |= deleteDeadInstructions(DeletedAllocas); | |||
5106 | ||||
5107 | // Remove the deleted allocas from various lists so that we don't try to | |||
5108 | // continue processing them. | |||
5109 | if (!DeletedAllocas.empty()) { | |||
5110 | auto IsInSet = [&](AllocaInst *AI) { return DeletedAllocas.count(AI); }; | |||
5111 | Worklist.remove_if(IsInSet); | |||
5112 | PostPromotionWorklist.remove_if(IsInSet); | |||
5113 | llvm::erase_if(PromotableAllocas, IsInSet); | |||
5114 | DeletedAllocas.clear(); | |||
5115 | } | |||
5116 | } | |||
5117 | ||||
5118 | Changed |= promoteAllocas(F); | |||
5119 | ||||
5120 | Worklist = PostPromotionWorklist; | |||
5121 | PostPromotionWorklist.clear(); | |||
5122 | } while (!Worklist.empty()); | |||
5123 | ||||
5124 | assert((!CFGChanged || Changed) && "Can not only modify the CFG.")(static_cast <bool> ((!CFGChanged || Changed) && "Can not only modify the CFG.") ? void (0) : __assert_fail ( "(!CFGChanged || Changed) && \"Can not only modify the CFG.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 5124, __extension__ __PRETTY_FUNCTION__ )); | |||
5125 | assert((!CFGChanged || !PreserveCFG) &&(static_cast <bool> ((!CFGChanged || !PreserveCFG) && "Should not have modified the CFG when told to preserve it." ) ? void (0) : __assert_fail ("(!CFGChanged || !PreserveCFG) && \"Should not have modified the CFG when told to preserve it.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 5126, __extension__ __PRETTY_FUNCTION__ )) | |||
5126 | "Should not have modified the CFG when told to preserve it.")(static_cast <bool> ((!CFGChanged || !PreserveCFG) && "Should not have modified the CFG when told to preserve it." ) ? void (0) : __assert_fail ("(!CFGChanged || !PreserveCFG) && \"Should not have modified the CFG when told to preserve it.\"" , "llvm/lib/Transforms/Scalar/SROA.cpp", 5126, __extension__ __PRETTY_FUNCTION__ )); | |||
5127 | ||||
5128 | if (!Changed) | |||
5129 | return PreservedAnalyses::all(); | |||
5130 | ||||
5131 | if (isAssignmentTrackingEnabled(*F.getParent())) { | |||
5132 | for (auto &BB : F) | |||
5133 | RemoveRedundantDbgInstrs(&BB); | |||
5134 | } | |||
5135 | ||||
5136 | PreservedAnalyses PA; | |||
5137 | if (!CFGChanged) | |||
5138 | PA.preserveSet<CFGAnalyses>(); | |||
5139 | PA.preserve<DominatorTreeAnalysis>(); | |||
5140 | return PA; | |||
5141 | } | |||
5142 | ||||
5143 | PreservedAnalyses SROAPass::runImpl(Function &F, DominatorTree &RunDT, | |||
5144 | AssumptionCache &RunAC) { | |||
5145 | DomTreeUpdater DTU(RunDT, DomTreeUpdater::UpdateStrategy::Lazy); | |||
5146 | return runImpl(F, DTU, RunAC); | |||
5147 | } | |||
5148 | ||||
5149 | PreservedAnalyses SROAPass::run(Function &F, FunctionAnalysisManager &AM) { | |||
5150 | DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(F); | |||
5151 | AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(F); | |||
5152 | return runImpl(F, DT, AC); | |||
5153 | } | |||
5154 | ||||
5155 | void SROAPass::printPipeline( | |||
5156 | raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) { | |||
5157 | static_cast<PassInfoMixin<SROAPass> *>(this)->printPipeline( | |||
5158 | OS, MapClassName2PassName); | |||
5159 | OS << (PreserveCFG ? "<preserve-cfg>" : "<modify-cfg>"); | |||
5160 | } | |||
5161 | ||||
5162 | SROAPass::SROAPass(SROAOptions PreserveCFG_) | |||
5163 | : PreserveCFG(PreserveCFG_ == SROAOptions::PreserveCFG) {} | |||
5164 | ||||
5165 | /// A legacy pass for the legacy pass manager that wraps the \c SROA pass. | |||
5166 | /// | |||
5167 | /// This is in the llvm namespace purely to allow it to be a friend of the \c | |||
5168 | /// SROA pass. | |||
5169 | class llvm::sroa::SROALegacyPass : public FunctionPass { | |||
5170 | /// The SROA implementation. | |||
5171 | SROAPass Impl; | |||
5172 | ||||
5173 | public: | |||
5174 | static char ID; | |||
5175 | ||||
5176 | SROALegacyPass(SROAOptions PreserveCFG = SROAOptions::PreserveCFG) | |||
5177 | : FunctionPass(ID), Impl(PreserveCFG) { | |||
5178 | initializeSROALegacyPassPass(*PassRegistry::getPassRegistry()); | |||
5179 | } | |||
5180 | ||||
5181 | bool runOnFunction(Function &F) override { | |||
5182 | if (skipFunction(F)) | |||
5183 | return false; | |||
5184 | ||||
5185 | auto PA = Impl.runImpl( | |||
5186 | F, getAnalysis<DominatorTreeWrapperPass>().getDomTree(), | |||
5187 | getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F)); | |||
5188 | return !PA.areAllPreserved(); | |||
5189 | } | |||
5190 | ||||
5191 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
5192 | AU.addRequired<AssumptionCacheTracker>(); | |||
5193 | AU.addRequired<DominatorTreeWrapperPass>(); | |||
5194 | AU.addPreserved<GlobalsAAWrapperPass>(); | |||
5195 | AU.addPreserved<DominatorTreeWrapperPass>(); | |||
5196 | } | |||
5197 | ||||
5198 | StringRef getPassName() const override { return "SROA"; } | |||
5199 | }; | |||
5200 | ||||
5201 | char SROALegacyPass::ID = 0; | |||
5202 | ||||
5203 | FunctionPass *llvm::createSROAPass(bool PreserveCFG) { | |||
5204 | return new SROALegacyPass(PreserveCFG ? SROAOptions::PreserveCFG | |||
5205 | : SROAOptions::ModifyCFG); | |||
5206 | } | |||
5207 | ||||
5208 | INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",static void *initializeSROALegacyPassPassOnce(PassRegistry & Registry) { | |||
5209 | "Scalar Replacement Of Aggregates", false, false)static void *initializeSROALegacyPassPassOnce(PassRegistry & Registry) { | |||
5210 | INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry); | |||
5211 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); | |||
5212 | INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",PassInfo *PI = new PassInfo( "Scalar Replacement Of Aggregates" , "sroa", &SROALegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <SROALegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeSROALegacyPassPassFlag ; void llvm::initializeSROALegacyPassPass(PassRegistry &Registry ) { llvm::call_once(InitializeSROALegacyPassPassFlag, initializeSROALegacyPassPassOnce , std::ref(Registry)); } | |||
5213 | false, false)PassInfo *PI = new PassInfo( "Scalar Replacement Of Aggregates" , "sroa", &SROALegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <SROALegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeSROALegacyPassPassFlag ; void llvm::initializeSROALegacyPassPass(PassRegistry &Registry ) { llvm::call_once(InitializeSROALegacyPassPassFlag, initializeSROALegacyPassPassOnce , std::ref(Registry)); } |