File: | llvm/lib/Transforms/Scalar/SROA.cpp |
Warning: | line 2956, column 54 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | /// \file | |||
9 | /// This transformation implements the well known scalar replacement of | |||
10 | /// aggregates transformation. It tries to identify promotable elements of an | |||
11 | /// aggregate alloca, and promote them to registers. It will also try to | |||
12 | /// convert uses of an element (or set of elements) of an alloca into a vector | |||
13 | /// or bitfield-style integer scalar if appropriate. | |||
14 | /// | |||
15 | /// It works to do this with minimal slicing of the alloca so that regions | |||
16 | /// which are merely transferred in and out of external memory remain unchanged | |||
17 | /// and are not decomposed to scalar code. | |||
18 | /// | |||
19 | /// Because this also performs alloca promotion, it can be thought of as also | |||
20 | /// serving the purpose of SSA formation. The algorithm iterates on the | |||
21 | /// function until all opportunities for promotion have been realized. | |||
22 | /// | |||
23 | //===----------------------------------------------------------------------===// | |||
24 | ||||
25 | #include "llvm/Transforms/Scalar/SROA.h" | |||
26 | #include "llvm/ADT/APInt.h" | |||
27 | #include "llvm/ADT/ArrayRef.h" | |||
28 | #include "llvm/ADT/DenseMap.h" | |||
29 | #include "llvm/ADT/PointerIntPair.h" | |||
30 | #include "llvm/ADT/STLExtras.h" | |||
31 | #include "llvm/ADT/SetVector.h" | |||
32 | #include "llvm/ADT/SmallBitVector.h" | |||
33 | #include "llvm/ADT/SmallPtrSet.h" | |||
34 | #include "llvm/ADT/SmallVector.h" | |||
35 | #include "llvm/ADT/Statistic.h" | |||
36 | #include "llvm/ADT/StringRef.h" | |||
37 | #include "llvm/ADT/Twine.h" | |||
38 | #include "llvm/ADT/iterator.h" | |||
39 | #include "llvm/ADT/iterator_range.h" | |||
40 | #include "llvm/Analysis/AssumptionCache.h" | |||
41 | #include "llvm/Analysis/GlobalsModRef.h" | |||
42 | #include "llvm/Analysis/Loads.h" | |||
43 | #include "llvm/Analysis/PtrUseVisitor.h" | |||
44 | #include "llvm/Config/llvm-config.h" | |||
45 | #include "llvm/IR/BasicBlock.h" | |||
46 | #include "llvm/IR/Constant.h" | |||
47 | #include "llvm/IR/ConstantFolder.h" | |||
48 | #include "llvm/IR/Constants.h" | |||
49 | #include "llvm/IR/DIBuilder.h" | |||
50 | #include "llvm/IR/DataLayout.h" | |||
51 | #include "llvm/IR/DebugInfoMetadata.h" | |||
52 | #include "llvm/IR/DerivedTypes.h" | |||
53 | #include "llvm/IR/Dominators.h" | |||
54 | #include "llvm/IR/Function.h" | |||
55 | #include "llvm/IR/GetElementPtrTypeIterator.h" | |||
56 | #include "llvm/IR/GlobalAlias.h" | |||
57 | #include "llvm/IR/IRBuilder.h" | |||
58 | #include "llvm/IR/InstVisitor.h" | |||
59 | #include "llvm/IR/InstrTypes.h" | |||
60 | #include "llvm/IR/Instruction.h" | |||
61 | #include "llvm/IR/Instructions.h" | |||
62 | #include "llvm/IR/IntrinsicInst.h" | |||
63 | #include "llvm/IR/Intrinsics.h" | |||
64 | #include "llvm/IR/LLVMContext.h" | |||
65 | #include "llvm/IR/Metadata.h" | |||
66 | #include "llvm/IR/Module.h" | |||
67 | #include "llvm/IR/Operator.h" | |||
68 | #include "llvm/IR/PassManager.h" | |||
69 | #include "llvm/IR/Type.h" | |||
70 | #include "llvm/IR/Use.h" | |||
71 | #include "llvm/IR/User.h" | |||
72 | #include "llvm/IR/Value.h" | |||
73 | #include "llvm/InitializePasses.h" | |||
74 | #include "llvm/Pass.h" | |||
75 | #include "llvm/Support/Casting.h" | |||
76 | #include "llvm/Support/CommandLine.h" | |||
77 | #include "llvm/Support/Compiler.h" | |||
78 | #include "llvm/Support/Debug.h" | |||
79 | #include "llvm/Support/ErrorHandling.h" | |||
80 | #include "llvm/Support/MathExtras.h" | |||
81 | #include "llvm/Support/raw_ostream.h" | |||
82 | #include "llvm/Transforms/Scalar.h" | |||
83 | #include "llvm/Transforms/Utils/Local.h" | |||
84 | #include "llvm/Transforms/Utils/PromoteMemToReg.h" | |||
85 | #include <algorithm> | |||
86 | #include <cassert> | |||
87 | #include <chrono> | |||
88 | #include <cstddef> | |||
89 | #include <cstdint> | |||
90 | #include <cstring> | |||
91 | #include <iterator> | |||
92 | #include <string> | |||
93 | #include <tuple> | |||
94 | #include <utility> | |||
95 | #include <vector> | |||
96 | ||||
97 | using namespace llvm; | |||
98 | using namespace llvm::sroa; | |||
99 | ||||
100 | #define DEBUG_TYPE"sroa" "sroa" | |||
101 | ||||
102 | STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement")static llvm::Statistic NumAllocasAnalyzed = {"sroa", "NumAllocasAnalyzed" , "Number of allocas analyzed for replacement"}; | |||
103 | STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed")static llvm::Statistic NumAllocaPartitions = {"sroa", "NumAllocaPartitions" , "Number of alloca partitions formed"}; | |||
104 | STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca")static llvm::Statistic MaxPartitionsPerAlloca = {"sroa", "MaxPartitionsPerAlloca" , "Maximum number of partitions per alloca"}; | |||
105 | STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten")static llvm::Statistic NumAllocaPartitionUses = {"sroa", "NumAllocaPartitionUses" , "Number of alloca partition uses rewritten"}; | |||
106 | STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition")static llvm::Statistic MaxUsesPerAllocaPartition = {"sroa", "MaxUsesPerAllocaPartition" , "Maximum number of uses of a partition"}; | |||
107 | STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced")static llvm::Statistic NumNewAllocas = {"sroa", "NumNewAllocas" , "Number of new, smaller allocas introduced"}; | |||
108 | STATISTIC(NumPromoted, "Number of allocas promoted to SSA values")static llvm::Statistic NumPromoted = {"sroa", "NumPromoted", "Number of allocas promoted to SSA values" }; | |||
109 | STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion")static llvm::Statistic NumLoadsSpeculated = {"sroa", "NumLoadsSpeculated" , "Number of loads speculated to allow promotion"}; | |||
110 | STATISTIC(NumDeleted, "Number of instructions deleted")static llvm::Statistic NumDeleted = {"sroa", "NumDeleted", "Number of instructions deleted" }; | |||
111 | STATISTIC(NumVectorized, "Number of vectorized aggregates")static llvm::Statistic NumVectorized = {"sroa", "NumVectorized" , "Number of vectorized aggregates"}; | |||
112 | ||||
113 | /// Hidden option to experiment with completely strict handling of inbounds | |||
114 | /// GEPs. | |||
115 | static cl::opt<bool> SROAStrictInbounds("sroa-strict-inbounds", cl::init(false), | |||
116 | cl::Hidden); | |||
117 | ||||
118 | namespace { | |||
119 | ||||
120 | /// A custom IRBuilder inserter which prefixes all names, but only in | |||
121 | /// Assert builds. | |||
122 | class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter { | |||
123 | std::string Prefix; | |||
124 | ||||
125 | Twine getNameWithPrefix(const Twine &Name) const { | |||
126 | return Name.isTriviallyEmpty() ? Name : Prefix + Name; | |||
127 | } | |||
128 | ||||
129 | public: | |||
130 | void SetNamePrefix(const Twine &P) { Prefix = P.str(); } | |||
131 | ||||
132 | void InsertHelper(Instruction *I, const Twine &Name, BasicBlock *BB, | |||
133 | BasicBlock::iterator InsertPt) const override { | |||
134 | IRBuilderDefaultInserter::InsertHelper(I, getNameWithPrefix(Name), BB, | |||
135 | InsertPt); | |||
136 | } | |||
137 | }; | |||
138 | ||||
139 | /// Provide a type for IRBuilder that drops names in release builds. | |||
140 | using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>; | |||
141 | ||||
142 | /// A used slice of an alloca. | |||
143 | /// | |||
144 | /// This structure represents a slice of an alloca used by some instruction. It | |||
145 | /// stores both the begin and end offsets of this use, a pointer to the use | |||
146 | /// itself, and a flag indicating whether we can classify the use as splittable | |||
147 | /// or not when forming partitions of the alloca. | |||
148 | class Slice { | |||
149 | /// The beginning offset of the range. | |||
150 | uint64_t BeginOffset = 0; | |||
151 | ||||
152 | /// The ending offset, not included in the range. | |||
153 | uint64_t EndOffset = 0; | |||
154 | ||||
155 | /// Storage for both the use of this slice and whether it can be | |||
156 | /// split. | |||
157 | PointerIntPair<Use *, 1, bool> UseAndIsSplittable; | |||
158 | ||||
159 | public: | |||
160 | Slice() = default; | |||
161 | ||||
162 | Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable) | |||
163 | : BeginOffset(BeginOffset), EndOffset(EndOffset), | |||
164 | UseAndIsSplittable(U, IsSplittable) {} | |||
165 | ||||
166 | uint64_t beginOffset() const { return BeginOffset; } | |||
167 | uint64_t endOffset() const { return EndOffset; } | |||
168 | ||||
169 | bool isSplittable() const { return UseAndIsSplittable.getInt(); } | |||
170 | void makeUnsplittable() { UseAndIsSplittable.setInt(false); } | |||
171 | ||||
172 | Use *getUse() const { return UseAndIsSplittable.getPointer(); } | |||
173 | ||||
174 | bool isDead() const { return getUse() == nullptr; } | |||
175 | void kill() { UseAndIsSplittable.setPointer(nullptr); } | |||
176 | ||||
177 | /// Support for ordering ranges. | |||
178 | /// | |||
179 | /// This provides an ordering over ranges such that start offsets are | |||
180 | /// always increasing, and within equal start offsets, the end offsets are | |||
181 | /// decreasing. Thus the spanning range comes first in a cluster with the | |||
182 | /// same start position. | |||
183 | bool operator<(const Slice &RHS) const { | |||
184 | if (beginOffset() < RHS.beginOffset()) | |||
185 | return true; | |||
186 | if (beginOffset() > RHS.beginOffset()) | |||
187 | return false; | |||
188 | if (isSplittable() != RHS.isSplittable()) | |||
189 | return !isSplittable(); | |||
190 | if (endOffset() > RHS.endOffset()) | |||
191 | return true; | |||
192 | return false; | |||
193 | } | |||
194 | ||||
195 | /// Support comparison with a single offset to allow binary searches. | |||
196 | friend LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__)) bool operator<(const Slice &LHS, | |||
197 | uint64_t RHSOffset) { | |||
198 | return LHS.beginOffset() < RHSOffset; | |||
199 | } | |||
200 | friend LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__)) bool operator<(uint64_t LHSOffset, | |||
201 | const Slice &RHS) { | |||
202 | return LHSOffset < RHS.beginOffset(); | |||
203 | } | |||
204 | ||||
205 | bool operator==(const Slice &RHS) const { | |||
206 | return isSplittable() == RHS.isSplittable() && | |||
207 | beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset(); | |||
208 | } | |||
209 | bool operator!=(const Slice &RHS) const { return !operator==(RHS); } | |||
210 | }; | |||
211 | ||||
212 | } // end anonymous namespace | |||
213 | ||||
214 | /// Representation of the alloca slices. | |||
215 | /// | |||
216 | /// This class represents the slices of an alloca which are formed by its | |||
217 | /// various uses. If a pointer escapes, we can't fully build a representation | |||
218 | /// for the slices used and we reflect that in this structure. The uses are | |||
219 | /// stored, sorted by increasing beginning offset and with unsplittable slices | |||
220 | /// starting at a particular offset before splittable slices. | |||
221 | class llvm::sroa::AllocaSlices { | |||
222 | public: | |||
223 | /// Construct the slices of a particular alloca. | |||
224 | AllocaSlices(const DataLayout &DL, AllocaInst &AI); | |||
225 | ||||
226 | /// Test whether a pointer to the allocation escapes our analysis. | |||
227 | /// | |||
228 | /// If this is true, the slices are never fully built and should be | |||
229 | /// ignored. | |||
230 | bool isEscaped() const { return PointerEscapingInstr; } | |||
231 | ||||
232 | /// Support for iterating over the slices. | |||
233 | /// @{ | |||
234 | using iterator = SmallVectorImpl<Slice>::iterator; | |||
235 | using range = iterator_range<iterator>; | |||
236 | ||||
237 | iterator begin() { return Slices.begin(); } | |||
238 | iterator end() { return Slices.end(); } | |||
239 | ||||
240 | using const_iterator = SmallVectorImpl<Slice>::const_iterator; | |||
241 | using const_range = iterator_range<const_iterator>; | |||
242 | ||||
243 | const_iterator begin() const { return Slices.begin(); } | |||
244 | const_iterator end() const { return Slices.end(); } | |||
245 | /// @} | |||
246 | ||||
247 | /// Erase a range of slices. | |||
248 | void erase(iterator Start, iterator Stop) { Slices.erase(Start, Stop); } | |||
249 | ||||
250 | /// Insert new slices for this alloca. | |||
251 | /// | |||
252 | /// This moves the slices into the alloca's slices collection, and re-sorts | |||
253 | /// everything so that the usual ordering properties of the alloca's slices | |||
254 | /// hold. | |||
255 | void insert(ArrayRef<Slice> NewSlices) { | |||
256 | int OldSize = Slices.size(); | |||
257 | Slices.append(NewSlices.begin(), NewSlices.end()); | |||
258 | auto SliceI = Slices.begin() + OldSize; | |||
259 | llvm::sort(SliceI, Slices.end()); | |||
260 | std::inplace_merge(Slices.begin(), SliceI, Slices.end()); | |||
261 | } | |||
262 | ||||
263 | // Forward declare the iterator and range accessor for walking the | |||
264 | // partitions. | |||
265 | class partition_iterator; | |||
266 | iterator_range<partition_iterator> partitions(); | |||
267 | ||||
268 | /// Access the dead users for this alloca. | |||
269 | ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; } | |||
270 | ||||
271 | /// Access Uses that should be dropped if the alloca is promotable. | |||
272 | ArrayRef<Use *> getDeadUsesIfPromotable() const { | |||
273 | return DeadUseIfPromotable; | |||
274 | } | |||
275 | ||||
276 | /// Access the dead operands referring to this alloca. | |||
277 | /// | |||
278 | /// These are operands which have cannot actually be used to refer to the | |||
279 | /// alloca as they are outside its range and the user doesn't correct for | |||
280 | /// that. These mostly consist of PHI node inputs and the like which we just | |||
281 | /// need to replace with undef. | |||
282 | ArrayRef<Use *> getDeadOperands() const { return DeadOperands; } | |||
283 | ||||
284 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
285 | void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const; | |||
286 | void printSlice(raw_ostream &OS, const_iterator I, | |||
287 | StringRef Indent = " ") const; | |||
288 | void printUse(raw_ostream &OS, const_iterator I, | |||
289 | StringRef Indent = " ") const; | |||
290 | void print(raw_ostream &OS) const; | |||
291 | void dump(const_iterator I) const; | |||
292 | void dump() const; | |||
293 | #endif | |||
294 | ||||
295 | private: | |||
296 | template <typename DerivedT, typename RetT = void> class BuilderBase; | |||
297 | class SliceBuilder; | |||
298 | ||||
299 | friend class AllocaSlices::SliceBuilder; | |||
300 | ||||
301 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
302 | /// Handle to alloca instruction to simplify method interfaces. | |||
303 | AllocaInst &AI; | |||
304 | #endif | |||
305 | ||||
306 | /// The instruction responsible for this alloca not having a known set | |||
307 | /// of slices. | |||
308 | /// | |||
309 | /// When an instruction (potentially) escapes the pointer to the alloca, we | |||
310 | /// store a pointer to that here and abort trying to form slices of the | |||
311 | /// alloca. This will be null if the alloca slices are analyzed successfully. | |||
312 | Instruction *PointerEscapingInstr; | |||
313 | ||||
314 | /// The slices of the alloca. | |||
315 | /// | |||
316 | /// We store a vector of the slices formed by uses of the alloca here. This | |||
317 | /// vector is sorted by increasing begin offset, and then the unsplittable | |||
318 | /// slices before the splittable ones. See the Slice inner class for more | |||
319 | /// details. | |||
320 | SmallVector<Slice, 8> Slices; | |||
321 | ||||
322 | /// Instructions which will become dead if we rewrite the alloca. | |||
323 | /// | |||
324 | /// Note that these are not separated by slice. This is because we expect an | |||
325 | /// alloca to be completely rewritten or not rewritten at all. If rewritten, | |||
326 | /// all these instructions can simply be removed and replaced with undef as | |||
327 | /// they come from outside of the allocated space. | |||
328 | SmallVector<Instruction *, 8> DeadUsers; | |||
329 | ||||
330 | /// Uses which will become dead if can promote the alloca. | |||
331 | SmallVector<Use *, 8> DeadUseIfPromotable; | |||
332 | ||||
333 | /// Operands which will become dead if we rewrite the alloca. | |||
334 | /// | |||
335 | /// These are operands that in their particular use can be replaced with | |||
336 | /// undef when we rewrite the alloca. These show up in out-of-bounds inputs | |||
337 | /// to PHI nodes and the like. They aren't entirely dead (there might be | |||
338 | /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we | |||
339 | /// want to swap this particular input for undef to simplify the use lists of | |||
340 | /// the alloca. | |||
341 | SmallVector<Use *, 8> DeadOperands; | |||
342 | }; | |||
343 | ||||
344 | /// A partition of the slices. | |||
345 | /// | |||
346 | /// An ephemeral representation for a range of slices which can be viewed as | |||
347 | /// a partition of the alloca. This range represents a span of the alloca's | |||
348 | /// memory which cannot be split, and provides access to all of the slices | |||
349 | /// overlapping some part of the partition. | |||
350 | /// | |||
351 | /// Objects of this type are produced by traversing the alloca's slices, but | |||
352 | /// are only ephemeral and not persistent. | |||
353 | class llvm::sroa::Partition { | |||
354 | private: | |||
355 | friend class AllocaSlices; | |||
356 | friend class AllocaSlices::partition_iterator; | |||
357 | ||||
358 | using iterator = AllocaSlices::iterator; | |||
359 | ||||
360 | /// The beginning and ending offsets of the alloca for this | |||
361 | /// partition. | |||
362 | uint64_t BeginOffset = 0, EndOffset = 0; | |||
363 | ||||
364 | /// The start and end iterators of this partition. | |||
365 | iterator SI, SJ; | |||
366 | ||||
367 | /// A collection of split slice tails overlapping the partition. | |||
368 | SmallVector<Slice *, 4> SplitTails; | |||
369 | ||||
370 | /// Raw constructor builds an empty partition starting and ending at | |||
371 | /// the given iterator. | |||
372 | Partition(iterator SI) : SI(SI), SJ(SI) {} | |||
373 | ||||
374 | public: | |||
375 | /// The start offset of this partition. | |||
376 | /// | |||
377 | /// All of the contained slices start at or after this offset. | |||
378 | uint64_t beginOffset() const { return BeginOffset; } | |||
379 | ||||
380 | /// The end offset of this partition. | |||
381 | /// | |||
382 | /// All of the contained slices end at or before this offset. | |||
383 | uint64_t endOffset() const { return EndOffset; } | |||
384 | ||||
385 | /// The size of the partition. | |||
386 | /// | |||
387 | /// Note that this can never be zero. | |||
388 | uint64_t size() const { | |||
389 | assert(BeginOffset < EndOffset && "Partitions must span some bytes!")(static_cast <bool> (BeginOffset < EndOffset && "Partitions must span some bytes!") ? void (0) : __assert_fail ("BeginOffset < EndOffset && \"Partitions must span some bytes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 389, __extension__ __PRETTY_FUNCTION__)); | |||
390 | return EndOffset - BeginOffset; | |||
391 | } | |||
392 | ||||
393 | /// Test whether this partition contains no slices, and merely spans | |||
394 | /// a region occupied by split slices. | |||
395 | bool empty() const { return SI == SJ; } | |||
396 | ||||
397 | /// \name Iterate slices that start within the partition. | |||
398 | /// These may be splittable or unsplittable. They have a begin offset >= the | |||
399 | /// partition begin offset. | |||
400 | /// @{ | |||
401 | // FIXME: We should probably define a "concat_iterator" helper and use that | |||
402 | // to stitch together pointee_iterators over the split tails and the | |||
403 | // contiguous iterators of the partition. That would give a much nicer | |||
404 | // interface here. We could then additionally expose filtered iterators for | |||
405 | // split, unsplit, and unsplittable splices based on the usage patterns. | |||
406 | iterator begin() const { return SI; } | |||
407 | iterator end() const { return SJ; } | |||
408 | /// @} | |||
409 | ||||
410 | /// Get the sequence of split slice tails. | |||
411 | /// | |||
412 | /// These tails are of slices which start before this partition but are | |||
413 | /// split and overlap into the partition. We accumulate these while forming | |||
414 | /// partitions. | |||
415 | ArrayRef<Slice *> splitSliceTails() const { return SplitTails; } | |||
416 | }; | |||
417 | ||||
418 | /// An iterator over partitions of the alloca's slices. | |||
419 | /// | |||
420 | /// This iterator implements the core algorithm for partitioning the alloca's | |||
421 | /// slices. It is a forward iterator as we don't support backtracking for | |||
422 | /// efficiency reasons, and re-use a single storage area to maintain the | |||
423 | /// current set of split slices. | |||
424 | /// | |||
425 | /// It is templated on the slice iterator type to use so that it can operate | |||
426 | /// with either const or non-const slice iterators. | |||
427 | class AllocaSlices::partition_iterator | |||
428 | : public iterator_facade_base<partition_iterator, std::forward_iterator_tag, | |||
429 | Partition> { | |||
430 | friend class AllocaSlices; | |||
431 | ||||
432 | /// Most of the state for walking the partitions is held in a class | |||
433 | /// with a nice interface for examining them. | |||
434 | Partition P; | |||
435 | ||||
436 | /// We need to keep the end of the slices to know when to stop. | |||
437 | AllocaSlices::iterator SE; | |||
438 | ||||
439 | /// We also need to keep track of the maximum split end offset seen. | |||
440 | /// FIXME: Do we really? | |||
441 | uint64_t MaxSplitSliceEndOffset = 0; | |||
442 | ||||
443 | /// Sets the partition to be empty at given iterator, and sets the | |||
444 | /// end iterator. | |||
445 | partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE) | |||
446 | : P(SI), SE(SE) { | |||
447 | // If not already at the end, advance our state to form the initial | |||
448 | // partition. | |||
449 | if (SI != SE) | |||
450 | advance(); | |||
451 | } | |||
452 | ||||
453 | /// Advance the iterator to the next partition. | |||
454 | /// | |||
455 | /// Requires that the iterator not be at the end of the slices. | |||
456 | void advance() { | |||
457 | assert((P.SI != SE || !P.SplitTails.empty()) &&(static_cast <bool> ((P.SI != SE || !P.SplitTails.empty ()) && "Cannot advance past the end of the slices!") ? void (0) : __assert_fail ("(P.SI != SE || !P.SplitTails.empty()) && \"Cannot advance past the end of the slices!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 458, __extension__ __PRETTY_FUNCTION__)) | |||
458 | "Cannot advance past the end of the slices!")(static_cast <bool> ((P.SI != SE || !P.SplitTails.empty ()) && "Cannot advance past the end of the slices!") ? void (0) : __assert_fail ("(P.SI != SE || !P.SplitTails.empty()) && \"Cannot advance past the end of the slices!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 458, __extension__ __PRETTY_FUNCTION__)); | |||
459 | ||||
460 | // Clear out any split uses which have ended. | |||
461 | if (!P.SplitTails.empty()) { | |||
462 | if (P.EndOffset >= MaxSplitSliceEndOffset) { | |||
463 | // If we've finished all splits, this is easy. | |||
464 | P.SplitTails.clear(); | |||
465 | MaxSplitSliceEndOffset = 0; | |||
466 | } else { | |||
467 | // Remove the uses which have ended in the prior partition. This | |||
468 | // cannot change the max split slice end because we just checked that | |||
469 | // the prior partition ended prior to that max. | |||
470 | llvm::erase_if(P.SplitTails, | |||
471 | [&](Slice *S) { return S->endOffset() <= P.EndOffset; }); | |||
472 | assert(llvm::any_of(P.SplitTails,(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 476, __extension__ __PRETTY_FUNCTION__)) | |||
473 | [&](Slice *S) {(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 476, __extension__ __PRETTY_FUNCTION__)) | |||
474 | return S->endOffset() == MaxSplitSliceEndOffset;(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 476, __extension__ __PRETTY_FUNCTION__)) | |||
475 | }) &&(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 476, __extension__ __PRETTY_FUNCTION__)) | |||
476 | "Could not find the current max split slice offset!")(static_cast <bool> (llvm::any_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset ; }) && "Could not find the current max split slice offset!" ) ? void (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 476, __extension__ __PRETTY_FUNCTION__)); | |||
477 | assert(llvm::all_of(P.SplitTails,(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 481, __extension__ __PRETTY_FUNCTION__)) | |||
478 | [&](Slice *S) {(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 481, __extension__ __PRETTY_FUNCTION__)) | |||
479 | return S->endOffset() <= MaxSplitSliceEndOffset;(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 481, __extension__ __PRETTY_FUNCTION__)) | |||
480 | }) &&(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 481, __extension__ __PRETTY_FUNCTION__)) | |||
481 | "Max split slice end offset is not actually the max!")(static_cast <bool> (llvm::all_of(P.SplitTails, [&] (Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset ; }) && "Max split slice end offset is not actually the max!" ) ? void (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 481, __extension__ __PRETTY_FUNCTION__)); | |||
482 | } | |||
483 | } | |||
484 | ||||
485 | // If P.SI is already at the end, then we've cleared the split tail and | |||
486 | // now have an end iterator. | |||
487 | if (P.SI == SE) { | |||
488 | assert(P.SplitTails.empty() && "Failed to clear the split slices!")(static_cast <bool> (P.SplitTails.empty() && "Failed to clear the split slices!" ) ? void (0) : __assert_fail ("P.SplitTails.empty() && \"Failed to clear the split slices!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 488, __extension__ __PRETTY_FUNCTION__)); | |||
489 | return; | |||
490 | } | |||
491 | ||||
492 | // If we had a non-empty partition previously, set up the state for | |||
493 | // subsequent partitions. | |||
494 | if (P.SI != P.SJ) { | |||
495 | // Accumulate all the splittable slices which started in the old | |||
496 | // partition into the split list. | |||
497 | for (Slice &S : P) | |||
498 | if (S.isSplittable() && S.endOffset() > P.EndOffset) { | |||
499 | P.SplitTails.push_back(&S); | |||
500 | MaxSplitSliceEndOffset = | |||
501 | std::max(S.endOffset(), MaxSplitSliceEndOffset); | |||
502 | } | |||
503 | ||||
504 | // Start from the end of the previous partition. | |||
505 | P.SI = P.SJ; | |||
506 | ||||
507 | // If P.SI is now at the end, we at most have a tail of split slices. | |||
508 | if (P.SI == SE) { | |||
509 | P.BeginOffset = P.EndOffset; | |||
510 | P.EndOffset = MaxSplitSliceEndOffset; | |||
511 | return; | |||
512 | } | |||
513 | ||||
514 | // If the we have split slices and the next slice is after a gap and is | |||
515 | // not splittable immediately form an empty partition for the split | |||
516 | // slices up until the next slice begins. | |||
517 | if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset && | |||
518 | !P.SI->isSplittable()) { | |||
519 | P.BeginOffset = P.EndOffset; | |||
520 | P.EndOffset = P.SI->beginOffset(); | |||
521 | return; | |||
522 | } | |||
523 | } | |||
524 | ||||
525 | // OK, we need to consume new slices. Set the end offset based on the | |||
526 | // current slice, and step SJ past it. The beginning offset of the | |||
527 | // partition is the beginning offset of the next slice unless we have | |||
528 | // pre-existing split slices that are continuing, in which case we begin | |||
529 | // at the prior end offset. | |||
530 | P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset; | |||
531 | P.EndOffset = P.SI->endOffset(); | |||
532 | ++P.SJ; | |||
533 | ||||
534 | // There are two strategies to form a partition based on whether the | |||
535 | // partition starts with an unsplittable slice or a splittable slice. | |||
536 | if (!P.SI->isSplittable()) { | |||
537 | // When we're forming an unsplittable region, it must always start at | |||
538 | // the first slice and will extend through its end. | |||
539 | assert(P.BeginOffset == P.SI->beginOffset())(static_cast <bool> (P.BeginOffset == P.SI->beginOffset ()) ? void (0) : __assert_fail ("P.BeginOffset == P.SI->beginOffset()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 539, __extension__ __PRETTY_FUNCTION__)); | |||
540 | ||||
541 | // Form a partition including all of the overlapping slices with this | |||
542 | // unsplittable slice. | |||
543 | while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) { | |||
544 | if (!P.SJ->isSplittable()) | |||
545 | P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset()); | |||
546 | ++P.SJ; | |||
547 | } | |||
548 | ||||
549 | // We have a partition across a set of overlapping unsplittable | |||
550 | // partitions. | |||
551 | return; | |||
552 | } | |||
553 | ||||
554 | // If we're starting with a splittable slice, then we need to form | |||
555 | // a synthetic partition spanning it and any other overlapping splittable | |||
556 | // splices. | |||
557 | assert(P.SI->isSplittable() && "Forming a splittable partition!")(static_cast <bool> (P.SI->isSplittable() && "Forming a splittable partition!") ? void (0) : __assert_fail ("P.SI->isSplittable() && \"Forming a splittable partition!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 557, __extension__ __PRETTY_FUNCTION__)); | |||
558 | ||||
559 | // Collect all of the overlapping splittable slices. | |||
560 | while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset && | |||
561 | P.SJ->isSplittable()) { | |||
562 | P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset()); | |||
563 | ++P.SJ; | |||
564 | } | |||
565 | ||||
566 | // Back upiP.EndOffset if we ended the span early when encountering an | |||
567 | // unsplittable slice. This synthesizes the early end offset of | |||
568 | // a partition spanning only splittable slices. | |||
569 | if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) { | |||
570 | assert(!P.SJ->isSplittable())(static_cast <bool> (!P.SJ->isSplittable()) ? void ( 0) : __assert_fail ("!P.SJ->isSplittable()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 570, __extension__ __PRETTY_FUNCTION__)); | |||
571 | P.EndOffset = P.SJ->beginOffset(); | |||
572 | } | |||
573 | } | |||
574 | ||||
575 | public: | |||
576 | bool operator==(const partition_iterator &RHS) const { | |||
577 | assert(SE == RHS.SE &&(static_cast <bool> (SE == RHS.SE && "End iterators don't match between compared partition iterators!" ) ? void (0) : __assert_fail ("SE == RHS.SE && \"End iterators don't match between compared partition iterators!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 578, __extension__ __PRETTY_FUNCTION__)) | |||
578 | "End iterators don't match between compared partition iterators!")(static_cast <bool> (SE == RHS.SE && "End iterators don't match between compared partition iterators!" ) ? void (0) : __assert_fail ("SE == RHS.SE && \"End iterators don't match between compared partition iterators!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 578, __extension__ __PRETTY_FUNCTION__)); | |||
579 | ||||
580 | // The observed positions of partitions is marked by the P.SI iterator and | |||
581 | // the emptiness of the split slices. The latter is only relevant when | |||
582 | // P.SI == SE, as the end iterator will additionally have an empty split | |||
583 | // slices list, but the prior may have the same P.SI and a tail of split | |||
584 | // slices. | |||
585 | if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) { | |||
586 | assert(P.SJ == RHS.P.SJ &&(static_cast <bool> (P.SJ == RHS.P.SJ && "Same set of slices formed two different sized partitions!" ) ? void (0) : __assert_fail ("P.SJ == RHS.P.SJ && \"Same set of slices formed two different sized partitions!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 587, __extension__ __PRETTY_FUNCTION__)) | |||
587 | "Same set of slices formed two different sized partitions!")(static_cast <bool> (P.SJ == RHS.P.SJ && "Same set of slices formed two different sized partitions!" ) ? void (0) : __assert_fail ("P.SJ == RHS.P.SJ && \"Same set of slices formed two different sized partitions!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 587, __extension__ __PRETTY_FUNCTION__)); | |||
588 | assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&(static_cast <bool> (P.SplitTails.size() == RHS.P.SplitTails .size() && "Same slice position with differently sized non-empty split " "slice tails!") ? void (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 590, __extension__ __PRETTY_FUNCTION__)) | |||
589 | "Same slice position with differently sized non-empty split "(static_cast <bool> (P.SplitTails.size() == RHS.P.SplitTails .size() && "Same slice position with differently sized non-empty split " "slice tails!") ? void (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 590, __extension__ __PRETTY_FUNCTION__)) | |||
590 | "slice tails!")(static_cast <bool> (P.SplitTails.size() == RHS.P.SplitTails .size() && "Same slice position with differently sized non-empty split " "slice tails!") ? void (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 590, __extension__ __PRETTY_FUNCTION__)); | |||
591 | return true; | |||
592 | } | |||
593 | return false; | |||
594 | } | |||
595 | ||||
596 | partition_iterator &operator++() { | |||
597 | advance(); | |||
598 | return *this; | |||
599 | } | |||
600 | ||||
601 | Partition &operator*() { return P; } | |||
602 | }; | |||
603 | ||||
604 | /// A forward range over the partitions of the alloca's slices. | |||
605 | /// | |||
606 | /// This accesses an iterator range over the partitions of the alloca's | |||
607 | /// slices. It computes these partitions on the fly based on the overlapping | |||
608 | /// offsets of the slices and the ability to split them. It will visit "empty" | |||
609 | /// partitions to cover regions of the alloca only accessed via split | |||
610 | /// slices. | |||
611 | iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() { | |||
612 | return make_range(partition_iterator(begin(), end()), | |||
613 | partition_iterator(end(), end())); | |||
614 | } | |||
615 | ||||
616 | static Value *foldSelectInst(SelectInst &SI) { | |||
617 | // If the condition being selected on is a constant or the same value is | |||
618 | // being selected between, fold the select. Yes this does (rarely) happen | |||
619 | // early on. | |||
620 | if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition())) | |||
621 | return SI.getOperand(1 + CI->isZero()); | |||
622 | if (SI.getOperand(1) == SI.getOperand(2)) | |||
623 | return SI.getOperand(1); | |||
624 | ||||
625 | return nullptr; | |||
626 | } | |||
627 | ||||
628 | /// A helper that folds a PHI node or a select. | |||
629 | static Value *foldPHINodeOrSelectInst(Instruction &I) { | |||
630 | if (PHINode *PN = dyn_cast<PHINode>(&I)) { | |||
631 | // If PN merges together the same value, return that value. | |||
632 | return PN->hasConstantValue(); | |||
633 | } | |||
634 | return foldSelectInst(cast<SelectInst>(I)); | |||
635 | } | |||
636 | ||||
637 | /// Builder for the alloca slices. | |||
638 | /// | |||
639 | /// This class builds a set of alloca slices by recursively visiting the uses | |||
640 | /// of an alloca and making a slice for each load and store at each offset. | |||
641 | class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> { | |||
642 | friend class PtrUseVisitor<SliceBuilder>; | |||
643 | friend class InstVisitor<SliceBuilder>; | |||
644 | ||||
645 | using Base = PtrUseVisitor<SliceBuilder>; | |||
646 | ||||
647 | const uint64_t AllocSize; | |||
648 | AllocaSlices &AS; | |||
649 | ||||
650 | SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap; | |||
651 | SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes; | |||
652 | ||||
653 | /// Set to de-duplicate dead instructions found in the use walk. | |||
654 | SmallPtrSet<Instruction *, 4> VisitedDeadInsts; | |||
655 | ||||
656 | public: | |||
657 | SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS) | |||
658 | : PtrUseVisitor<SliceBuilder>(DL), | |||
659 | AllocSize(DL.getTypeAllocSize(AI.getAllocatedType()).getFixedSize()), | |||
660 | AS(AS) {} | |||
661 | ||||
662 | private: | |||
663 | void markAsDead(Instruction &I) { | |||
664 | if (VisitedDeadInsts.insert(&I).second) | |||
665 | AS.DeadUsers.push_back(&I); | |||
666 | } | |||
667 | ||||
668 | void insertUse(Instruction &I, const APInt &Offset, uint64_t Size, | |||
669 | bool IsSplittable = false) { | |||
670 | // Completely skip uses which have a zero size or start either before or | |||
671 | // past the end of the allocation. | |||
672 | if (Size == 0 || Offset.uge(AllocSize)) { | |||
673 | LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
674 | << Offsetdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
675 | << " which has zero size or starts outside of the "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
676 | << AllocSize << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
677 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
678 | << " use: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false); | |||
679 | return markAsDead(I); | |||
680 | } | |||
681 | ||||
682 | uint64_t BeginOffset = Offset.getZExtValue(); | |||
683 | uint64_t EndOffset = BeginOffset + Size; | |||
684 | ||||
685 | // Clamp the end offset to the end of the allocation. Note that this is | |||
686 | // formulated to handle even the case where "BeginOffset + Size" overflows. | |||
687 | // This may appear superficially to be something we could ignore entirely, | |||
688 | // but that is not so! There may be widened loads or PHI-node uses where | |||
689 | // some instructions are dead but not others. We can't completely ignore | |||
690 | // them, and so have to record at least the information here. | |||
691 | assert(AllocSize >= BeginOffset)(static_cast <bool> (AllocSize >= BeginOffset) ? void (0) : __assert_fail ("AllocSize >= BeginOffset", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 691, __extension__ __PRETTY_FUNCTION__)); // Established above. | |||
692 | if (Size > AllocSize - BeginOffset) { | |||
693 | LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
694 | << Offset << " to remain within the " << AllocSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
695 | << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
696 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||
697 | << " use: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false); | |||
698 | EndOffset = AllocSize; | |||
699 | } | |||
700 | ||||
701 | AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable)); | |||
702 | } | |||
703 | ||||
704 | void visitBitCastInst(BitCastInst &BC) { | |||
705 | if (BC.use_empty()) | |||
706 | return markAsDead(BC); | |||
707 | ||||
708 | return Base::visitBitCastInst(BC); | |||
709 | } | |||
710 | ||||
711 | void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) { | |||
712 | if (ASC.use_empty()) | |||
713 | return markAsDead(ASC); | |||
714 | ||||
715 | return Base::visitAddrSpaceCastInst(ASC); | |||
716 | } | |||
717 | ||||
718 | void visitGetElementPtrInst(GetElementPtrInst &GEPI) { | |||
719 | if (GEPI.use_empty()) | |||
720 | return markAsDead(GEPI); | |||
721 | ||||
722 | if (SROAStrictInbounds && GEPI.isInBounds()) { | |||
723 | // FIXME: This is a manually un-factored variant of the basic code inside | |||
724 | // of GEPs with checking of the inbounds invariant specified in the | |||
725 | // langref in a very strict sense. If we ever want to enable | |||
726 | // SROAStrictInbounds, this code should be factored cleanly into | |||
727 | // PtrUseVisitor, but it is easier to experiment with SROAStrictInbounds | |||
728 | // by writing out the code here where we have the underlying allocation | |||
729 | // size readily available. | |||
730 | APInt GEPOffset = Offset; | |||
731 | const DataLayout &DL = GEPI.getModule()->getDataLayout(); | |||
732 | for (gep_type_iterator GTI = gep_type_begin(GEPI), | |||
733 | GTE = gep_type_end(GEPI); | |||
734 | GTI != GTE; ++GTI) { | |||
735 | ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand()); | |||
736 | if (!OpC) | |||
737 | break; | |||
738 | ||||
739 | // Handle a struct index, which adds its field offset to the pointer. | |||
740 | if (StructType *STy = GTI.getStructTypeOrNull()) { | |||
741 | unsigned ElementIdx = OpC->getZExtValue(); | |||
742 | const StructLayout *SL = DL.getStructLayout(STy); | |||
743 | GEPOffset += | |||
744 | APInt(Offset.getBitWidth(), SL->getElementOffset(ElementIdx)); | |||
745 | } else { | |||
746 | // For array or vector indices, scale the index by the size of the | |||
747 | // type. | |||
748 | APInt Index = OpC->getValue().sextOrTrunc(Offset.getBitWidth()); | |||
749 | GEPOffset += | |||
750 | Index * | |||
751 | APInt(Offset.getBitWidth(), | |||
752 | DL.getTypeAllocSize(GTI.getIndexedType()).getFixedSize()); | |||
753 | } | |||
754 | ||||
755 | // If this index has computed an intermediate pointer which is not | |||
756 | // inbounds, then the result of the GEP is a poison value and we can | |||
757 | // delete it and all uses. | |||
758 | if (GEPOffset.ugt(AllocSize)) | |||
759 | return markAsDead(GEPI); | |||
760 | } | |||
761 | } | |||
762 | ||||
763 | return Base::visitGetElementPtrInst(GEPI); | |||
764 | } | |||
765 | ||||
766 | void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset, | |||
767 | uint64_t Size, bool IsVolatile) { | |||
768 | // We allow splitting of non-volatile loads and stores where the type is an | |||
769 | // integer type. These may be used to implement 'memcpy' or other "transfer | |||
770 | // of bits" patterns. | |||
771 | bool IsSplittable = | |||
772 | Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty); | |||
773 | ||||
774 | insertUse(I, Offset, Size, IsSplittable); | |||
775 | } | |||
776 | ||||
777 | void visitLoadInst(LoadInst &LI) { | |||
778 | assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&(static_cast <bool> ((!LI.isSimple() || LI.getType()-> isSingleValueType()) && "All simple FCA loads should have been pre-split" ) ? void (0) : __assert_fail ("(!LI.isSimple() || LI.getType()->isSingleValueType()) && \"All simple FCA loads should have been pre-split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 779, __extension__ __PRETTY_FUNCTION__)) | |||
779 | "All simple FCA loads should have been pre-split")(static_cast <bool> ((!LI.isSimple() || LI.getType()-> isSingleValueType()) && "All simple FCA loads should have been pre-split" ) ? void (0) : __assert_fail ("(!LI.isSimple() || LI.getType()->isSingleValueType()) && \"All simple FCA loads should have been pre-split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 779, __extension__ __PRETTY_FUNCTION__)); | |||
780 | ||||
781 | if (!IsOffsetKnown) | |||
782 | return PI.setAborted(&LI); | |||
783 | ||||
784 | if (LI.isVolatile() && | |||
785 | LI.getPointerAddressSpace() != DL.getAllocaAddrSpace()) | |||
786 | return PI.setAborted(&LI); | |||
787 | ||||
788 | if (isa<ScalableVectorType>(LI.getType())) | |||
789 | return PI.setAborted(&LI); | |||
790 | ||||
791 | uint64_t Size = DL.getTypeStoreSize(LI.getType()).getFixedSize(); | |||
792 | return handleLoadOrStore(LI.getType(), LI, Offset, Size, LI.isVolatile()); | |||
793 | } | |||
794 | ||||
795 | void visitStoreInst(StoreInst &SI) { | |||
796 | Value *ValOp = SI.getValueOperand(); | |||
797 | if (ValOp == *U) | |||
798 | return PI.setEscapedAndAborted(&SI); | |||
799 | if (!IsOffsetKnown) | |||
800 | return PI.setAborted(&SI); | |||
801 | ||||
802 | if (SI.isVolatile() && | |||
803 | SI.getPointerAddressSpace() != DL.getAllocaAddrSpace()) | |||
804 | return PI.setAborted(&SI); | |||
805 | ||||
806 | if (isa<ScalableVectorType>(ValOp->getType())) | |||
807 | return PI.setAborted(&SI); | |||
808 | ||||
809 | uint64_t Size = DL.getTypeStoreSize(ValOp->getType()).getFixedSize(); | |||
810 | ||||
811 | // If this memory access can be shown to *statically* extend outside the | |||
812 | // bounds of the allocation, it's behavior is undefined, so simply | |||
813 | // ignore it. Note that this is more strict than the generic clamping | |||
814 | // behavior of insertUse. We also try to handle cases which might run the | |||
815 | // risk of overflow. | |||
816 | // FIXME: We should instead consider the pointer to have escaped if this | |||
817 | // function is being instrumented for addressing bugs or race conditions. | |||
818 | if (Size > AllocSize || Offset.ugt(AllocSize - Size)) { | |||
819 | LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||
820 | << Offset << " which extends past the end of the "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||
821 | << AllocSize << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||
822 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||
823 | << " use: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false); | |||
824 | return markAsDead(SI); | |||
825 | } | |||
826 | ||||
827 | assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&(static_cast <bool> ((!SI.isSimple() || ValOp->getType ()->isSingleValueType()) && "All simple FCA stores should have been pre-split" ) ? void (0) : __assert_fail ("(!SI.isSimple() || ValOp->getType()->isSingleValueType()) && \"All simple FCA stores should have been pre-split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 828, __extension__ __PRETTY_FUNCTION__)) | |||
828 | "All simple FCA stores should have been pre-split")(static_cast <bool> ((!SI.isSimple() || ValOp->getType ()->isSingleValueType()) && "All simple FCA stores should have been pre-split" ) ? void (0) : __assert_fail ("(!SI.isSimple() || ValOp->getType()->isSingleValueType()) && \"All simple FCA stores should have been pre-split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 828, __extension__ __PRETTY_FUNCTION__)); | |||
829 | handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile()); | |||
830 | } | |||
831 | ||||
832 | void visitMemSetInst(MemSetInst &II) { | |||
833 | assert(II.getRawDest() == *U && "Pointer use is not the destination?")(static_cast <bool> (II.getRawDest() == *U && "Pointer use is not the destination?" ) ? void (0) : __assert_fail ("II.getRawDest() == *U && \"Pointer use is not the destination?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 833, __extension__ __PRETTY_FUNCTION__)); | |||
834 | ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); | |||
835 | if ((Length && Length->getValue() == 0) || | |||
836 | (IsOffsetKnown && Offset.uge(AllocSize))) | |||
837 | // Zero-length mem transfer intrinsics can be ignored entirely. | |||
838 | return markAsDead(II); | |||
839 | ||||
840 | if (!IsOffsetKnown) | |||
841 | return PI.setAborted(&II); | |||
842 | ||||
843 | // Don't replace this with a store with a different address space. TODO: | |||
844 | // Use a store with the casted new alloca? | |||
845 | if (II.isVolatile() && II.getDestAddressSpace() != DL.getAllocaAddrSpace()) | |||
846 | return PI.setAborted(&II); | |||
847 | ||||
848 | insertUse(II, Offset, Length ? Length->getLimitedValue() | |||
849 | : AllocSize - Offset.getLimitedValue(), | |||
850 | (bool)Length); | |||
851 | } | |||
852 | ||||
853 | void visitMemTransferInst(MemTransferInst &II) { | |||
854 | ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); | |||
855 | if (Length && Length->getValue() == 0) | |||
856 | // Zero-length mem transfer intrinsics can be ignored entirely. | |||
857 | return markAsDead(II); | |||
858 | ||||
859 | // Because we can visit these intrinsics twice, also check to see if the | |||
860 | // first time marked this instruction as dead. If so, skip it. | |||
861 | if (VisitedDeadInsts.count(&II)) | |||
862 | return; | |||
863 | ||||
864 | if (!IsOffsetKnown) | |||
865 | return PI.setAborted(&II); | |||
866 | ||||
867 | // Don't replace this with a load/store with a different address space. | |||
868 | // TODO: Use a store with the casted new alloca? | |||
869 | if (II.isVolatile() && | |||
870 | (II.getDestAddressSpace() != DL.getAllocaAddrSpace() || | |||
871 | II.getSourceAddressSpace() != DL.getAllocaAddrSpace())) | |||
872 | return PI.setAborted(&II); | |||
873 | ||||
874 | // This side of the transfer is completely out-of-bounds, and so we can | |||
875 | // nuke the entire transfer. However, we also need to nuke the other side | |||
876 | // if already added to our partitions. | |||
877 | // FIXME: Yet another place we really should bypass this when | |||
878 | // instrumenting for ASan. | |||
879 | if (Offset.uge(AllocSize)) { | |||
880 | SmallDenseMap<Instruction *, unsigned>::iterator MTPI = | |||
881 | MemTransferSliceMap.find(&II); | |||
882 | if (MTPI != MemTransferSliceMap.end()) | |||
883 | AS.Slices[MTPI->second].kill(); | |||
884 | return markAsDead(II); | |||
885 | } | |||
886 | ||||
887 | uint64_t RawOffset = Offset.getLimitedValue(); | |||
888 | uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset; | |||
889 | ||||
890 | // Check for the special case where the same exact value is used for both | |||
891 | // source and dest. | |||
892 | if (*U == II.getRawDest() && *U == II.getRawSource()) { | |||
893 | // For non-volatile transfers this is a no-op. | |||
894 | if (!II.isVolatile()) | |||
895 | return markAsDead(II); | |||
896 | ||||
897 | return insertUse(II, Offset, Size, /*IsSplittable=*/false); | |||
898 | } | |||
899 | ||||
900 | // If we have seen both source and destination for a mem transfer, then | |||
901 | // they both point to the same alloca. | |||
902 | bool Inserted; | |||
903 | SmallDenseMap<Instruction *, unsigned>::iterator MTPI; | |||
904 | std::tie(MTPI, Inserted) = | |||
905 | MemTransferSliceMap.insert(std::make_pair(&II, AS.Slices.size())); | |||
906 | unsigned PrevIdx = MTPI->second; | |||
907 | if (!Inserted) { | |||
908 | Slice &PrevP = AS.Slices[PrevIdx]; | |||
909 | ||||
910 | // Check if the begin offsets match and this is a non-volatile transfer. | |||
911 | // In that case, we can completely elide the transfer. | |||
912 | if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) { | |||
913 | PrevP.kill(); | |||
914 | return markAsDead(II); | |||
915 | } | |||
916 | ||||
917 | // Otherwise we have an offset transfer within the same alloca. We can't | |||
918 | // split those. | |||
919 | PrevP.makeUnsplittable(); | |||
920 | } | |||
921 | ||||
922 | // Insert the use now that we've fixed up the splittable nature. | |||
923 | insertUse(II, Offset, Size, /*IsSplittable=*/Inserted && Length); | |||
924 | ||||
925 | // Check that we ended up with a valid index in the map. | |||
926 | assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&(static_cast <bool> (AS.Slices[PrevIdx].getUse()->getUser () == &II && "Map index doesn't point back to a slice with this user." ) ? void (0) : __assert_fail ("AS.Slices[PrevIdx].getUse()->getUser() == &II && \"Map index doesn't point back to a slice with this user.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 927, __extension__ __PRETTY_FUNCTION__)) | |||
927 | "Map index doesn't point back to a slice with this user.")(static_cast <bool> (AS.Slices[PrevIdx].getUse()->getUser () == &II && "Map index doesn't point back to a slice with this user." ) ? void (0) : __assert_fail ("AS.Slices[PrevIdx].getUse()->getUser() == &II && \"Map index doesn't point back to a slice with this user.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 927, __extension__ __PRETTY_FUNCTION__)); | |||
928 | } | |||
929 | ||||
930 | // Disable SRoA for any intrinsics except for lifetime invariants and | |||
931 | // invariant group. | |||
932 | // FIXME: What about debug intrinsics? This matches old behavior, but | |||
933 | // doesn't make sense. | |||
934 | void visitIntrinsicInst(IntrinsicInst &II) { | |||
935 | if (II.isDroppable()) { | |||
936 | AS.DeadUseIfPromotable.push_back(U); | |||
937 | return; | |||
938 | } | |||
939 | ||||
940 | if (!IsOffsetKnown) | |||
941 | return PI.setAborted(&II); | |||
942 | ||||
943 | if (II.isLifetimeStartOrEnd()) { | |||
944 | ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0)); | |||
945 | uint64_t Size = std::min(AllocSize - Offset.getLimitedValue(), | |||
946 | Length->getLimitedValue()); | |||
947 | insertUse(II, Offset, Size, true); | |||
948 | return; | |||
949 | } | |||
950 | ||||
951 | if (II.isLaunderOrStripInvariantGroup()) { | |||
952 | enqueueUsers(II); | |||
953 | return; | |||
954 | } | |||
955 | ||||
956 | Base::visitIntrinsicInst(II); | |||
957 | } | |||
958 | ||||
959 | Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) { | |||
960 | // We consider any PHI or select that results in a direct load or store of | |||
961 | // the same offset to be a viable use for slicing purposes. These uses | |||
962 | // are considered unsplittable and the size is the maximum loaded or stored | |||
963 | // size. | |||
964 | SmallPtrSet<Instruction *, 4> Visited; | |||
965 | SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses; | |||
966 | Visited.insert(Root); | |||
967 | Uses.push_back(std::make_pair(cast<Instruction>(*U), Root)); | |||
968 | const DataLayout &DL = Root->getModule()->getDataLayout(); | |||
969 | // If there are no loads or stores, the access is dead. We mark that as | |||
970 | // a size zero access. | |||
971 | Size = 0; | |||
972 | do { | |||
973 | Instruction *I, *UsedI; | |||
974 | std::tie(UsedI, I) = Uses.pop_back_val(); | |||
975 | ||||
976 | if (LoadInst *LI = dyn_cast<LoadInst>(I)) { | |||
977 | Size = std::max(Size, | |||
978 | DL.getTypeStoreSize(LI->getType()).getFixedSize()); | |||
979 | continue; | |||
980 | } | |||
981 | if (StoreInst *SI = dyn_cast<StoreInst>(I)) { | |||
982 | Value *Op = SI->getOperand(0); | |||
983 | if (Op == UsedI) | |||
984 | return SI; | |||
985 | Size = std::max(Size, | |||
986 | DL.getTypeStoreSize(Op->getType()).getFixedSize()); | |||
987 | continue; | |||
988 | } | |||
989 | ||||
990 | if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) { | |||
991 | if (!GEP->hasAllZeroIndices()) | |||
992 | return GEP; | |||
993 | } else if (!isa<BitCastInst>(I) && !isa<PHINode>(I) && | |||
994 | !isa<SelectInst>(I) && !isa<AddrSpaceCastInst>(I)) { | |||
995 | return I; | |||
996 | } | |||
997 | ||||
998 | for (User *U : I->users()) | |||
999 | if (Visited.insert(cast<Instruction>(U)).second) | |||
1000 | Uses.push_back(std::make_pair(I, cast<Instruction>(U))); | |||
1001 | } while (!Uses.empty()); | |||
1002 | ||||
1003 | return nullptr; | |||
1004 | } | |||
1005 | ||||
1006 | void visitPHINodeOrSelectInst(Instruction &I) { | |||
1007 | assert(isa<PHINode>(I) || isa<SelectInst>(I))(static_cast <bool> (isa<PHINode>(I) || isa<SelectInst >(I)) ? void (0) : __assert_fail ("isa<PHINode>(I) || isa<SelectInst>(I)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1007, __extension__ __PRETTY_FUNCTION__)); | |||
1008 | if (I.use_empty()) | |||
1009 | return markAsDead(I); | |||
1010 | ||||
1011 | // TODO: We could use SimplifyInstruction here to fold PHINodes and | |||
1012 | // SelectInsts. However, doing so requires to change the current | |||
1013 | // dead-operand-tracking mechanism. For instance, suppose neither loading | |||
1014 | // from %U nor %other traps. Then "load (select undef, %U, %other)" does not | |||
1015 | // trap either. However, if we simply replace %U with undef using the | |||
1016 | // current dead-operand-tracking mechanism, "load (select undef, undef, | |||
1017 | // %other)" may trap because the select may return the first operand | |||
1018 | // "undef". | |||
1019 | if (Value *Result = foldPHINodeOrSelectInst(I)) { | |||
1020 | if (Result == *U) | |||
1021 | // If the result of the constant fold will be the pointer, recurse | |||
1022 | // through the PHI/select as if we had RAUW'ed it. | |||
1023 | enqueueUsers(I); | |||
1024 | else | |||
1025 | // Otherwise the operand to the PHI/select is dead, and we can replace | |||
1026 | // it with undef. | |||
1027 | AS.DeadOperands.push_back(U); | |||
1028 | ||||
1029 | return; | |||
1030 | } | |||
1031 | ||||
1032 | if (!IsOffsetKnown) | |||
1033 | return PI.setAborted(&I); | |||
1034 | ||||
1035 | // See if we already have computed info on this node. | |||
1036 | uint64_t &Size = PHIOrSelectSizes[&I]; | |||
1037 | if (!Size) { | |||
1038 | // This is a new PHI/Select, check for an unsafe use of it. | |||
1039 | if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(&I, Size)) | |||
1040 | return PI.setAborted(UnsafeI); | |||
1041 | } | |||
1042 | ||||
1043 | // For PHI and select operands outside the alloca, we can't nuke the entire | |||
1044 | // phi or select -- the other side might still be relevant, so we special | |||
1045 | // case them here and use a separate structure to track the operands | |||
1046 | // themselves which should be replaced with undef. | |||
1047 | // FIXME: This should instead be escaped in the event we're instrumenting | |||
1048 | // for address sanitization. | |||
1049 | if (Offset.uge(AllocSize)) { | |||
1050 | AS.DeadOperands.push_back(U); | |||
1051 | return; | |||
1052 | } | |||
1053 | ||||
1054 | insertUse(I, Offset, Size); | |||
1055 | } | |||
1056 | ||||
1057 | void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(PN); } | |||
1058 | ||||
1059 | void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); } | |||
1060 | ||||
1061 | /// Disable SROA entirely if there are unhandled users of the alloca. | |||
1062 | void visitInstruction(Instruction &I) { PI.setAborted(&I); } | |||
1063 | }; | |||
1064 | ||||
1065 | AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI) | |||
1066 | : | |||
1067 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1068 | AI(AI), | |||
1069 | #endif | |||
1070 | PointerEscapingInstr(nullptr) { | |||
1071 | SliceBuilder PB(DL, AI, *this); | |||
1072 | SliceBuilder::PtrInfo PtrI = PB.visitPtr(AI); | |||
1073 | if (PtrI.isEscaped() || PtrI.isAborted()) { | |||
1074 | // FIXME: We should sink the escape vs. abort info into the caller nicely, | |||
1075 | // possibly by just storing the PtrInfo in the AllocaSlices. | |||
1076 | PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst() | |||
1077 | : PtrI.getAbortingInst(); | |||
1078 | assert(PointerEscapingInstr && "Did not track a bad instruction")(static_cast <bool> (PointerEscapingInstr && "Did not track a bad instruction" ) ? void (0) : __assert_fail ("PointerEscapingInstr && \"Did not track a bad instruction\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1078, __extension__ __PRETTY_FUNCTION__)); | |||
1079 | return; | |||
1080 | } | |||
1081 | ||||
1082 | llvm::erase_if(Slices, [](const Slice &S) { return S.isDead(); }); | |||
1083 | ||||
1084 | // Sort the uses. This arranges for the offsets to be in ascending order, | |||
1085 | // and the sizes to be in descending order. | |||
1086 | llvm::stable_sort(Slices); | |||
1087 | } | |||
1088 | ||||
1089 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1090 | ||||
1091 | void AllocaSlices::print(raw_ostream &OS, const_iterator I, | |||
1092 | StringRef Indent) const { | |||
1093 | printSlice(OS, I, Indent); | |||
1094 | OS << "\n"; | |||
1095 | printUse(OS, I, Indent); | |||
1096 | } | |||
1097 | ||||
1098 | void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I, | |||
1099 | StringRef Indent) const { | |||
1100 | OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")" | |||
1101 | << " slice #" << (I - begin()) | |||
1102 | << (I->isSplittable() ? " (splittable)" : ""); | |||
1103 | } | |||
1104 | ||||
1105 | void AllocaSlices::printUse(raw_ostream &OS, const_iterator I, | |||
1106 | StringRef Indent) const { | |||
1107 | OS << Indent << " used by: " << *I->getUse()->getUser() << "\n"; | |||
1108 | } | |||
1109 | ||||
1110 | void AllocaSlices::print(raw_ostream &OS) const { | |||
1111 | if (PointerEscapingInstr) { | |||
1112 | OS << "Can't analyze slices for alloca: " << AI << "\n" | |||
1113 | << " A pointer to this alloca escaped by:\n" | |||
1114 | << " " << *PointerEscapingInstr << "\n"; | |||
1115 | return; | |||
1116 | } | |||
1117 | ||||
1118 | OS << "Slices of alloca: " << AI << "\n"; | |||
1119 | for (const_iterator I = begin(), E = end(); I != E; ++I) | |||
1120 | print(OS, I); | |||
1121 | } | |||
1122 | ||||
1123 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void AllocaSlices::dump(const_iterator I) const { | |||
1124 | print(dbgs(), I); | |||
1125 | } | |||
1126 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void AllocaSlices::dump() const { print(dbgs()); } | |||
1127 | ||||
1128 | #endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||
1129 | ||||
1130 | /// Walk the range of a partitioning looking for a common type to cover this | |||
1131 | /// sequence of slices. | |||
1132 | static std::pair<Type *, IntegerType *> | |||
1133 | findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E, | |||
1134 | uint64_t EndOffset) { | |||
1135 | Type *Ty = nullptr; | |||
1136 | bool TyIsCommon = true; | |||
1137 | IntegerType *ITy = nullptr; | |||
1138 | ||||
1139 | // Note that we need to look at *every* alloca slice's Use to ensure we | |||
1140 | // always get consistent results regardless of the order of slices. | |||
1141 | for (AllocaSlices::const_iterator I = B; I != E; ++I) { | |||
1142 | Use *U = I->getUse(); | |||
1143 | if (isa<IntrinsicInst>(*U->getUser())) | |||
1144 | continue; | |||
1145 | if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset) | |||
1146 | continue; | |||
1147 | ||||
1148 | Type *UserTy = nullptr; | |||
1149 | if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | |||
1150 | UserTy = LI->getType(); | |||
1151 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | |||
1152 | UserTy = SI->getValueOperand()->getType(); | |||
1153 | } | |||
1154 | ||||
1155 | if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(UserTy)) { | |||
1156 | // If the type is larger than the partition, skip it. We only encounter | |||
1157 | // this for split integer operations where we want to use the type of the | |||
1158 | // entity causing the split. Also skip if the type is not a byte width | |||
1159 | // multiple. | |||
1160 | if (UserITy->getBitWidth() % 8 != 0 || | |||
1161 | UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset())) | |||
1162 | continue; | |||
1163 | ||||
1164 | // Track the largest bitwidth integer type used in this way in case there | |||
1165 | // is no common type. | |||
1166 | if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth()) | |||
1167 | ITy = UserITy; | |||
1168 | } | |||
1169 | ||||
1170 | // To avoid depending on the order of slices, Ty and TyIsCommon must not | |||
1171 | // depend on types skipped above. | |||
1172 | if (!UserTy || (Ty && Ty != UserTy)) | |||
1173 | TyIsCommon = false; // Give up on anything but an iN type. | |||
1174 | else | |||
1175 | Ty = UserTy; | |||
1176 | } | |||
1177 | ||||
1178 | return {TyIsCommon ? Ty : nullptr, ITy}; | |||
1179 | } | |||
1180 | ||||
1181 | /// PHI instructions that use an alloca and are subsequently loaded can be | |||
1182 | /// rewritten to load both input pointers in the pred blocks and then PHI the | |||
1183 | /// results, allowing the load of the alloca to be promoted. | |||
1184 | /// From this: | |||
1185 | /// %P2 = phi [i32* %Alloca, i32* %Other] | |||
1186 | /// %V = load i32* %P2 | |||
1187 | /// to: | |||
1188 | /// %V1 = load i32* %Alloca -> will be mem2reg'd | |||
1189 | /// ... | |||
1190 | /// %V2 = load i32* %Other | |||
1191 | /// ... | |||
1192 | /// %V = phi [i32 %V1, i32 %V2] | |||
1193 | /// | |||
1194 | /// We can do this to a select if its only uses are loads and if the operands | |||
1195 | /// to the select can be loaded unconditionally. | |||
1196 | /// | |||
1197 | /// FIXME: This should be hoisted into a generic utility, likely in | |||
1198 | /// Transforms/Util/Local.h | |||
1199 | static bool isSafePHIToSpeculate(PHINode &PN) { | |||
1200 | const DataLayout &DL = PN.getModule()->getDataLayout(); | |||
1201 | ||||
1202 | // For now, we can only do this promotion if the load is in the same block | |||
1203 | // as the PHI, and if there are no stores between the phi and load. | |||
1204 | // TODO: Allow recursive phi users. | |||
1205 | // TODO: Allow stores. | |||
1206 | BasicBlock *BB = PN.getParent(); | |||
1207 | Align MaxAlign; | |||
1208 | uint64_t APWidth = DL.getIndexTypeSizeInBits(PN.getType()); | |||
1209 | APInt MaxSize(APWidth, 0); | |||
1210 | bool HaveLoad = false; | |||
1211 | for (User *U : PN.users()) { | |||
1212 | LoadInst *LI = dyn_cast<LoadInst>(U); | |||
1213 | if (!LI || !LI->isSimple()) | |||
1214 | return false; | |||
1215 | ||||
1216 | // For now we only allow loads in the same block as the PHI. This is | |||
1217 | // a common case that happens when instcombine merges two loads through | |||
1218 | // a PHI. | |||
1219 | if (LI->getParent() != BB) | |||
1220 | return false; | |||
1221 | ||||
1222 | // Ensure that there are no instructions between the PHI and the load that | |||
1223 | // could store. | |||
1224 | for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI) | |||
1225 | if (BBI->mayWriteToMemory()) | |||
1226 | return false; | |||
1227 | ||||
1228 | uint64_t Size = DL.getTypeStoreSize(LI->getType()).getFixedSize(); | |||
1229 | MaxAlign = std::max(MaxAlign, LI->getAlign()); | |||
1230 | MaxSize = MaxSize.ult(Size) ? APInt(APWidth, Size) : MaxSize; | |||
1231 | HaveLoad = true; | |||
1232 | } | |||
1233 | ||||
1234 | if (!HaveLoad) | |||
1235 | return false; | |||
1236 | ||||
1237 | // We can only transform this if it is safe to push the loads into the | |||
1238 | // predecessor blocks. The only thing to watch out for is that we can't put | |||
1239 | // a possibly trapping load in the predecessor if it is a critical edge. | |||
1240 | for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { | |||
1241 | Instruction *TI = PN.getIncomingBlock(Idx)->getTerminator(); | |||
1242 | Value *InVal = PN.getIncomingValue(Idx); | |||
1243 | ||||
1244 | // If the value is produced by the terminator of the predecessor (an | |||
1245 | // invoke) or it has side-effects, there is no valid place to put a load | |||
1246 | // in the predecessor. | |||
1247 | if (TI == InVal || TI->mayHaveSideEffects()) | |||
1248 | return false; | |||
1249 | ||||
1250 | // If the predecessor has a single successor, then the edge isn't | |||
1251 | // critical. | |||
1252 | if (TI->getNumSuccessors() == 1) | |||
1253 | continue; | |||
1254 | ||||
1255 | // If this pointer is always safe to load, or if we can prove that there | |||
1256 | // is already a load in the block, then we can move the load to the pred | |||
1257 | // block. | |||
1258 | if (isSafeToLoadUnconditionally(InVal, MaxAlign, MaxSize, DL, TI)) | |||
1259 | continue; | |||
1260 | ||||
1261 | return false; | |||
1262 | } | |||
1263 | ||||
1264 | return true; | |||
1265 | } | |||
1266 | ||||
1267 | static void speculatePHINodeLoads(PHINode &PN) { | |||
1268 | LLVM_DEBUG(dbgs() << " original: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << PN << "\n"; } } while (false); | |||
1269 | ||||
1270 | LoadInst *SomeLoad = cast<LoadInst>(PN.user_back()); | |||
1271 | Type *LoadTy = SomeLoad->getType(); | |||
1272 | IRBuilderTy PHIBuilder(&PN); | |||
1273 | PHINode *NewPN = PHIBuilder.CreatePHI(LoadTy, PN.getNumIncomingValues(), | |||
1274 | PN.getName() + ".sroa.speculated"); | |||
1275 | ||||
1276 | // Get the AA tags and alignment to use from one of the loads. It does not | |||
1277 | // matter which one we get and if any differ. | |||
1278 | AAMDNodes AATags; | |||
1279 | SomeLoad->getAAMetadata(AATags); | |||
1280 | Align Alignment = SomeLoad->getAlign(); | |||
1281 | ||||
1282 | // Rewrite all loads of the PN to use the new PHI. | |||
1283 | while (!PN.use_empty()) { | |||
1284 | LoadInst *LI = cast<LoadInst>(PN.user_back()); | |||
1285 | LI->replaceAllUsesWith(NewPN); | |||
1286 | LI->eraseFromParent(); | |||
1287 | } | |||
1288 | ||||
1289 | // Inject loads into all of the pred blocks. | |||
1290 | DenseMap<BasicBlock*, Value*> InjectedLoads; | |||
1291 | for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { | |||
1292 | BasicBlock *Pred = PN.getIncomingBlock(Idx); | |||
1293 | Value *InVal = PN.getIncomingValue(Idx); | |||
1294 | ||||
1295 | // A PHI node is allowed to have multiple (duplicated) entries for the same | |||
1296 | // basic block, as long as the value is the same. So if we already injected | |||
1297 | // a load in the predecessor, then we should reuse the same load for all | |||
1298 | // duplicated entries. | |||
1299 | if (Value* V = InjectedLoads.lookup(Pred)) { | |||
1300 | NewPN->addIncoming(V, Pred); | |||
1301 | continue; | |||
1302 | } | |||
1303 | ||||
1304 | Instruction *TI = Pred->getTerminator(); | |||
1305 | IRBuilderTy PredBuilder(TI); | |||
1306 | ||||
1307 | LoadInst *Load = PredBuilder.CreateAlignedLoad( | |||
1308 | LoadTy, InVal, Alignment, | |||
1309 | (PN.getName() + ".sroa.speculate.load." + Pred->getName())); | |||
1310 | ++NumLoadsSpeculated; | |||
1311 | if (AATags) | |||
1312 | Load->setAAMetadata(AATags); | |||
1313 | NewPN->addIncoming(Load, Pred); | |||
1314 | InjectedLoads[Pred] = Load; | |||
1315 | } | |||
1316 | ||||
1317 | LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " speculated to: " << *NewPN << "\n"; } } while (false); | |||
1318 | PN.eraseFromParent(); | |||
1319 | } | |||
1320 | ||||
1321 | /// Select instructions that use an alloca and are subsequently loaded can be | |||
1322 | /// rewritten to load both input pointers and then select between the result, | |||
1323 | /// allowing the load of the alloca to be promoted. | |||
1324 | /// From this: | |||
1325 | /// %P2 = select i1 %cond, i32* %Alloca, i32* %Other | |||
1326 | /// %V = load i32* %P2 | |||
1327 | /// to: | |||
1328 | /// %V1 = load i32* %Alloca -> will be mem2reg'd | |||
1329 | /// %V2 = load i32* %Other | |||
1330 | /// %V = select i1 %cond, i32 %V1, i32 %V2 | |||
1331 | /// | |||
1332 | /// We can do this to a select if its only uses are loads and if the operand | |||
1333 | /// to the select can be loaded unconditionally. If found an intervening bitcast | |||
1334 | /// with a single use of the load, allow the promotion. | |||
1335 | static bool isSafeSelectToSpeculate(SelectInst &SI) { | |||
1336 | Value *TValue = SI.getTrueValue(); | |||
1337 | Value *FValue = SI.getFalseValue(); | |||
1338 | const DataLayout &DL = SI.getModule()->getDataLayout(); | |||
1339 | ||||
1340 | for (User *U : SI.users()) { | |||
1341 | LoadInst *LI; | |||
1342 | BitCastInst *BC = dyn_cast<BitCastInst>(U); | |||
1343 | if (BC && BC->hasOneUse()) | |||
1344 | LI = dyn_cast<LoadInst>(*BC->user_begin()); | |||
1345 | else | |||
1346 | LI = dyn_cast<LoadInst>(U); | |||
1347 | ||||
1348 | if (!LI || !LI->isSimple()) | |||
1349 | return false; | |||
1350 | ||||
1351 | // Both operands to the select need to be dereferenceable, either | |||
1352 | // absolutely (e.g. allocas) or at this point because we can see other | |||
1353 | // accesses to it. | |||
1354 | if (!isSafeToLoadUnconditionally(TValue, LI->getType(), | |||
1355 | LI->getAlign(), DL, LI)) | |||
1356 | return false; | |||
1357 | if (!isSafeToLoadUnconditionally(FValue, LI->getType(), | |||
1358 | LI->getAlign(), DL, LI)) | |||
1359 | return false; | |||
1360 | } | |||
1361 | ||||
1362 | return true; | |||
1363 | } | |||
1364 | ||||
1365 | static void speculateSelectInstLoads(SelectInst &SI) { | |||
1366 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||
1367 | ||||
1368 | IRBuilderTy IRB(&SI); | |||
1369 | Value *TV = SI.getTrueValue(); | |||
1370 | Value *FV = SI.getFalseValue(); | |||
1371 | // Replace the loads of the select with a select of two loads. | |||
1372 | while (!SI.use_empty()) { | |||
1373 | LoadInst *LI; | |||
1374 | BitCastInst *BC = dyn_cast<BitCastInst>(SI.user_back()); | |||
1375 | if (BC) { | |||
1376 | assert(BC->hasOneUse() && "Bitcast should have a single use.")(static_cast <bool> (BC->hasOneUse() && "Bitcast should have a single use." ) ? void (0) : __assert_fail ("BC->hasOneUse() && \"Bitcast should have a single use.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1376, __extension__ __PRETTY_FUNCTION__)); | |||
1377 | LI = cast<LoadInst>(BC->user_back()); | |||
1378 | } else { | |||
1379 | LI = cast<LoadInst>(SI.user_back()); | |||
1380 | } | |||
1381 | ||||
1382 | assert(LI->isSimple() && "We only speculate simple loads")(static_cast <bool> (LI->isSimple() && "We only speculate simple loads" ) ? void (0) : __assert_fail ("LI->isSimple() && \"We only speculate simple loads\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1382, __extension__ __PRETTY_FUNCTION__)); | |||
1383 | ||||
1384 | IRB.SetInsertPoint(LI); | |||
1385 | Value *NewTV = | |||
1386 | BC ? IRB.CreateBitCast(TV, BC->getType(), TV->getName() + ".sroa.cast") | |||
1387 | : TV; | |||
1388 | Value *NewFV = | |||
1389 | BC ? IRB.CreateBitCast(FV, BC->getType(), FV->getName() + ".sroa.cast") | |||
1390 | : FV; | |||
1391 | LoadInst *TL = IRB.CreateLoad(LI->getType(), NewTV, | |||
1392 | LI->getName() + ".sroa.speculate.load.true"); | |||
1393 | LoadInst *FL = IRB.CreateLoad(LI->getType(), NewFV, | |||
1394 | LI->getName() + ".sroa.speculate.load.false"); | |||
1395 | NumLoadsSpeculated += 2; | |||
1396 | ||||
1397 | // Transfer alignment and AA info if present. | |||
1398 | TL->setAlignment(LI->getAlign()); | |||
1399 | FL->setAlignment(LI->getAlign()); | |||
1400 | ||||
1401 | AAMDNodes Tags; | |||
1402 | LI->getAAMetadata(Tags); | |||
1403 | if (Tags) { | |||
1404 | TL->setAAMetadata(Tags); | |||
1405 | FL->setAAMetadata(Tags); | |||
1406 | } | |||
1407 | ||||
1408 | Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, | |||
1409 | LI->getName() + ".sroa.speculated"); | |||
1410 | ||||
1411 | LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " speculated to: " << *V << "\n"; } } while (false); | |||
1412 | LI->replaceAllUsesWith(V); | |||
1413 | LI->eraseFromParent(); | |||
1414 | if (BC) | |||
1415 | BC->eraseFromParent(); | |||
1416 | } | |||
1417 | SI.eraseFromParent(); | |||
1418 | } | |||
1419 | ||||
1420 | /// Build a GEP out of a base pointer and indices. | |||
1421 | /// | |||
1422 | /// This will return the BasePtr if that is valid, or build a new GEP | |||
1423 | /// instruction using the IRBuilder if GEP-ing is needed. | |||
1424 | static Value *buildGEP(IRBuilderTy &IRB, Value *BasePtr, | |||
1425 | SmallVectorImpl<Value *> &Indices, | |||
1426 | const Twine &NamePrefix) { | |||
1427 | if (Indices.empty()) | |||
1428 | return BasePtr; | |||
1429 | ||||
1430 | // A single zero index is a no-op, so check for this and avoid building a GEP | |||
1431 | // in that case. | |||
1432 | if (Indices.size() == 1 && cast<ConstantInt>(Indices.back())->isZero()) | |||
1433 | return BasePtr; | |||
1434 | ||||
1435 | return IRB.CreateInBoundsGEP(BasePtr->getType()->getPointerElementType(), | |||
1436 | BasePtr, Indices, NamePrefix + "sroa_idx"); | |||
1437 | } | |||
1438 | ||||
1439 | /// Get a natural GEP off of the BasePtr walking through Ty toward | |||
1440 | /// TargetTy without changing the offset of the pointer. | |||
1441 | /// | |||
1442 | /// This routine assumes we've already established a properly offset GEP with | |||
1443 | /// Indices, and arrived at the Ty type. The goal is to continue to GEP with | |||
1444 | /// zero-indices down through type layers until we find one the same as | |||
1445 | /// TargetTy. If we can't find one with the same type, we at least try to use | |||
1446 | /// one with the same size. If none of that works, we just produce the GEP as | |||
1447 | /// indicated by Indices to have the correct offset. | |||
1448 | static Value *getNaturalGEPWithType(IRBuilderTy &IRB, const DataLayout &DL, | |||
1449 | Value *BasePtr, Type *Ty, Type *TargetTy, | |||
1450 | SmallVectorImpl<Value *> &Indices, | |||
1451 | const Twine &NamePrefix) { | |||
1452 | if (Ty == TargetTy) | |||
1453 | return buildGEP(IRB, BasePtr, Indices, NamePrefix); | |||
1454 | ||||
1455 | // Offset size to use for the indices. | |||
1456 | unsigned OffsetSize = DL.getIndexTypeSizeInBits(BasePtr->getType()); | |||
1457 | ||||
1458 | // See if we can descend into a struct and locate a field with the correct | |||
1459 | // type. | |||
1460 | unsigned NumLayers = 0; | |||
1461 | Type *ElementTy = Ty; | |||
1462 | do { | |||
1463 | if (ElementTy->isPointerTy()) | |||
1464 | break; | |||
1465 | ||||
1466 | if (ArrayType *ArrayTy = dyn_cast<ArrayType>(ElementTy)) { | |||
1467 | ElementTy = ArrayTy->getElementType(); | |||
1468 | Indices.push_back(IRB.getIntN(OffsetSize, 0)); | |||
1469 | } else if (VectorType *VectorTy = dyn_cast<VectorType>(ElementTy)) { | |||
1470 | ElementTy = VectorTy->getElementType(); | |||
1471 | Indices.push_back(IRB.getInt32(0)); | |||
1472 | } else if (StructType *STy = dyn_cast<StructType>(ElementTy)) { | |||
1473 | if (STy->element_begin() == STy->element_end()) | |||
1474 | break; // Nothing left to descend into. | |||
1475 | ElementTy = *STy->element_begin(); | |||
1476 | Indices.push_back(IRB.getInt32(0)); | |||
1477 | } else { | |||
1478 | break; | |||
1479 | } | |||
1480 | ++NumLayers; | |||
1481 | } while (ElementTy != TargetTy); | |||
1482 | if (ElementTy != TargetTy) | |||
1483 | Indices.erase(Indices.end() - NumLayers, Indices.end()); | |||
1484 | ||||
1485 | return buildGEP(IRB, BasePtr, Indices, NamePrefix); | |||
1486 | } | |||
1487 | ||||
1488 | /// Recursively compute indices for a natural GEP. | |||
1489 | /// | |||
1490 | /// This is the recursive step for getNaturalGEPWithOffset that walks down the | |||
1491 | /// element types adding appropriate indices for the GEP. | |||
1492 | static Value *getNaturalGEPRecursively(IRBuilderTy &IRB, const DataLayout &DL, | |||
1493 | Value *Ptr, Type *Ty, APInt &Offset, | |||
1494 | Type *TargetTy, | |||
1495 | SmallVectorImpl<Value *> &Indices, | |||
1496 | const Twine &NamePrefix) { | |||
1497 | if (Offset == 0) | |||
1498 | return getNaturalGEPWithType(IRB, DL, Ptr, Ty, TargetTy, Indices, | |||
1499 | NamePrefix); | |||
1500 | ||||
1501 | // We can't recurse through pointer types. | |||
1502 | if (Ty->isPointerTy()) | |||
1503 | return nullptr; | |||
1504 | ||||
1505 | // We try to analyze GEPs over vectors here, but note that these GEPs are | |||
1506 | // extremely poorly defined currently. The long-term goal is to remove GEPing | |||
1507 | // over a vector from the IR completely. | |||
1508 | if (VectorType *VecTy = dyn_cast<VectorType>(Ty)) { | |||
1509 | unsigned ElementSizeInBits = | |||
1510 | DL.getTypeSizeInBits(VecTy->getScalarType()).getFixedSize(); | |||
1511 | if (ElementSizeInBits % 8 != 0) { | |||
1512 | // GEPs over non-multiple of 8 size vector elements are invalid. | |||
1513 | return nullptr; | |||
1514 | } | |||
1515 | APInt ElementSize(Offset.getBitWidth(), ElementSizeInBits / 8); | |||
1516 | APInt NumSkippedElements = Offset.sdiv(ElementSize); | |||
1517 | if (NumSkippedElements.ugt(cast<FixedVectorType>(VecTy)->getNumElements())) | |||
1518 | return nullptr; | |||
1519 | Offset -= NumSkippedElements * ElementSize; | |||
1520 | Indices.push_back(IRB.getInt(NumSkippedElements)); | |||
1521 | return getNaturalGEPRecursively(IRB, DL, Ptr, VecTy->getElementType(), | |||
1522 | Offset, TargetTy, Indices, NamePrefix); | |||
1523 | } | |||
1524 | ||||
1525 | if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) { | |||
1526 | Type *ElementTy = ArrTy->getElementType(); | |||
1527 | APInt ElementSize(Offset.getBitWidth(), | |||
1528 | DL.getTypeAllocSize(ElementTy).getFixedSize()); | |||
1529 | APInt NumSkippedElements = Offset.sdiv(ElementSize); | |||
1530 | if (NumSkippedElements.ugt(ArrTy->getNumElements())) | |||
1531 | return nullptr; | |||
1532 | ||||
1533 | Offset -= NumSkippedElements * ElementSize; | |||
1534 | Indices.push_back(IRB.getInt(NumSkippedElements)); | |||
1535 | return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy, | |||
1536 | Indices, NamePrefix); | |||
1537 | } | |||
1538 | ||||
1539 | StructType *STy = dyn_cast<StructType>(Ty); | |||
1540 | if (!STy) | |||
1541 | return nullptr; | |||
1542 | ||||
1543 | const StructLayout *SL = DL.getStructLayout(STy); | |||
1544 | uint64_t StructOffset = Offset.getZExtValue(); | |||
1545 | if (StructOffset >= SL->getSizeInBytes()) | |||
1546 | return nullptr; | |||
1547 | unsigned Index = SL->getElementContainingOffset(StructOffset); | |||
1548 | Offset -= APInt(Offset.getBitWidth(), SL->getElementOffset(Index)); | |||
1549 | Type *ElementTy = STy->getElementType(Index); | |||
1550 | if (Offset.uge(DL.getTypeAllocSize(ElementTy).getFixedSize())) | |||
1551 | return nullptr; // The offset points into alignment padding. | |||
1552 | ||||
1553 | Indices.push_back(IRB.getInt32(Index)); | |||
1554 | return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy, | |||
1555 | Indices, NamePrefix); | |||
1556 | } | |||
1557 | ||||
1558 | /// Get a natural GEP from a base pointer to a particular offset and | |||
1559 | /// resulting in a particular type. | |||
1560 | /// | |||
1561 | /// The goal is to produce a "natural" looking GEP that works with the existing | |||
1562 | /// composite types to arrive at the appropriate offset and element type for | |||
1563 | /// a pointer. TargetTy is the element type the returned GEP should point-to if | |||
1564 | /// possible. We recurse by decreasing Offset, adding the appropriate index to | |||
1565 | /// Indices, and setting Ty to the result subtype. | |||
1566 | /// | |||
1567 | /// If no natural GEP can be constructed, this function returns null. | |||
1568 | static Value *getNaturalGEPWithOffset(IRBuilderTy &IRB, const DataLayout &DL, | |||
1569 | Value *Ptr, APInt Offset, Type *TargetTy, | |||
1570 | SmallVectorImpl<Value *> &Indices, | |||
1571 | const Twine &NamePrefix) { | |||
1572 | PointerType *Ty = cast<PointerType>(Ptr->getType()); | |||
1573 | ||||
1574 | // Don't consider any GEPs through an i8* as natural unless the TargetTy is | |||
1575 | // an i8. | |||
1576 | if (Ty == IRB.getInt8PtrTy(Ty->getAddressSpace()) && TargetTy->isIntegerTy(8)) | |||
1577 | return nullptr; | |||
1578 | ||||
1579 | Type *ElementTy = Ty->getElementType(); | |||
1580 | if (!ElementTy->isSized()) | |||
1581 | return nullptr; // We can't GEP through an unsized element. | |||
1582 | if (isa<ScalableVectorType>(ElementTy)) | |||
1583 | return nullptr; | |||
1584 | APInt ElementSize(Offset.getBitWidth(), | |||
1585 | DL.getTypeAllocSize(ElementTy).getFixedSize()); | |||
1586 | if (ElementSize == 0) | |||
1587 | return nullptr; // Zero-length arrays can't help us build a natural GEP. | |||
1588 | APInt NumSkippedElements = Offset.sdiv(ElementSize); | |||
1589 | ||||
1590 | Offset -= NumSkippedElements * ElementSize; | |||
1591 | Indices.push_back(IRB.getInt(NumSkippedElements)); | |||
1592 | return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy, | |||
1593 | Indices, NamePrefix); | |||
1594 | } | |||
1595 | ||||
1596 | /// Compute an adjusted pointer from Ptr by Offset bytes where the | |||
1597 | /// resulting pointer has PointerTy. | |||
1598 | /// | |||
1599 | /// This tries very hard to compute a "natural" GEP which arrives at the offset | |||
1600 | /// and produces the pointer type desired. Where it cannot, it will try to use | |||
1601 | /// the natural GEP to arrive at the offset and bitcast to the type. Where that | |||
1602 | /// fails, it will try to use an existing i8* and GEP to the byte offset and | |||
1603 | /// bitcast to the type. | |||
1604 | /// | |||
1605 | /// The strategy for finding the more natural GEPs is to peel off layers of the | |||
1606 | /// pointer, walking back through bit casts and GEPs, searching for a base | |||
1607 | /// pointer from which we can compute a natural GEP with the desired | |||
1608 | /// properties. The algorithm tries to fold as many constant indices into | |||
1609 | /// a single GEP as possible, thus making each GEP more independent of the | |||
1610 | /// surrounding code. | |||
1611 | static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr, | |||
1612 | APInt Offset, Type *PointerTy, | |||
1613 | const Twine &NamePrefix) { | |||
1614 | // Even though we don't look through PHI nodes, we could be called on an | |||
1615 | // instruction in an unreachable block, which may be on a cycle. | |||
1616 | SmallPtrSet<Value *, 4> Visited; | |||
1617 | Visited.insert(Ptr); | |||
1618 | SmallVector<Value *, 4> Indices; | |||
1619 | ||||
1620 | // We may end up computing an offset pointer that has the wrong type. If we | |||
1621 | // never are able to compute one directly that has the correct type, we'll | |||
1622 | // fall back to it, so keep it and the base it was computed from around here. | |||
1623 | Value *OffsetPtr = nullptr; | |||
1624 | Value *OffsetBasePtr; | |||
1625 | ||||
1626 | // Remember any i8 pointer we come across to re-use if we need to do a raw | |||
1627 | // byte offset. | |||
1628 | Value *Int8Ptr = nullptr; | |||
1629 | APInt Int8PtrOffset(Offset.getBitWidth(), 0); | |||
1630 | ||||
1631 | PointerType *TargetPtrTy = cast<PointerType>(PointerTy); | |||
1632 | Type *TargetTy = TargetPtrTy->getElementType(); | |||
1633 | ||||
1634 | // As `addrspacecast` is , `Ptr` (the storage pointer) may have different | |||
1635 | // address space from the expected `PointerTy` (the pointer to be used). | |||
1636 | // Adjust the pointer type based the original storage pointer. | |||
1637 | auto AS = cast<PointerType>(Ptr->getType())->getAddressSpace(); | |||
1638 | PointerTy = TargetTy->getPointerTo(AS); | |||
1639 | ||||
1640 | do { | |||
1641 | // First fold any existing GEPs into the offset. | |||
1642 | while (GEPOperator *GEP = dyn_cast<GEPOperator>(Ptr)) { | |||
1643 | APInt GEPOffset(Offset.getBitWidth(), 0); | |||
1644 | if (!GEP->accumulateConstantOffset(DL, GEPOffset)) | |||
1645 | break; | |||
1646 | Offset += GEPOffset; | |||
1647 | Ptr = GEP->getPointerOperand(); | |||
1648 | if (!Visited.insert(Ptr).second) | |||
1649 | break; | |||
1650 | } | |||
1651 | ||||
1652 | // See if we can perform a natural GEP here. | |||
1653 | Indices.clear(); | |||
1654 | if (Value *P = getNaturalGEPWithOffset(IRB, DL, Ptr, Offset, TargetTy, | |||
1655 | Indices, NamePrefix)) { | |||
1656 | // If we have a new natural pointer at the offset, clear out any old | |||
1657 | // offset pointer we computed. Unless it is the base pointer or | |||
1658 | // a non-instruction, we built a GEP we don't need. Zap it. | |||
1659 | if (OffsetPtr && OffsetPtr != OffsetBasePtr) | |||
1660 | if (Instruction *I = dyn_cast<Instruction>(OffsetPtr)) { | |||
1661 | assert(I->use_empty() && "Built a GEP with uses some how!")(static_cast <bool> (I->use_empty() && "Built a GEP with uses some how!" ) ? void (0) : __assert_fail ("I->use_empty() && \"Built a GEP with uses some how!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1661, __extension__ __PRETTY_FUNCTION__)); | |||
1662 | I->eraseFromParent(); | |||
1663 | } | |||
1664 | OffsetPtr = P; | |||
1665 | OffsetBasePtr = Ptr; | |||
1666 | // If we also found a pointer of the right type, we're done. | |||
1667 | if (P->getType() == PointerTy) | |||
1668 | break; | |||
1669 | } | |||
1670 | ||||
1671 | // Stash this pointer if we've found an i8*. | |||
1672 | if (Ptr->getType()->isIntegerTy(8)) { | |||
1673 | Int8Ptr = Ptr; | |||
1674 | Int8PtrOffset = Offset; | |||
1675 | } | |||
1676 | ||||
1677 | // Peel off a layer of the pointer and update the offset appropriately. | |||
1678 | if (Operator::getOpcode(Ptr) == Instruction::BitCast) { | |||
1679 | Ptr = cast<Operator>(Ptr)->getOperand(0); | |||
1680 | } else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(Ptr)) { | |||
1681 | if (GA->isInterposable()) | |||
1682 | break; | |||
1683 | Ptr = GA->getAliasee(); | |||
1684 | } else { | |||
1685 | break; | |||
1686 | } | |||
1687 | assert(Ptr->getType()->isPointerTy() && "Unexpected operand type!")(static_cast <bool> (Ptr->getType()->isPointerTy( ) && "Unexpected operand type!") ? void (0) : __assert_fail ("Ptr->getType()->isPointerTy() && \"Unexpected operand type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1687, __extension__ __PRETTY_FUNCTION__)); | |||
1688 | } while (Visited.insert(Ptr).second); | |||
1689 | ||||
1690 | if (!OffsetPtr) { | |||
1691 | if (!Int8Ptr) { | |||
1692 | Int8Ptr = IRB.CreateBitCast( | |||
1693 | Ptr, IRB.getInt8PtrTy(PointerTy->getPointerAddressSpace()), | |||
1694 | NamePrefix + "sroa_raw_cast"); | |||
1695 | Int8PtrOffset = Offset; | |||
1696 | } | |||
1697 | ||||
1698 | OffsetPtr = Int8PtrOffset == 0 | |||
1699 | ? Int8Ptr | |||
1700 | : IRB.CreateInBoundsGEP(IRB.getInt8Ty(), Int8Ptr, | |||
1701 | IRB.getInt(Int8PtrOffset), | |||
1702 | NamePrefix + "sroa_raw_idx"); | |||
1703 | } | |||
1704 | Ptr = OffsetPtr; | |||
1705 | ||||
1706 | // On the off chance we were targeting i8*, guard the bitcast here. | |||
1707 | if (cast<PointerType>(Ptr->getType()) != TargetPtrTy) { | |||
1708 | Ptr = IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, | |||
1709 | TargetPtrTy, | |||
1710 | NamePrefix + "sroa_cast"); | |||
1711 | } | |||
1712 | ||||
1713 | return Ptr; | |||
1714 | } | |||
1715 | ||||
1716 | /// Compute the adjusted alignment for a load or store from an offset. | |||
1717 | static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) { | |||
1718 | return commonAlignment(getLoadStoreAlignment(I), Offset); | |||
1719 | } | |||
1720 | ||||
1721 | /// Test whether we can convert a value from the old to the new type. | |||
1722 | /// | |||
1723 | /// This predicate should be used to guard calls to convertValue in order to | |||
1724 | /// ensure that we only try to convert viable values. The strategy is that we | |||
1725 | /// will peel off single element struct and array wrappings to get to an | |||
1726 | /// underlying value, and convert that value. | |||
1727 | static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) { | |||
1728 | if (OldTy == NewTy) | |||
1729 | return true; | |||
1730 | ||||
1731 | // For integer types, we can't handle any bit-width differences. This would | |||
1732 | // break both vector conversions with extension and introduce endianness | |||
1733 | // issues when in conjunction with loads and stores. | |||
1734 | if (isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) { | |||
1735 | assert(cast<IntegerType>(OldTy)->getBitWidth() !=(static_cast <bool> (cast<IntegerType>(OldTy)-> getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth () && "We can't have the same bitwidth for different int types" ) ? void (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1737, __extension__ __PRETTY_FUNCTION__)) | |||
1736 | cast<IntegerType>(NewTy)->getBitWidth() &&(static_cast <bool> (cast<IntegerType>(OldTy)-> getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth () && "We can't have the same bitwidth for different int types" ) ? void (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1737, __extension__ __PRETTY_FUNCTION__)) | |||
1737 | "We can't have the same bitwidth for different int types")(static_cast <bool> (cast<IntegerType>(OldTy)-> getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth () && "We can't have the same bitwidth for different int types" ) ? void (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1737, __extension__ __PRETTY_FUNCTION__)); | |||
1738 | return false; | |||
1739 | } | |||
1740 | ||||
1741 | if (DL.getTypeSizeInBits(NewTy).getFixedSize() != | |||
1742 | DL.getTypeSizeInBits(OldTy).getFixedSize()) | |||
1743 | return false; | |||
1744 | if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType()) | |||
1745 | return false; | |||
1746 | ||||
1747 | // We can convert pointers to integers and vice-versa. Same for vectors | |||
1748 | // of pointers and integers. | |||
1749 | OldTy = OldTy->getScalarType(); | |||
1750 | NewTy = NewTy->getScalarType(); | |||
1751 | if (NewTy->isPointerTy() || OldTy->isPointerTy()) { | |||
1752 | if (NewTy->isPointerTy() && OldTy->isPointerTy()) { | |||
1753 | unsigned OldAS = OldTy->getPointerAddressSpace(); | |||
1754 | unsigned NewAS = NewTy->getPointerAddressSpace(); | |||
1755 | // Convert pointers if they are pointers from the same address space or | |||
1756 | // different integral (not non-integral) address spaces with the same | |||
1757 | // pointer size. | |||
1758 | return OldAS == NewAS || | |||
1759 | (!DL.isNonIntegralAddressSpace(OldAS) && | |||
1760 | !DL.isNonIntegralAddressSpace(NewAS) && | |||
1761 | DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS)); | |||
1762 | } | |||
1763 | ||||
1764 | // We can convert integers to integral pointers, but not to non-integral | |||
1765 | // pointers. | |||
1766 | if (OldTy->isIntegerTy()) | |||
1767 | return !DL.isNonIntegralPointerType(NewTy); | |||
1768 | ||||
1769 | // We can convert integral pointers to integers, but non-integral pointers | |||
1770 | // need to remain pointers. | |||
1771 | if (!DL.isNonIntegralPointerType(OldTy)) | |||
1772 | return NewTy->isIntegerTy(); | |||
1773 | ||||
1774 | return false; | |||
1775 | } | |||
1776 | ||||
1777 | return true; | |||
1778 | } | |||
1779 | ||||
1780 | /// Generic routine to convert an SSA value to a value of a different | |||
1781 | /// type. | |||
1782 | /// | |||
1783 | /// This will try various different casting techniques, such as bitcasts, | |||
1784 | /// inttoptr, and ptrtoint casts. Use the \c canConvertValue predicate to test | |||
1785 | /// two types for viability with this routine. | |||
1786 | static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V, | |||
1787 | Type *NewTy) { | |||
1788 | Type *OldTy = V->getType(); | |||
1789 | assert(canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type")(static_cast <bool> (canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type") ? void (0) : __assert_fail ( "canConvertValue(DL, OldTy, NewTy) && \"Value not convertable to type\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1789, __extension__ __PRETTY_FUNCTION__)); | |||
1790 | ||||
1791 | if (OldTy == NewTy) | |||
1792 | return V; | |||
1793 | ||||
1794 | assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&(static_cast <bool> (!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && "Integer types must be the exact same to convert." ) ? void (0) : __assert_fail ("!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && \"Integer types must be the exact same to convert.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1795, __extension__ __PRETTY_FUNCTION__)) | |||
1795 | "Integer types must be the exact same to convert.")(static_cast <bool> (!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && "Integer types must be the exact same to convert." ) ? void (0) : __assert_fail ("!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && \"Integer types must be the exact same to convert.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1795, __extension__ __PRETTY_FUNCTION__)); | |||
1796 | ||||
1797 | // See if we need inttoptr for this type pair. May require additional bitcast. | |||
1798 | if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) { | |||
1799 | // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8* | |||
1800 | // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*> | |||
1801 | // Expand <4 x i32> to <2 x i8*> --> <4 x i32> to <2 x i64> to <2 x i8*> | |||
1802 | // Directly handle i64 to i8* | |||
1803 | return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)), | |||
1804 | NewTy); | |||
1805 | } | |||
1806 | ||||
1807 | // See if we need ptrtoint for this type pair. May require additional bitcast. | |||
1808 | if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) { | |||
1809 | // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128 | |||
1810 | // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32> | |||
1811 | // Expand <2 x i8*> to <4 x i32> --> <2 x i8*> to <2 x i64> to <4 x i32> | |||
1812 | // Expand i8* to i64 --> i8* to i64 to i64 | |||
1813 | return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)), | |||
1814 | NewTy); | |||
1815 | } | |||
1816 | ||||
1817 | if (OldTy->isPtrOrPtrVectorTy() && NewTy->isPtrOrPtrVectorTy()) { | |||
1818 | unsigned OldAS = OldTy->getPointerAddressSpace(); | |||
1819 | unsigned NewAS = NewTy->getPointerAddressSpace(); | |||
1820 | // To convert pointers with different address spaces (they are already | |||
1821 | // checked convertible, i.e. they have the same pointer size), so far we | |||
1822 | // cannot use `bitcast` (which has restrict on the same address space) or | |||
1823 | // `addrspacecast` (which is not always no-op casting). Instead, use a pair | |||
1824 | // of no-op `ptrtoint`/`inttoptr` casts through an integer with the same bit | |||
1825 | // size. | |||
1826 | if (OldAS != NewAS) { | |||
1827 | assert(DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS))(static_cast <bool> (DL.getPointerSize(OldAS) == DL.getPointerSize (NewAS)) ? void (0) : __assert_fail ("DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1827, __extension__ __PRETTY_FUNCTION__)); | |||
1828 | return IRB.CreateIntToPtr(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)), | |||
1829 | NewTy); | |||
1830 | } | |||
1831 | } | |||
1832 | ||||
1833 | return IRB.CreateBitCast(V, NewTy); | |||
1834 | } | |||
1835 | ||||
1836 | /// Test whether the given slice use can be promoted to a vector. | |||
1837 | /// | |||
1838 | /// This function is called to test each entry in a partition which is slated | |||
1839 | /// for a single slice. | |||
1840 | static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S, | |||
1841 | VectorType *Ty, | |||
1842 | uint64_t ElementSize, | |||
1843 | const DataLayout &DL) { | |||
1844 | // First validate the slice offsets. | |||
1845 | uint64_t BeginOffset = | |||
1846 | std::max(S.beginOffset(), P.beginOffset()) - P.beginOffset(); | |||
1847 | uint64_t BeginIndex = BeginOffset / ElementSize; | |||
1848 | if (BeginIndex * ElementSize != BeginOffset || | |||
1849 | BeginIndex >= cast<FixedVectorType>(Ty)->getNumElements()) | |||
1850 | return false; | |||
1851 | uint64_t EndOffset = | |||
1852 | std::min(S.endOffset(), P.endOffset()) - P.beginOffset(); | |||
1853 | uint64_t EndIndex = EndOffset / ElementSize; | |||
1854 | if (EndIndex * ElementSize != EndOffset || | |||
1855 | EndIndex > cast<FixedVectorType>(Ty)->getNumElements()) | |||
1856 | return false; | |||
1857 | ||||
1858 | assert(EndIndex > BeginIndex && "Empty vector!")(static_cast <bool> (EndIndex > BeginIndex && "Empty vector!") ? void (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1858, __extension__ __PRETTY_FUNCTION__)); | |||
1859 | uint64_t NumElements = EndIndex - BeginIndex; | |||
1860 | Type *SliceTy = (NumElements == 1) | |||
1861 | ? Ty->getElementType() | |||
1862 | : FixedVectorType::get(Ty->getElementType(), NumElements); | |||
1863 | ||||
1864 | Type *SplitIntTy = | |||
1865 | Type::getIntNTy(Ty->getContext(), NumElements * ElementSize * 8); | |||
1866 | ||||
1867 | Use *U = S.getUse(); | |||
1868 | ||||
1869 | if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { | |||
1870 | if (MI->isVolatile()) | |||
1871 | return false; | |||
1872 | if (!S.isSplittable()) | |||
1873 | return false; // Skip any unsplittable intrinsics. | |||
1874 | } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) { | |||
1875 | if (!II->isLifetimeStartOrEnd() && !II->isDroppable()) | |||
1876 | return false; | |||
1877 | } else if (U->get()->getType()->getPointerElementType()->isStructTy()) { | |||
1878 | // Disable vector promotion when there are loads or stores of an FCA. | |||
1879 | return false; | |||
1880 | } else if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | |||
1881 | if (LI->isVolatile()) | |||
1882 | return false; | |||
1883 | Type *LTy = LI->getType(); | |||
1884 | if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) { | |||
1885 | assert(LTy->isIntegerTy())(static_cast <bool> (LTy->isIntegerTy()) ? void (0) : __assert_fail ("LTy->isIntegerTy()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1885, __extension__ __PRETTY_FUNCTION__)); | |||
1886 | LTy = SplitIntTy; | |||
1887 | } | |||
1888 | if (!canConvertValue(DL, SliceTy, LTy)) | |||
1889 | return false; | |||
1890 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | |||
1891 | if (SI->isVolatile()) | |||
1892 | return false; | |||
1893 | Type *STy = SI->getValueOperand()->getType(); | |||
1894 | if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) { | |||
1895 | assert(STy->isIntegerTy())(static_cast <bool> (STy->isIntegerTy()) ? void (0) : __assert_fail ("STy->isIntegerTy()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1895, __extension__ __PRETTY_FUNCTION__)); | |||
1896 | STy = SplitIntTy; | |||
1897 | } | |||
1898 | if (!canConvertValue(DL, STy, SliceTy)) | |||
1899 | return false; | |||
1900 | } else { | |||
1901 | return false; | |||
1902 | } | |||
1903 | ||||
1904 | return true; | |||
1905 | } | |||
1906 | ||||
1907 | /// Test whether the given alloca partitioning and range of slices can be | |||
1908 | /// promoted to a vector. | |||
1909 | /// | |||
1910 | /// This is a quick test to check whether we can rewrite a particular alloca | |||
1911 | /// partition (and its newly formed alloca) into a vector alloca with only | |||
1912 | /// whole-vector loads and stores such that it could be promoted to a vector | |||
1913 | /// SSA value. We only can ensure this for a limited set of operations, and we | |||
1914 | /// don't want to do the rewrites unless we are confident that the result will | |||
1915 | /// be promotable, so we have an early test here. | |||
1916 | static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) { | |||
1917 | // Collect the candidate types for vector-based promotion. Also track whether | |||
1918 | // we have different element types. | |||
1919 | SmallVector<VectorType *, 4> CandidateTys; | |||
1920 | Type *CommonEltTy = nullptr; | |||
1921 | bool HaveCommonEltTy = true; | |||
1922 | auto CheckCandidateType = [&](Type *Ty) { | |||
1923 | if (auto *VTy = dyn_cast<VectorType>(Ty)) { | |||
1924 | // Return if bitcast to vectors is different for total size in bits. | |||
1925 | if (!CandidateTys.empty()) { | |||
1926 | VectorType *V = CandidateTys[0]; | |||
1927 | if (DL.getTypeSizeInBits(VTy).getFixedSize() != | |||
1928 | DL.getTypeSizeInBits(V).getFixedSize()) { | |||
1929 | CandidateTys.clear(); | |||
1930 | return; | |||
1931 | } | |||
1932 | } | |||
1933 | CandidateTys.push_back(VTy); | |||
1934 | if (!CommonEltTy) | |||
1935 | CommonEltTy = VTy->getElementType(); | |||
1936 | else if (CommonEltTy != VTy->getElementType()) | |||
1937 | HaveCommonEltTy = false; | |||
1938 | } | |||
1939 | }; | |||
1940 | // Consider any loads or stores that are the exact size of the slice. | |||
1941 | for (const Slice &S : P) | |||
1942 | if (S.beginOffset() == P.beginOffset() && | |||
1943 | S.endOffset() == P.endOffset()) { | |||
1944 | if (auto *LI = dyn_cast<LoadInst>(S.getUse()->getUser())) | |||
1945 | CheckCandidateType(LI->getType()); | |||
1946 | else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser())) | |||
1947 | CheckCandidateType(SI->getValueOperand()->getType()); | |||
1948 | } | |||
1949 | ||||
1950 | // If we didn't find a vector type, nothing to do here. | |||
1951 | if (CandidateTys.empty()) | |||
1952 | return nullptr; | |||
1953 | ||||
1954 | // Remove non-integer vector types if we had multiple common element types. | |||
1955 | // FIXME: It'd be nice to replace them with integer vector types, but we can't | |||
1956 | // do that until all the backends are known to produce good code for all | |||
1957 | // integer vector types. | |||
1958 | if (!HaveCommonEltTy) { | |||
1959 | llvm::erase_if(CandidateTys, [](VectorType *VTy) { | |||
1960 | return !VTy->getElementType()->isIntegerTy(); | |||
1961 | }); | |||
1962 | ||||
1963 | // If there were no integer vector types, give up. | |||
1964 | if (CandidateTys.empty()) | |||
1965 | return nullptr; | |||
1966 | ||||
1967 | // Rank the remaining candidate vector types. This is easy because we know | |||
1968 | // they're all integer vectors. We sort by ascending number of elements. | |||
1969 | auto RankVectorTypes = [&DL](VectorType *RHSTy, VectorType *LHSTy) { | |||
1970 | (void)DL; | |||
1971 | assert(DL.getTypeSizeInBits(RHSTy).getFixedSize() ==(static_cast <bool> (DL.getTypeSizeInBits(RHSTy).getFixedSize () == DL.getTypeSizeInBits(LHSTy).getFixedSize() && "Cannot have vector types of different sizes!" ) ? void (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy).getFixedSize() == DL.getTypeSizeInBits(LHSTy).getFixedSize() && \"Cannot have vector types of different sizes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1973, __extension__ __PRETTY_FUNCTION__)) | |||
1972 | DL.getTypeSizeInBits(LHSTy).getFixedSize() &&(static_cast <bool> (DL.getTypeSizeInBits(RHSTy).getFixedSize () == DL.getTypeSizeInBits(LHSTy).getFixedSize() && "Cannot have vector types of different sizes!" ) ? void (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy).getFixedSize() == DL.getTypeSizeInBits(LHSTy).getFixedSize() && \"Cannot have vector types of different sizes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1973, __extension__ __PRETTY_FUNCTION__)) | |||
1973 | "Cannot have vector types of different sizes!")(static_cast <bool> (DL.getTypeSizeInBits(RHSTy).getFixedSize () == DL.getTypeSizeInBits(LHSTy).getFixedSize() && "Cannot have vector types of different sizes!" ) ? void (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy).getFixedSize() == DL.getTypeSizeInBits(LHSTy).getFixedSize() && \"Cannot have vector types of different sizes!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1973, __extension__ __PRETTY_FUNCTION__)); | |||
1974 | assert(RHSTy->getElementType()->isIntegerTy() &&(static_cast <bool> (RHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1975, __extension__ __PRETTY_FUNCTION__)) | |||
1975 | "All non-integer types eliminated!")(static_cast <bool> (RHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1975, __extension__ __PRETTY_FUNCTION__)); | |||
1976 | assert(LHSTy->getElementType()->isIntegerTy() &&(static_cast <bool> (LHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1977, __extension__ __PRETTY_FUNCTION__)) | |||
1977 | "All non-integer types eliminated!")(static_cast <bool> (LHSTy->getElementType()->isIntegerTy () && "All non-integer types eliminated!") ? void (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1977, __extension__ __PRETTY_FUNCTION__)); | |||
1978 | return cast<FixedVectorType>(RHSTy)->getNumElements() < | |||
1979 | cast<FixedVectorType>(LHSTy)->getNumElements(); | |||
1980 | }; | |||
1981 | llvm::sort(CandidateTys, RankVectorTypes); | |||
1982 | CandidateTys.erase( | |||
1983 | std::unique(CandidateTys.begin(), CandidateTys.end(), RankVectorTypes), | |||
1984 | CandidateTys.end()); | |||
1985 | } else { | |||
1986 | // The only way to have the same element type in every vector type is to | |||
1987 | // have the same vector type. Check that and remove all but one. | |||
1988 | #ifndef NDEBUG | |||
1989 | for (VectorType *VTy : CandidateTys) { | |||
1990 | assert(VTy->getElementType() == CommonEltTy &&(static_cast <bool> (VTy->getElementType() == CommonEltTy && "Unaccounted for element type!") ? void (0) : __assert_fail ("VTy->getElementType() == CommonEltTy && \"Unaccounted for element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1991, __extension__ __PRETTY_FUNCTION__)) | |||
1991 | "Unaccounted for element type!")(static_cast <bool> (VTy->getElementType() == CommonEltTy && "Unaccounted for element type!") ? void (0) : __assert_fail ("VTy->getElementType() == CommonEltTy && \"Unaccounted for element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1991, __extension__ __PRETTY_FUNCTION__)); | |||
1992 | assert(VTy == CandidateTys[0] &&(static_cast <bool> (VTy == CandidateTys[0] && "Different vector types with the same element type!" ) ? void (0) : __assert_fail ("VTy == CandidateTys[0] && \"Different vector types with the same element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1993, __extension__ __PRETTY_FUNCTION__)) | |||
1993 | "Different vector types with the same element type!")(static_cast <bool> (VTy == CandidateTys[0] && "Different vector types with the same element type!" ) ? void (0) : __assert_fail ("VTy == CandidateTys[0] && \"Different vector types with the same element type!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 1993, __extension__ __PRETTY_FUNCTION__)); | |||
1994 | } | |||
1995 | #endif | |||
1996 | CandidateTys.resize(1); | |||
1997 | } | |||
1998 | ||||
1999 | // Try each vector type, and return the one which works. | |||
2000 | auto CheckVectorTypeForPromotion = [&](VectorType *VTy) { | |||
2001 | uint64_t ElementSize = | |||
2002 | DL.getTypeSizeInBits(VTy->getElementType()).getFixedSize(); | |||
2003 | ||||
2004 | // While the definition of LLVM vectors is bitpacked, we don't support sizes | |||
2005 | // that aren't byte sized. | |||
2006 | if (ElementSize % 8) | |||
2007 | return false; | |||
2008 | assert((DL.getTypeSizeInBits(VTy).getFixedSize() % 8) == 0 &&(static_cast <bool> ((DL.getTypeSizeInBits(VTy).getFixedSize () % 8) == 0 && "vector size not a multiple of element size?" ) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(VTy).getFixedSize() % 8) == 0 && \"vector size not a multiple of element size?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2009, __extension__ __PRETTY_FUNCTION__)) | |||
2009 | "vector size not a multiple of element size?")(static_cast <bool> ((DL.getTypeSizeInBits(VTy).getFixedSize () % 8) == 0 && "vector size not a multiple of element size?" ) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(VTy).getFixedSize() % 8) == 0 && \"vector size not a multiple of element size?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2009, __extension__ __PRETTY_FUNCTION__)); | |||
2010 | ElementSize /= 8; | |||
2011 | ||||
2012 | for (const Slice &S : P) | |||
2013 | if (!isVectorPromotionViableForSlice(P, S, VTy, ElementSize, DL)) | |||
2014 | return false; | |||
2015 | ||||
2016 | for (const Slice *S : P.splitSliceTails()) | |||
2017 | if (!isVectorPromotionViableForSlice(P, *S, VTy, ElementSize, DL)) | |||
2018 | return false; | |||
2019 | ||||
2020 | return true; | |||
2021 | }; | |||
2022 | for (VectorType *VTy : CandidateTys) | |||
2023 | if (CheckVectorTypeForPromotion(VTy)) | |||
2024 | return VTy; | |||
2025 | ||||
2026 | return nullptr; | |||
2027 | } | |||
2028 | ||||
2029 | /// Test whether a slice of an alloca is valid for integer widening. | |||
2030 | /// | |||
2031 | /// This implements the necessary checking for the \c isIntegerWideningViable | |||
2032 | /// test below on a single slice of the alloca. | |||
2033 | static bool isIntegerWideningViableForSlice(const Slice &S, | |||
2034 | uint64_t AllocBeginOffset, | |||
2035 | Type *AllocaTy, | |||
2036 | const DataLayout &DL, | |||
2037 | bool &WholeAllocaOp) { | |||
2038 | uint64_t Size = DL.getTypeStoreSize(AllocaTy).getFixedSize(); | |||
2039 | ||||
2040 | uint64_t RelBegin = S.beginOffset() - AllocBeginOffset; | |||
2041 | uint64_t RelEnd = S.endOffset() - AllocBeginOffset; | |||
2042 | ||||
2043 | // We can't reasonably handle cases where the load or store extends past | |||
2044 | // the end of the alloca's type and into its padding. | |||
2045 | if (RelEnd > Size) | |||
2046 | return false; | |||
2047 | ||||
2048 | Use *U = S.getUse(); | |||
2049 | ||||
2050 | if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | |||
2051 | if (LI->isVolatile()) | |||
2052 | return false; | |||
2053 | // We can't handle loads that extend past the allocated memory. | |||
2054 | if (DL.getTypeStoreSize(LI->getType()).getFixedSize() > Size) | |||
2055 | return false; | |||
2056 | // So far, AllocaSliceRewriter does not support widening split slice tails | |||
2057 | // in rewriteIntegerLoad. | |||
2058 | if (S.beginOffset() < AllocBeginOffset) | |||
2059 | return false; | |||
2060 | // Note that we don't count vector loads or stores as whole-alloca | |||
2061 | // operations which enable integer widening because we would prefer to use | |||
2062 | // vector widening instead. | |||
2063 | if (!isa<VectorType>(LI->getType()) && RelBegin == 0 && RelEnd == Size) | |||
2064 | WholeAllocaOp = true; | |||
2065 | if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) { | |||
2066 | if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedSize()) | |||
2067 | return false; | |||
2068 | } else if (RelBegin != 0 || RelEnd != Size || | |||
2069 | !canConvertValue(DL, AllocaTy, LI->getType())) { | |||
2070 | // Non-integer loads need to be convertible from the alloca type so that | |||
2071 | // they are promotable. | |||
2072 | return false; | |||
2073 | } | |||
2074 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | |||
2075 | Type *ValueTy = SI->getValueOperand()->getType(); | |||
2076 | if (SI->isVolatile()) | |||
2077 | return false; | |||
2078 | // We can't handle stores that extend past the allocated memory. | |||
2079 | if (DL.getTypeStoreSize(ValueTy).getFixedSize() > Size) | |||
2080 | return false; | |||
2081 | // So far, AllocaSliceRewriter does not support widening split slice tails | |||
2082 | // in rewriteIntegerStore. | |||
2083 | if (S.beginOffset() < AllocBeginOffset) | |||
2084 | return false; | |||
2085 | // Note that we don't count vector loads or stores as whole-alloca | |||
2086 | // operations which enable integer widening because we would prefer to use | |||
2087 | // vector widening instead. | |||
2088 | if (!isa<VectorType>(ValueTy) && RelBegin == 0 && RelEnd == Size) | |||
2089 | WholeAllocaOp = true; | |||
2090 | if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) { | |||
2091 | if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedSize()) | |||
2092 | return false; | |||
2093 | } else if (RelBegin != 0 || RelEnd != Size || | |||
2094 | !canConvertValue(DL, ValueTy, AllocaTy)) { | |||
2095 | // Non-integer stores need to be convertible to the alloca type so that | |||
2096 | // they are promotable. | |||
2097 | return false; | |||
2098 | } | |||
2099 | } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { | |||
2100 | if (MI->isVolatile() || !isa<Constant>(MI->getLength())) | |||
2101 | return false; | |||
2102 | if (!S.isSplittable()) | |||
2103 | return false; // Skip any unsplittable intrinsics. | |||
2104 | } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) { | |||
2105 | if (!II->isLifetimeStartOrEnd() && !II->isDroppable()) | |||
2106 | return false; | |||
2107 | } else { | |||
2108 | return false; | |||
2109 | } | |||
2110 | ||||
2111 | return true; | |||
2112 | } | |||
2113 | ||||
2114 | /// Test whether the given alloca partition's integer operations can be | |||
2115 | /// widened to promotable ones. | |||
2116 | /// | |||
2117 | /// This is a quick test to check whether we can rewrite the integer loads and | |||
2118 | /// stores to a particular alloca into wider loads and stores and be able to | |||
2119 | /// promote the resulting alloca. | |||
2120 | static bool isIntegerWideningViable(Partition &P, Type *AllocaTy, | |||
2121 | const DataLayout &DL) { | |||
2122 | uint64_t SizeInBits = DL.getTypeSizeInBits(AllocaTy).getFixedSize(); | |||
2123 | // Don't create integer types larger than the maximum bitwidth. | |||
2124 | if (SizeInBits > IntegerType::MAX_INT_BITS) | |||
2125 | return false; | |||
2126 | ||||
2127 | // Don't try to handle allocas with bit-padding. | |||
2128 | if (SizeInBits != DL.getTypeStoreSizeInBits(AllocaTy).getFixedSize()) | |||
2129 | return false; | |||
2130 | ||||
2131 | // We need to ensure that an integer type with the appropriate bitwidth can | |||
2132 | // be converted to the alloca type, whatever that is. We don't want to force | |||
2133 | // the alloca itself to have an integer type if there is a more suitable one. | |||
2134 | Type *IntTy = Type::getIntNTy(AllocaTy->getContext(), SizeInBits); | |||
2135 | if (!canConvertValue(DL, AllocaTy, IntTy) || | |||
2136 | !canConvertValue(DL, IntTy, AllocaTy)) | |||
2137 | return false; | |||
2138 | ||||
2139 | // While examining uses, we ensure that the alloca has a covering load or | |||
2140 | // store. We don't want to widen the integer operations only to fail to | |||
2141 | // promote due to some other unsplittable entry (which we may make splittable | |||
2142 | // later). However, if there are only splittable uses, go ahead and assume | |||
2143 | // that we cover the alloca. | |||
2144 | // FIXME: We shouldn't consider split slices that happen to start in the | |||
2145 | // partition here... | |||
2146 | bool WholeAllocaOp = P.empty() && DL.isLegalInteger(SizeInBits); | |||
2147 | ||||
2148 | for (const Slice &S : P) | |||
2149 | if (!isIntegerWideningViableForSlice(S, P.beginOffset(), AllocaTy, DL, | |||
2150 | WholeAllocaOp)) | |||
2151 | return false; | |||
2152 | ||||
2153 | for (const Slice *S : P.splitSliceTails()) | |||
2154 | if (!isIntegerWideningViableForSlice(*S, P.beginOffset(), AllocaTy, DL, | |||
2155 | WholeAllocaOp)) | |||
2156 | return false; | |||
2157 | ||||
2158 | return WholeAllocaOp; | |||
2159 | } | |||
2160 | ||||
2161 | static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V, | |||
2162 | IntegerType *Ty, uint64_t Offset, | |||
2163 | const Twine &Name) { | |||
2164 | LLVM_DEBUG(dbgs() << " start: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " start: " << *V << "\n"; } } while (false); | |||
2165 | IntegerType *IntTy = cast<IntegerType>(V->getType()); | |||
2166 | assert(DL.getTypeStoreSize(Ty).getFixedSize() + Offset <=(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedSize () + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && "Element extends past full value") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedSize() + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2168, __extension__ __PRETTY_FUNCTION__)) | |||
2167 | DL.getTypeStoreSize(IntTy).getFixedSize() &&(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedSize () + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && "Element extends past full value") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedSize() + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2168, __extension__ __PRETTY_FUNCTION__)) | |||
2168 | "Element extends past full value")(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedSize () + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && "Element extends past full value") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedSize() + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2168, __extension__ __PRETTY_FUNCTION__)); | |||
2169 | uint64_t ShAmt = 8 * Offset; | |||
2170 | if (DL.isBigEndian()) | |||
2171 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedSize() - | |||
2172 | DL.getTypeStoreSize(Ty).getFixedSize() - Offset); | |||
2173 | if (ShAmt) { | |||
2174 | V = IRB.CreateLShr(V, ShAmt, Name + ".shift"); | |||
2175 | LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shifted: " << *V << "\n"; } } while (false); | |||
2176 | } | |||
2177 | assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&(static_cast <bool> (Ty->getBitWidth() <= IntTy-> getBitWidth() && "Cannot extract to a larger integer!" ) ? void (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2178, __extension__ __PRETTY_FUNCTION__)) | |||
2178 | "Cannot extract to a larger integer!")(static_cast <bool> (Ty->getBitWidth() <= IntTy-> getBitWidth() && "Cannot extract to a larger integer!" ) ? void (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2178, __extension__ __PRETTY_FUNCTION__)); | |||
2179 | if (Ty != IntTy) { | |||
2180 | V = IRB.CreateTrunc(V, Ty, Name + ".trunc"); | |||
2181 | LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " trunced: " << *V << "\n"; } } while (false); | |||
2182 | } | |||
2183 | return V; | |||
2184 | } | |||
2185 | ||||
2186 | static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old, | |||
2187 | Value *V, uint64_t Offset, const Twine &Name) { | |||
2188 | IntegerType *IntTy = cast<IntegerType>(Old->getType()); | |||
2189 | IntegerType *Ty = cast<IntegerType>(V->getType()); | |||
2190 | assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&(static_cast <bool> (Ty->getBitWidth() <= IntTy-> getBitWidth() && "Cannot insert a larger integer!") ? void (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot insert a larger integer!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2191, __extension__ __PRETTY_FUNCTION__)) | |||
2191 | "Cannot insert a larger integer!")(static_cast <bool> (Ty->getBitWidth() <= IntTy-> getBitWidth() && "Cannot insert a larger integer!") ? void (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot insert a larger integer!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2191, __extension__ __PRETTY_FUNCTION__)); | |||
2192 | LLVM_DEBUG(dbgs() << " start: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " start: " << *V << "\n"; } } while (false); | |||
2193 | if (Ty != IntTy) { | |||
2194 | V = IRB.CreateZExt(V, IntTy, Name + ".ext"); | |||
2195 | LLVM_DEBUG(dbgs() << " extended: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " extended: " << *V << "\n"; } } while (false); | |||
2196 | } | |||
2197 | assert(DL.getTypeStoreSize(Ty).getFixedSize() + Offset <=(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedSize () + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && "Element store outside of alloca store") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedSize() + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && \"Element store outside of alloca store\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2199, __extension__ __PRETTY_FUNCTION__)) | |||
2198 | DL.getTypeStoreSize(IntTy).getFixedSize() &&(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedSize () + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && "Element store outside of alloca store") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedSize() + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && \"Element store outside of alloca store\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2199, __extension__ __PRETTY_FUNCTION__)) | |||
2199 | "Element store outside of alloca store")(static_cast <bool> (DL.getTypeStoreSize(Ty).getFixedSize () + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && "Element store outside of alloca store") ? void (0) : __assert_fail ("DL.getTypeStoreSize(Ty).getFixedSize() + Offset <= DL.getTypeStoreSize(IntTy).getFixedSize() && \"Element store outside of alloca store\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2199, __extension__ __PRETTY_FUNCTION__)); | |||
2200 | uint64_t ShAmt = 8 * Offset; | |||
2201 | if (DL.isBigEndian()) | |||
2202 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedSize() - | |||
2203 | DL.getTypeStoreSize(Ty).getFixedSize() - Offset); | |||
2204 | if (ShAmt) { | |||
2205 | V = IRB.CreateShl(V, ShAmt, Name + ".shift"); | |||
2206 | LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shifted: " << *V << "\n"; } } while (false); | |||
2207 | } | |||
2208 | ||||
2209 | if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) { | |||
2210 | APInt Mask = ~Ty->getMask().zext(IntTy->getBitWidth()).shl(ShAmt); | |||
2211 | Old = IRB.CreateAnd(Old, Mask, Name + ".mask"); | |||
2212 | LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " masked: " << *Old << "\n"; } } while (false); | |||
2213 | V = IRB.CreateOr(Old, V, Name + ".insert"); | |||
2214 | LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " inserted: " << *V << "\n"; } } while (false); | |||
2215 | } | |||
2216 | return V; | |||
2217 | } | |||
2218 | ||||
2219 | static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex, | |||
2220 | unsigned EndIndex, const Twine &Name) { | |||
2221 | auto *VecTy = cast<FixedVectorType>(V->getType()); | |||
2222 | unsigned NumElements = EndIndex - BeginIndex; | |||
2223 | assert(NumElements <= VecTy->getNumElements() && "Too many elements!")(static_cast <bool> (NumElements <= VecTy->getNumElements () && "Too many elements!") ? void (0) : __assert_fail ("NumElements <= VecTy->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2223, __extension__ __PRETTY_FUNCTION__)); | |||
2224 | ||||
2225 | if (NumElements == VecTy->getNumElements()) | |||
2226 | return V; | |||
2227 | ||||
2228 | if (NumElements == 1) { | |||
2229 | V = IRB.CreateExtractElement(V, IRB.getInt32(BeginIndex), | |||
2230 | Name + ".extract"); | |||
2231 | LLVM_DEBUG(dbgs() << " extract: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " extract: " << *V << "\n"; } } while (false); | |||
2232 | return V; | |||
2233 | } | |||
2234 | ||||
2235 | SmallVector<int, 8> Mask; | |||
2236 | Mask.reserve(NumElements); | |||
2237 | for (unsigned i = BeginIndex; i != EndIndex; ++i) | |||
2238 | Mask.push_back(i); | |||
2239 | V = IRB.CreateShuffleVector(V, Mask, Name + ".extract"); | |||
2240 | LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shuffle: " << *V << "\n"; } } while (false); | |||
2241 | return V; | |||
2242 | } | |||
2243 | ||||
2244 | static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V, | |||
2245 | unsigned BeginIndex, const Twine &Name) { | |||
2246 | VectorType *VecTy = cast<VectorType>(Old->getType()); | |||
2247 | assert(VecTy && "Can only insert a vector into a vector")(static_cast <bool> (VecTy && "Can only insert a vector into a vector" ) ? void (0) : __assert_fail ("VecTy && \"Can only insert a vector into a vector\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2247, __extension__ __PRETTY_FUNCTION__)); | |||
2248 | ||||
2249 | VectorType *Ty = dyn_cast<VectorType>(V->getType()); | |||
2250 | if (!Ty) { | |||
2251 | // Single element to insert. | |||
2252 | V = IRB.CreateInsertElement(Old, V, IRB.getInt32(BeginIndex), | |||
2253 | Name + ".insert"); | |||
2254 | LLVM_DEBUG(dbgs() << " insert: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " insert: " << *V << "\n"; } } while (false); | |||
2255 | return V; | |||
2256 | } | |||
2257 | ||||
2258 | assert(cast<FixedVectorType>(Ty)->getNumElements() <=(static_cast <bool> (cast<FixedVectorType>(Ty)-> getNumElements() <= cast<FixedVectorType>(VecTy)-> getNumElements() && "Too many elements!") ? void (0) : __assert_fail ("cast<FixedVectorType>(Ty)->getNumElements() <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2260, __extension__ __PRETTY_FUNCTION__)) | |||
2259 | cast<FixedVectorType>(VecTy)->getNumElements() &&(static_cast <bool> (cast<FixedVectorType>(Ty)-> getNumElements() <= cast<FixedVectorType>(VecTy)-> getNumElements() && "Too many elements!") ? void (0) : __assert_fail ("cast<FixedVectorType>(Ty)->getNumElements() <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2260, __extension__ __PRETTY_FUNCTION__)) | |||
2260 | "Too many elements!")(static_cast <bool> (cast<FixedVectorType>(Ty)-> getNumElements() <= cast<FixedVectorType>(VecTy)-> getNumElements() && "Too many elements!") ? void (0) : __assert_fail ("cast<FixedVectorType>(Ty)->getNumElements() <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2260, __extension__ __PRETTY_FUNCTION__)); | |||
2261 | if (cast<FixedVectorType>(Ty)->getNumElements() == | |||
2262 | cast<FixedVectorType>(VecTy)->getNumElements()) { | |||
2263 | assert(V->getType() == VecTy && "Vector type mismatch")(static_cast <bool> (V->getType() == VecTy && "Vector type mismatch") ? void (0) : __assert_fail ("V->getType() == VecTy && \"Vector type mismatch\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2263, __extension__ __PRETTY_FUNCTION__)); | |||
2264 | return V; | |||
2265 | } | |||
2266 | unsigned EndIndex = BeginIndex + cast<FixedVectorType>(Ty)->getNumElements(); | |||
2267 | ||||
2268 | // When inserting a smaller vector into the larger to store, we first | |||
2269 | // use a shuffle vector to widen it with undef elements, and then | |||
2270 | // a second shuffle vector to select between the loaded vector and the | |||
2271 | // incoming vector. | |||
2272 | SmallVector<int, 8> Mask; | |||
2273 | Mask.reserve(cast<FixedVectorType>(VecTy)->getNumElements()); | |||
2274 | for (unsigned i = 0; i != cast<FixedVectorType>(VecTy)->getNumElements(); ++i) | |||
2275 | if (i >= BeginIndex && i < EndIndex) | |||
2276 | Mask.push_back(i - BeginIndex); | |||
2277 | else | |||
2278 | Mask.push_back(-1); | |||
2279 | V = IRB.CreateShuffleVector(V, Mask, Name + ".expand"); | |||
2280 | LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shuffle: " << *V << "\n"; } } while (false); | |||
2281 | ||||
2282 | SmallVector<Constant *, 8> Mask2; | |||
2283 | Mask2.reserve(cast<FixedVectorType>(VecTy)->getNumElements()); | |||
2284 | for (unsigned i = 0; i != cast<FixedVectorType>(VecTy)->getNumElements(); ++i) | |||
2285 | Mask2.push_back(IRB.getInt1(i >= BeginIndex && i < EndIndex)); | |||
2286 | ||||
2287 | V = IRB.CreateSelect(ConstantVector::get(Mask2), V, Old, Name + "blend"); | |||
2288 | ||||
2289 | LLVM_DEBUG(dbgs() << " blend: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " blend: " << *V << "\n"; } } while (false); | |||
2290 | return V; | |||
2291 | } | |||
2292 | ||||
2293 | /// Visitor to rewrite instructions using p particular slice of an alloca | |||
2294 | /// to use a new alloca. | |||
2295 | /// | |||
2296 | /// Also implements the rewriting to vector-based accesses when the partition | |||
2297 | /// passes the isVectorPromotionViable predicate. Most of the rewriting logic | |||
2298 | /// lives here. | |||
2299 | class llvm::sroa::AllocaSliceRewriter | |||
2300 | : public InstVisitor<AllocaSliceRewriter, bool> { | |||
2301 | // Befriend the base class so it can delegate to private visit methods. | |||
2302 | friend class InstVisitor<AllocaSliceRewriter, bool>; | |||
2303 | ||||
2304 | using Base = InstVisitor<AllocaSliceRewriter, bool>; | |||
2305 | ||||
2306 | const DataLayout &DL; | |||
2307 | AllocaSlices &AS; | |||
2308 | SROA &Pass; | |||
2309 | AllocaInst &OldAI, &NewAI; | |||
2310 | const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset; | |||
2311 | Type *NewAllocaTy; | |||
2312 | ||||
2313 | // This is a convenience and flag variable that will be null unless the new | |||
2314 | // alloca's integer operations should be widened to this integer type due to | |||
2315 | // passing isIntegerWideningViable above. If it is non-null, the desired | |||
2316 | // integer type will be stored here for easy access during rewriting. | |||
2317 | IntegerType *IntTy; | |||
2318 | ||||
2319 | // If we are rewriting an alloca partition which can be written as pure | |||
2320 | // vector operations, we stash extra information here. When VecTy is | |||
2321 | // non-null, we have some strict guarantees about the rewritten alloca: | |||
2322 | // - The new alloca is exactly the size of the vector type here. | |||
2323 | // - The accesses all either map to the entire vector or to a single | |||
2324 | // element. | |||
2325 | // - The set of accessing instructions is only one of those handled above | |||
2326 | // in isVectorPromotionViable. Generally these are the same access kinds | |||
2327 | // which are promotable via mem2reg. | |||
2328 | VectorType *VecTy; | |||
2329 | Type *ElementTy; | |||
2330 | uint64_t ElementSize; | |||
2331 | ||||
2332 | // The original offset of the slice currently being rewritten relative to | |||
2333 | // the original alloca. | |||
2334 | uint64_t BeginOffset = 0; | |||
2335 | uint64_t EndOffset = 0; | |||
2336 | ||||
2337 | // The new offsets of the slice currently being rewritten relative to the | |||
2338 | // original alloca. | |||
2339 | uint64_t NewBeginOffset = 0, NewEndOffset = 0; | |||
2340 | ||||
2341 | uint64_t SliceSize = 0; | |||
2342 | bool IsSplittable = false; | |||
2343 | bool IsSplit = false; | |||
2344 | Use *OldUse = nullptr; | |||
2345 | Instruction *OldPtr = nullptr; | |||
2346 | ||||
2347 | // Track post-rewrite users which are PHI nodes and Selects. | |||
2348 | SmallSetVector<PHINode *, 8> &PHIUsers; | |||
2349 | SmallSetVector<SelectInst *, 8> &SelectUsers; | |||
2350 | ||||
2351 | // Utility IR builder, whose name prefix is setup for each visited use, and | |||
2352 | // the insertion point is set to point to the user. | |||
2353 | IRBuilderTy IRB; | |||
2354 | ||||
2355 | public: | |||
2356 | AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass, | |||
2357 | AllocaInst &OldAI, AllocaInst &NewAI, | |||
2358 | uint64_t NewAllocaBeginOffset, | |||
2359 | uint64_t NewAllocaEndOffset, bool IsIntegerPromotable, | |||
2360 | VectorType *PromotableVecTy, | |||
2361 | SmallSetVector<PHINode *, 8> &PHIUsers, | |||
2362 | SmallSetVector<SelectInst *, 8> &SelectUsers) | |||
2363 | : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI), | |||
2364 | NewAllocaBeginOffset(NewAllocaBeginOffset), | |||
2365 | NewAllocaEndOffset(NewAllocaEndOffset), | |||
2366 | NewAllocaTy(NewAI.getAllocatedType()), | |||
2367 | IntTy( | |||
2368 | IsIntegerPromotable | |||
2369 | ? Type::getIntNTy(NewAI.getContext(), | |||
2370 | DL.getTypeSizeInBits(NewAI.getAllocatedType()) | |||
2371 | .getFixedSize()) | |||
2372 | : nullptr), | |||
2373 | VecTy(PromotableVecTy), | |||
2374 | ElementTy(VecTy ? VecTy->getElementType() : nullptr), | |||
2375 | ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy).getFixedSize() / 8 | |||
2376 | : 0), | |||
2377 | PHIUsers(PHIUsers), SelectUsers(SelectUsers), | |||
2378 | IRB(NewAI.getContext(), ConstantFolder()) { | |||
2379 | if (VecTy) { | |||
2380 | assert((DL.getTypeSizeInBits(ElementTy).getFixedSize() % 8) == 0 &&(static_cast <bool> ((DL.getTypeSizeInBits(ElementTy).getFixedSize () % 8) == 0 && "Only multiple-of-8 sized vector elements are viable" ) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(ElementTy).getFixedSize() % 8) == 0 && \"Only multiple-of-8 sized vector elements are viable\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2381, __extension__ __PRETTY_FUNCTION__)) | |||
2381 | "Only multiple-of-8 sized vector elements are viable")(static_cast <bool> ((DL.getTypeSizeInBits(ElementTy).getFixedSize () % 8) == 0 && "Only multiple-of-8 sized vector elements are viable" ) ? void (0) : __assert_fail ("(DL.getTypeSizeInBits(ElementTy).getFixedSize() % 8) == 0 && \"Only multiple-of-8 sized vector elements are viable\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2381, __extension__ __PRETTY_FUNCTION__)); | |||
2382 | ++NumVectorized; | |||
2383 | } | |||
2384 | assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy))(static_cast <bool> ((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy)) ? void (0) : __assert_fail ("(!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2384, __extension__ __PRETTY_FUNCTION__)); | |||
2385 | } | |||
2386 | ||||
2387 | bool visit(AllocaSlices::const_iterator I) { | |||
2388 | bool CanSROA = true; | |||
2389 | BeginOffset = I->beginOffset(); | |||
2390 | EndOffset = I->endOffset(); | |||
2391 | IsSplittable = I->isSplittable(); | |||
2392 | IsSplit = | |||
2393 | BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset; | |||
2394 | LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " rewriting " << (IsSplit ? "split " : ""); } } while (false); | |||
2395 | LLVM_DEBUG(AS.printSlice(dbgs(), I, ""))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.printSlice(dbgs(), I, ""); } } while (false); | |||
2396 | LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n"; } } while (false); | |||
2397 | ||||
2398 | // Compute the intersecting offset range. | |||
2399 | assert(BeginOffset < NewAllocaEndOffset)(static_cast <bool> (BeginOffset < NewAllocaEndOffset ) ? void (0) : __assert_fail ("BeginOffset < NewAllocaEndOffset" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2399, __extension__ __PRETTY_FUNCTION__)); | |||
2400 | assert(EndOffset > NewAllocaBeginOffset)(static_cast <bool> (EndOffset > NewAllocaBeginOffset ) ? void (0) : __assert_fail ("EndOffset > NewAllocaBeginOffset" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2400, __extension__ __PRETTY_FUNCTION__)); | |||
2401 | NewBeginOffset = std::max(BeginOffset, NewAllocaBeginOffset); | |||
2402 | NewEndOffset = std::min(EndOffset, NewAllocaEndOffset); | |||
2403 | ||||
2404 | SliceSize = NewEndOffset - NewBeginOffset; | |||
2405 | ||||
2406 | OldUse = I->getUse(); | |||
2407 | OldPtr = cast<Instruction>(OldUse->get()); | |||
2408 | ||||
2409 | Instruction *OldUserI = cast<Instruction>(OldUse->getUser()); | |||
2410 | IRB.SetInsertPoint(OldUserI); | |||
2411 | IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc()); | |||
2412 | IRB.getInserter().SetNamePrefix( | |||
2413 | Twine(NewAI.getName()) + "." + Twine(BeginOffset) + "."); | |||
2414 | ||||
2415 | CanSROA &= visit(cast<Instruction>(OldUse->getUser())); | |||
2416 | if (VecTy || IntTy) | |||
2417 | assert(CanSROA)(static_cast <bool> (CanSROA) ? void (0) : __assert_fail ("CanSROA", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2417, __extension__ __PRETTY_FUNCTION__)); | |||
2418 | return CanSROA; | |||
2419 | } | |||
2420 | ||||
2421 | private: | |||
2422 | // Make sure the other visit overloads are visible. | |||
2423 | using Base::visit; | |||
2424 | ||||
2425 | // Every instruction which can end up as a user must have a rewrite rule. | |||
2426 | bool visitInstruction(Instruction &I) { | |||
2427 | LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " !!!! Cannot rewrite: " << I << "\n"; } } while (false); | |||
2428 | llvm_unreachable("No rewrite rule for this instruction!")::llvm::llvm_unreachable_internal("No rewrite rule for this instruction!" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2428); | |||
2429 | } | |||
2430 | ||||
2431 | Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) { | |||
2432 | // Note that the offset computation can use BeginOffset or NewBeginOffset | |||
2433 | // interchangeably for unsplit slices. | |||
2434 | assert(IsSplit || BeginOffset == NewBeginOffset)(static_cast <bool> (IsSplit || BeginOffset == NewBeginOffset ) ? void (0) : __assert_fail ("IsSplit || BeginOffset == NewBeginOffset" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2434, __extension__ __PRETTY_FUNCTION__)); | |||
2435 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
2436 | ||||
2437 | #ifndef NDEBUG | |||
2438 | StringRef OldName = OldPtr->getName(); | |||
2439 | // Skip through the last '.sroa.' component of the name. | |||
2440 | size_t LastSROAPrefix = OldName.rfind(".sroa."); | |||
2441 | if (LastSROAPrefix != StringRef::npos) { | |||
2442 | OldName = OldName.substr(LastSROAPrefix + strlen(".sroa.")); | |||
2443 | // Look for an SROA slice index. | |||
2444 | size_t IndexEnd = OldName.find_first_not_of("0123456789"); | |||
2445 | if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') { | |||
2446 | // Strip the index and look for the offset. | |||
2447 | OldName = OldName.substr(IndexEnd + 1); | |||
2448 | size_t OffsetEnd = OldName.find_first_not_of("0123456789"); | |||
2449 | if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.') | |||
2450 | // Strip the offset. | |||
2451 | OldName = OldName.substr(OffsetEnd + 1); | |||
2452 | } | |||
2453 | } | |||
2454 | // Strip any SROA suffixes as well. | |||
2455 | OldName = OldName.substr(0, OldName.find(".sroa_")); | |||
2456 | #endif | |||
2457 | ||||
2458 | return getAdjustedPtr(IRB, DL, &NewAI, | |||
2459 | APInt(DL.getIndexTypeSizeInBits(PointerTy), Offset), | |||
2460 | PointerTy, | |||
2461 | #ifndef NDEBUG | |||
2462 | Twine(OldName) + "." | |||
2463 | #else | |||
2464 | Twine() | |||
2465 | #endif | |||
2466 | ); | |||
2467 | } | |||
2468 | ||||
2469 | /// Compute suitable alignment to access this slice of the *new* | |||
2470 | /// alloca. | |||
2471 | /// | |||
2472 | /// You can optionally pass a type to this routine and if that type's ABI | |||
2473 | /// alignment is itself suitable, this will return zero. | |||
2474 | Align getSliceAlign() { | |||
2475 | return commonAlignment(NewAI.getAlign(), | |||
2476 | NewBeginOffset - NewAllocaBeginOffset); | |||
2477 | } | |||
2478 | ||||
2479 | unsigned getIndex(uint64_t Offset) { | |||
2480 | assert(VecTy && "Can only call getIndex when rewriting a vector")(static_cast <bool> (VecTy && "Can only call getIndex when rewriting a vector" ) ? void (0) : __assert_fail ("VecTy && \"Can only call getIndex when rewriting a vector\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2480, __extension__ __PRETTY_FUNCTION__)); | |||
2481 | uint64_t RelOffset = Offset - NewAllocaBeginOffset; | |||
2482 | assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds")(static_cast <bool> (RelOffset / ElementSize < (4294967295U ) && "Index out of bounds") ? void (0) : __assert_fail ("RelOffset / ElementSize < UINT32_MAX && \"Index out of bounds\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2482, __extension__ __PRETTY_FUNCTION__)); | |||
2483 | uint32_t Index = RelOffset / ElementSize; | |||
2484 | assert(Index * ElementSize == RelOffset)(static_cast <bool> (Index * ElementSize == RelOffset) ? void (0) : __assert_fail ("Index * ElementSize == RelOffset" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2484, __extension__ __PRETTY_FUNCTION__)); | |||
2485 | return Index; | |||
2486 | } | |||
2487 | ||||
2488 | void deleteIfTriviallyDead(Value *V) { | |||
2489 | Instruction *I = cast<Instruction>(V); | |||
2490 | if (isInstructionTriviallyDead(I)) | |||
2491 | Pass.DeadInsts.push_back(I); | |||
2492 | } | |||
2493 | ||||
2494 | Value *rewriteVectorizedLoadInst(LoadInst &LI) { | |||
2495 | unsigned BeginIndex = getIndex(NewBeginOffset); | |||
2496 | unsigned EndIndex = getIndex(NewEndOffset); | |||
2497 | assert(EndIndex > BeginIndex && "Empty vector!")(static_cast <bool> (EndIndex > BeginIndex && "Empty vector!") ? void (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2497, __extension__ __PRETTY_FUNCTION__)); | |||
2498 | ||||
2499 | LoadInst *Load = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2500 | NewAI.getAlign(), "load"); | |||
2501 | ||||
2502 | Load->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2503 | LLVMContext::MD_access_group}); | |||
2504 | return extractVector(IRB, Load, BeginIndex, EndIndex, "vec"); | |||
2505 | } | |||
2506 | ||||
2507 | Value *rewriteIntegerLoad(LoadInst &LI) { | |||
2508 | assert(IntTy && "We cannot insert an integer to the alloca")(static_cast <bool> (IntTy && "We cannot insert an integer to the alloca" ) ? void (0) : __assert_fail ("IntTy && \"We cannot insert an integer to the alloca\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2508, __extension__ __PRETTY_FUNCTION__)); | |||
2509 | assert(!LI.isVolatile())(static_cast <bool> (!LI.isVolatile()) ? void (0) : __assert_fail ("!LI.isVolatile()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2509, __extension__ __PRETTY_FUNCTION__)); | |||
2510 | Value *V = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2511 | NewAI.getAlign(), "load"); | |||
2512 | V = convertValue(DL, IRB, V, IntTy); | |||
2513 | assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset")(static_cast <bool> (NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset") ? void (0) : __assert_fail ("NewBeginOffset >= NewAllocaBeginOffset && \"Out of bounds offset\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2513, __extension__ __PRETTY_FUNCTION__)); | |||
2514 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
2515 | if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) { | |||
2516 | IntegerType *ExtractTy = Type::getIntNTy(LI.getContext(), SliceSize * 8); | |||
2517 | V = extractInteger(DL, IRB, V, ExtractTy, Offset, "extract"); | |||
2518 | } | |||
2519 | // It is possible that the extracted type is not the load type. This | |||
2520 | // happens if there is a load past the end of the alloca, and as | |||
2521 | // a consequence the slice is narrower but still a candidate for integer | |||
2522 | // lowering. To handle this case, we just zero extend the extracted | |||
2523 | // integer. | |||
2524 | assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&(static_cast <bool> (cast<IntegerType>(LI.getType ())->getBitWidth() >= SliceSize * 8 && "Can only handle an extract for an overly wide load" ) ? void (0) : __assert_fail ("cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && \"Can only handle an extract for an overly wide load\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2525, __extension__ __PRETTY_FUNCTION__)) | |||
2525 | "Can only handle an extract for an overly wide load")(static_cast <bool> (cast<IntegerType>(LI.getType ())->getBitWidth() >= SliceSize * 8 && "Can only handle an extract for an overly wide load" ) ? void (0) : __assert_fail ("cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && \"Can only handle an extract for an overly wide load\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2525, __extension__ __PRETTY_FUNCTION__)); | |||
2526 | if (cast<IntegerType>(LI.getType())->getBitWidth() > SliceSize * 8) | |||
2527 | V = IRB.CreateZExt(V, LI.getType()); | |||
2528 | return V; | |||
2529 | } | |||
2530 | ||||
2531 | bool visitLoadInst(LoadInst &LI) { | |||
2532 | LLVM_DEBUG(dbgs() << " original: " << LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << LI << "\n"; } } while (false); | |||
2533 | Value *OldOp = LI.getOperand(0); | |||
2534 | assert(OldOp == OldPtr)(static_cast <bool> (OldOp == OldPtr) ? void (0) : __assert_fail ("OldOp == OldPtr", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2534, __extension__ __PRETTY_FUNCTION__)); | |||
2535 | ||||
2536 | AAMDNodes AATags; | |||
2537 | LI.getAAMetadata(AATags); | |||
2538 | ||||
2539 | unsigned AS = LI.getPointerAddressSpace(); | |||
2540 | ||||
2541 | Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8) | |||
2542 | : LI.getType(); | |||
2543 | const bool IsLoadPastEnd = | |||
2544 | DL.getTypeStoreSize(TargetTy).getFixedSize() > SliceSize; | |||
2545 | bool IsPtrAdjusted = false; | |||
2546 | Value *V; | |||
2547 | if (VecTy) { | |||
2548 | V = rewriteVectorizedLoadInst(LI); | |||
2549 | } else if (IntTy && LI.getType()->isIntegerTy()) { | |||
2550 | V = rewriteIntegerLoad(LI); | |||
2551 | } else if (NewBeginOffset == NewAllocaBeginOffset && | |||
2552 | NewEndOffset == NewAllocaEndOffset && | |||
2553 | (canConvertValue(DL, NewAllocaTy, TargetTy) || | |||
2554 | (IsLoadPastEnd && NewAllocaTy->isIntegerTy() && | |||
2555 | TargetTy->isIntegerTy()))) { | |||
2556 | LoadInst *NewLI = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2557 | NewAI.getAlign(), LI.isVolatile(), | |||
2558 | LI.getName()); | |||
2559 | if (AATags) | |||
2560 | NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2561 | if (LI.isVolatile()) | |||
2562 | NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); | |||
2563 | if (NewLI->isAtomic()) | |||
2564 | NewLI->setAlignment(LI.getAlign()); | |||
2565 | ||||
2566 | // Any !nonnull metadata or !range metadata on the old load is also valid | |||
2567 | // on the new load. This is even true in some cases even when the loads | |||
2568 | // are different types, for example by mapping !nonnull metadata to | |||
2569 | // !range metadata by modeling the null pointer constant converted to the | |||
2570 | // integer type. | |||
2571 | // FIXME: Add support for range metadata here. Currently the utilities | |||
2572 | // for this don't propagate range metadata in trivial cases from one | |||
2573 | // integer load to another, don't handle non-addrspace-0 null pointers | |||
2574 | // correctly, and don't have any support for mapping ranges as the | |||
2575 | // integer type becomes winder or narrower. | |||
2576 | if (MDNode *N = LI.getMetadata(LLVMContext::MD_nonnull)) | |||
2577 | copyNonnullMetadata(LI, N, *NewLI); | |||
2578 | ||||
2579 | // Try to preserve nonnull metadata | |||
2580 | V = NewLI; | |||
2581 | ||||
2582 | // If this is an integer load past the end of the slice (which means the | |||
2583 | // bytes outside the slice are undef or this load is dead) just forcibly | |||
2584 | // fix the integer size with correct handling of endianness. | |||
2585 | if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy)) | |||
2586 | if (auto *TITy = dyn_cast<IntegerType>(TargetTy)) | |||
2587 | if (AITy->getBitWidth() < TITy->getBitWidth()) { | |||
2588 | V = IRB.CreateZExt(V, TITy, "load.ext"); | |||
2589 | if (DL.isBigEndian()) | |||
2590 | V = IRB.CreateShl(V, TITy->getBitWidth() - AITy->getBitWidth(), | |||
2591 | "endian_shift"); | |||
2592 | } | |||
2593 | } else { | |||
2594 | Type *LTy = TargetTy->getPointerTo(AS); | |||
2595 | LoadInst *NewLI = | |||
2596 | IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy), | |||
2597 | getSliceAlign(), LI.isVolatile(), LI.getName()); | |||
2598 | if (AATags) | |||
2599 | NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2600 | if (LI.isVolatile()) | |||
2601 | NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); | |||
2602 | NewLI->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2603 | LLVMContext::MD_access_group}); | |||
2604 | ||||
2605 | V = NewLI; | |||
2606 | IsPtrAdjusted = true; | |||
2607 | } | |||
2608 | V = convertValue(DL, IRB, V, TargetTy); | |||
2609 | ||||
2610 | if (IsSplit) { | |||
2611 | assert(!LI.isVolatile())(static_cast <bool> (!LI.isVolatile()) ? void (0) : __assert_fail ("!LI.isVolatile()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2611, __extension__ __PRETTY_FUNCTION__)); | |||
2612 | assert(LI.getType()->isIntegerTy() &&(static_cast <bool> (LI.getType()->isIntegerTy() && "Only integer type loads and stores are split") ? void (0) : __assert_fail ("LI.getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2613, __extension__ __PRETTY_FUNCTION__)) | |||
2613 | "Only integer type loads and stores are split")(static_cast <bool> (LI.getType()->isIntegerTy() && "Only integer type loads and stores are split") ? void (0) : __assert_fail ("LI.getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2613, __extension__ __PRETTY_FUNCTION__)); | |||
2614 | assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedSize() &&(static_cast <bool> (SliceSize < DL.getTypeStoreSize (LI.getType()).getFixedSize() && "Split load isn't smaller than original load" ) ? void (0) : __assert_fail ("SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedSize() && \"Split load isn't smaller than original load\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2615, __extension__ __PRETTY_FUNCTION__)) | |||
2615 | "Split load isn't smaller than original load")(static_cast <bool> (SliceSize < DL.getTypeStoreSize (LI.getType()).getFixedSize() && "Split load isn't smaller than original load" ) ? void (0) : __assert_fail ("SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedSize() && \"Split load isn't smaller than original load\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2615, __extension__ __PRETTY_FUNCTION__)); | |||
2616 | assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&(static_cast <bool> (DL.typeSizeEqualsStoreSize(LI.getType ()) && "Non-byte-multiple bit width") ? void (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(LI.getType()) && \"Non-byte-multiple bit width\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2617, __extension__ __PRETTY_FUNCTION__)) | |||
2617 | "Non-byte-multiple bit width")(static_cast <bool> (DL.typeSizeEqualsStoreSize(LI.getType ()) && "Non-byte-multiple bit width") ? void (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(LI.getType()) && \"Non-byte-multiple bit width\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2617, __extension__ __PRETTY_FUNCTION__)); | |||
2618 | // Move the insertion point just past the load so that we can refer to it. | |||
2619 | IRB.SetInsertPoint(&*std::next(BasicBlock::iterator(&LI))); | |||
2620 | // Create a placeholder value with the same type as LI to use as the | |||
2621 | // basis for the new value. This allows us to replace the uses of LI with | |||
2622 | // the computed value, and then replace the placeholder with LI, leaving | |||
2623 | // LI only used for this computation. | |||
2624 | Value *Placeholder = new LoadInst( | |||
2625 | LI.getType(), UndefValue::get(LI.getType()->getPointerTo(AS)), "", | |||
2626 | false, Align(1)); | |||
2627 | V = insertInteger(DL, IRB, Placeholder, V, NewBeginOffset - BeginOffset, | |||
2628 | "insert"); | |||
2629 | LI.replaceAllUsesWith(V); | |||
2630 | Placeholder->replaceAllUsesWith(&LI); | |||
2631 | Placeholder->deleteValue(); | |||
2632 | } else { | |||
2633 | LI.replaceAllUsesWith(V); | |||
2634 | } | |||
2635 | ||||
2636 | Pass.DeadInsts.push_back(&LI); | |||
2637 | deleteIfTriviallyDead(OldOp); | |||
2638 | LLVM_DEBUG(dbgs() << " to: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *V << "\n"; } } while (false); | |||
2639 | return !LI.isVolatile() && !IsPtrAdjusted; | |||
2640 | } | |||
2641 | ||||
2642 | bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp, | |||
2643 | AAMDNodes AATags) { | |||
2644 | if (V->getType() != VecTy) { | |||
2645 | unsigned BeginIndex = getIndex(NewBeginOffset); | |||
2646 | unsigned EndIndex = getIndex(NewEndOffset); | |||
2647 | assert(EndIndex > BeginIndex && "Empty vector!")(static_cast <bool> (EndIndex > BeginIndex && "Empty vector!") ? void (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2647, __extension__ __PRETTY_FUNCTION__)); | |||
2648 | unsigned NumElements = EndIndex - BeginIndex; | |||
2649 | assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&(static_cast <bool> (NumElements <= cast<FixedVectorType >(VecTy)->getNumElements() && "Too many elements!" ) ? void (0) : __assert_fail ("NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2650, __extension__ __PRETTY_FUNCTION__)) | |||
2650 | "Too many elements!")(static_cast <bool> (NumElements <= cast<FixedVectorType >(VecTy)->getNumElements() && "Too many elements!" ) ? void (0) : __assert_fail ("NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2650, __extension__ __PRETTY_FUNCTION__)); | |||
2651 | Type *SliceTy = (NumElements == 1) | |||
2652 | ? ElementTy | |||
2653 | : FixedVectorType::get(ElementTy, NumElements); | |||
2654 | if (V->getType() != SliceTy) | |||
2655 | V = convertValue(DL, IRB, V, SliceTy); | |||
2656 | ||||
2657 | // Mix in the existing elements. | |||
2658 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2659 | NewAI.getAlign(), "load"); | |||
2660 | V = insertVector(IRB, Old, V, BeginIndex, "vec"); | |||
2661 | } | |||
2662 | StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign()); | |||
2663 | Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2664 | LLVMContext::MD_access_group}); | |||
2665 | if (AATags) | |||
2666 | Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2667 | Pass.DeadInsts.push_back(&SI); | |||
2668 | ||||
2669 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||
2670 | return true; | |||
2671 | } | |||
2672 | ||||
2673 | bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) { | |||
2674 | assert(IntTy && "We cannot extract an integer from the alloca")(static_cast <bool> (IntTy && "We cannot extract an integer from the alloca" ) ? void (0) : __assert_fail ("IntTy && \"We cannot extract an integer from the alloca\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2674, __extension__ __PRETTY_FUNCTION__)); | |||
2675 | assert(!SI.isVolatile())(static_cast <bool> (!SI.isVolatile()) ? void (0) : __assert_fail ("!SI.isVolatile()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2675, __extension__ __PRETTY_FUNCTION__)); | |||
2676 | if (DL.getTypeSizeInBits(V->getType()).getFixedSize() != | |||
2677 | IntTy->getBitWidth()) { | |||
2678 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2679 | NewAI.getAlign(), "oldload"); | |||
2680 | Old = convertValue(DL, IRB, Old, IntTy); | |||
2681 | assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset")(static_cast <bool> (BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset") ? void (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"Out of bounds offset\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2681, __extension__ __PRETTY_FUNCTION__)); | |||
2682 | uint64_t Offset = BeginOffset - NewAllocaBeginOffset; | |||
2683 | V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert"); | |||
2684 | } | |||
2685 | V = convertValue(DL, IRB, V, NewAllocaTy); | |||
2686 | StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign()); | |||
2687 | Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2688 | LLVMContext::MD_access_group}); | |||
2689 | if (AATags) | |||
2690 | Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2691 | Pass.DeadInsts.push_back(&SI); | |||
2692 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||
2693 | return true; | |||
2694 | } | |||
2695 | ||||
2696 | bool visitStoreInst(StoreInst &SI) { | |||
2697 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||
2698 | Value *OldOp = SI.getOperand(1); | |||
2699 | assert(OldOp == OldPtr)(static_cast <bool> (OldOp == OldPtr) ? void (0) : __assert_fail ("OldOp == OldPtr", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2699, __extension__ __PRETTY_FUNCTION__)); | |||
2700 | ||||
2701 | AAMDNodes AATags; | |||
2702 | SI.getAAMetadata(AATags); | |||
2703 | ||||
2704 | Value *V = SI.getValueOperand(); | |||
2705 | ||||
2706 | // Strip all inbounds GEPs and pointer casts to try to dig out any root | |||
2707 | // alloca that should be re-examined after promoting this alloca. | |||
2708 | if (V->getType()->isPointerTy()) | |||
2709 | if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets())) | |||
2710 | Pass.PostPromotionWorklist.insert(AI); | |||
2711 | ||||
2712 | if (SliceSize < DL.getTypeStoreSize(V->getType()).getFixedSize()) { | |||
2713 | assert(!SI.isVolatile())(static_cast <bool> (!SI.isVolatile()) ? void (0) : __assert_fail ("!SI.isVolatile()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2713, __extension__ __PRETTY_FUNCTION__)); | |||
2714 | assert(V->getType()->isIntegerTy() &&(static_cast <bool> (V->getType()->isIntegerTy() && "Only integer type loads and stores are split") ? void (0) : __assert_fail ("V->getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2715, __extension__ __PRETTY_FUNCTION__)) | |||
2715 | "Only integer type loads and stores are split")(static_cast <bool> (V->getType()->isIntegerTy() && "Only integer type loads and stores are split") ? void (0) : __assert_fail ("V->getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2715, __extension__ __PRETTY_FUNCTION__)); | |||
2716 | assert(DL.typeSizeEqualsStoreSize(V->getType()) &&(static_cast <bool> (DL.typeSizeEqualsStoreSize(V->getType ()) && "Non-byte-multiple bit width") ? void (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(V->getType()) && \"Non-byte-multiple bit width\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2717, __extension__ __PRETTY_FUNCTION__)) | |||
2717 | "Non-byte-multiple bit width")(static_cast <bool> (DL.typeSizeEqualsStoreSize(V->getType ()) && "Non-byte-multiple bit width") ? void (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(V->getType()) && \"Non-byte-multiple bit width\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2717, __extension__ __PRETTY_FUNCTION__)); | |||
2718 | IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8); | |||
2719 | V = extractInteger(DL, IRB, V, NarrowTy, NewBeginOffset - BeginOffset, | |||
2720 | "extract"); | |||
2721 | } | |||
2722 | ||||
2723 | if (VecTy) | |||
2724 | return rewriteVectorizedStoreInst(V, SI, OldOp, AATags); | |||
2725 | if (IntTy && V->getType()->isIntegerTy()) | |||
2726 | return rewriteIntegerStore(V, SI, AATags); | |||
2727 | ||||
2728 | const bool IsStorePastEnd = | |||
2729 | DL.getTypeStoreSize(V->getType()).getFixedSize() > SliceSize; | |||
2730 | StoreInst *NewSI; | |||
2731 | if (NewBeginOffset == NewAllocaBeginOffset && | |||
2732 | NewEndOffset == NewAllocaEndOffset && | |||
2733 | (canConvertValue(DL, V->getType(), NewAllocaTy) || | |||
2734 | (IsStorePastEnd && NewAllocaTy->isIntegerTy() && | |||
2735 | V->getType()->isIntegerTy()))) { | |||
2736 | // If this is an integer store past the end of slice (and thus the bytes | |||
2737 | // past that point are irrelevant or this is unreachable), truncate the | |||
2738 | // value prior to storing. | |||
2739 | if (auto *VITy = dyn_cast<IntegerType>(V->getType())) | |||
2740 | if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy)) | |||
2741 | if (VITy->getBitWidth() > AITy->getBitWidth()) { | |||
2742 | if (DL.isBigEndian()) | |||
2743 | V = IRB.CreateLShr(V, VITy->getBitWidth() - AITy->getBitWidth(), | |||
2744 | "endian_shift"); | |||
2745 | V = IRB.CreateTrunc(V, AITy, "load.trunc"); | |||
2746 | } | |||
2747 | ||||
2748 | V = convertValue(DL, IRB, V, NewAllocaTy); | |||
2749 | NewSI = | |||
2750 | IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign(), SI.isVolatile()); | |||
2751 | } else { | |||
2752 | unsigned AS = SI.getPointerAddressSpace(); | |||
2753 | Value *NewPtr = getNewAllocaSlicePtr(IRB, V->getType()->getPointerTo(AS)); | |||
2754 | NewSI = | |||
2755 | IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(), SI.isVolatile()); | |||
2756 | } | |||
2757 | NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
2758 | LLVMContext::MD_access_group}); | |||
2759 | if (AATags) | |||
2760 | NewSI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2761 | if (SI.isVolatile()) | |||
2762 | NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID()); | |||
2763 | if (NewSI->isAtomic()) | |||
2764 | NewSI->setAlignment(SI.getAlign()); | |||
2765 | Pass.DeadInsts.push_back(&SI); | |||
2766 | deleteIfTriviallyDead(OldOp); | |||
2767 | ||||
2768 | LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *NewSI << "\n"; } } while (false); | |||
2769 | return NewSI->getPointerOperand() == &NewAI && !SI.isVolatile(); | |||
2770 | } | |||
2771 | ||||
2772 | /// Compute an integer value from splatting an i8 across the given | |||
2773 | /// number of bytes. | |||
2774 | /// | |||
2775 | /// Note that this routine assumes an i8 is a byte. If that isn't true, don't | |||
2776 | /// call this routine. | |||
2777 | /// FIXME: Heed the advice above. | |||
2778 | /// | |||
2779 | /// \param V The i8 value to splat. | |||
2780 | /// \param Size The number of bytes in the output (assuming i8 is one byte) | |||
2781 | Value *getIntegerSplat(Value *V, unsigned Size) { | |||
2782 | assert(Size > 0 && "Expected a positive number of bytes.")(static_cast <bool> (Size > 0 && "Expected a positive number of bytes." ) ? void (0) : __assert_fail ("Size > 0 && \"Expected a positive number of bytes.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2782, __extension__ __PRETTY_FUNCTION__)); | |||
2783 | IntegerType *VTy = cast<IntegerType>(V->getType()); | |||
2784 | assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte")(static_cast <bool> (VTy->getBitWidth() == 8 && "Expected an i8 value for the byte") ? void (0) : __assert_fail ("VTy->getBitWidth() == 8 && \"Expected an i8 value for the byte\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2784, __extension__ __PRETTY_FUNCTION__)); | |||
2785 | if (Size == 1) | |||
2786 | return V; | |||
2787 | ||||
2788 | Type *SplatIntTy = Type::getIntNTy(VTy->getContext(), Size * 8); | |||
2789 | V = IRB.CreateMul( | |||
2790 | IRB.CreateZExt(V, SplatIntTy, "zext"), | |||
2791 | ConstantExpr::getUDiv( | |||
2792 | Constant::getAllOnesValue(SplatIntTy), | |||
2793 | ConstantExpr::getZExt(Constant::getAllOnesValue(V->getType()), | |||
2794 | SplatIntTy)), | |||
2795 | "isplat"); | |||
2796 | return V; | |||
2797 | } | |||
2798 | ||||
2799 | /// Compute a vector splat for a given element value. | |||
2800 | Value *getVectorSplat(Value *V, unsigned NumElements) { | |||
2801 | V = IRB.CreateVectorSplat(NumElements, V, "vsplat"); | |||
2802 | LLVM_DEBUG(dbgs() << " splat: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " splat: " << *V << "\n"; } } while (false); | |||
2803 | return V; | |||
2804 | } | |||
2805 | ||||
2806 | bool visitMemSetInst(MemSetInst &II) { | |||
2807 | LLVM_DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | |||
2808 | assert(II.getRawDest() == OldPtr)(static_cast <bool> (II.getRawDest() == OldPtr) ? void ( 0) : __assert_fail ("II.getRawDest() == OldPtr", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2808, __extension__ __PRETTY_FUNCTION__)); | |||
2809 | ||||
2810 | AAMDNodes AATags; | |||
2811 | II.getAAMetadata(AATags); | |||
2812 | ||||
2813 | // If the memset has a variable size, it cannot be split, just adjust the | |||
2814 | // pointer to the new alloca. | |||
2815 | if (!isa<ConstantInt>(II.getLength())) { | |||
2816 | assert(!IsSplit)(static_cast <bool> (!IsSplit) ? void (0) : __assert_fail ("!IsSplit", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2816, __extension__ __PRETTY_FUNCTION__)); | |||
2817 | assert(NewBeginOffset == BeginOffset)(static_cast <bool> (NewBeginOffset == BeginOffset) ? void (0) : __assert_fail ("NewBeginOffset == BeginOffset", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2817, __extension__ __PRETTY_FUNCTION__)); | |||
2818 | II.setDest(getNewAllocaSlicePtr(IRB, OldPtr->getType())); | |||
2819 | II.setDestAlignment(getSliceAlign()); | |||
2820 | ||||
2821 | deleteIfTriviallyDead(OldPtr); | |||
2822 | return false; | |||
2823 | } | |||
2824 | ||||
2825 | // Record this instruction for deletion. | |||
2826 | Pass.DeadInsts.push_back(&II); | |||
2827 | ||||
2828 | Type *AllocaTy = NewAI.getAllocatedType(); | |||
2829 | Type *ScalarTy = AllocaTy->getScalarType(); | |||
2830 | ||||
2831 | const bool CanContinue = [&]() { | |||
2832 | if (VecTy || IntTy) | |||
2833 | return true; | |||
2834 | if (BeginOffset > NewAllocaBeginOffset || | |||
2835 | EndOffset < NewAllocaEndOffset) | |||
2836 | return false; | |||
2837 | // Length must be in range for FixedVectorType. | |||
2838 | auto *C = cast<ConstantInt>(II.getLength()); | |||
2839 | const uint64_t Len = C->getLimitedValue(); | |||
2840 | if (Len > std::numeric_limits<unsigned>::max()) | |||
2841 | return false; | |||
2842 | auto *Int8Ty = IntegerType::getInt8Ty(NewAI.getContext()); | |||
2843 | auto *SrcTy = FixedVectorType::get(Int8Ty, Len); | |||
2844 | return canConvertValue(DL, SrcTy, AllocaTy) && | |||
2845 | DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy).getFixedSize()); | |||
2846 | }(); | |||
2847 | ||||
2848 | // If this doesn't map cleanly onto the alloca type, and that type isn't | |||
2849 | // a single value type, just emit a memset. | |||
2850 | if (!CanContinue) { | |||
2851 | Type *SizeTy = II.getLength()->getType(); | |||
2852 | Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset); | |||
2853 | CallInst *New = IRB.CreateMemSet( | |||
2854 | getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size, | |||
2855 | MaybeAlign(getSliceAlign()), II.isVolatile()); | |||
2856 | if (AATags) | |||
2857 | New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2858 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||
2859 | return false; | |||
2860 | } | |||
2861 | ||||
2862 | // If we can represent this as a simple value, we have to build the actual | |||
2863 | // value to store, which requires expanding the byte present in memset to | |||
2864 | // a sensible representation for the alloca type. This is essentially | |||
2865 | // splatting the byte to a sufficiently wide integer, splatting it across | |||
2866 | // any desired vector width, and bitcasting to the final type. | |||
2867 | Value *V; | |||
2868 | ||||
2869 | if (VecTy) { | |||
2870 | // If this is a memset of a vectorized alloca, insert it. | |||
2871 | assert(ElementTy == ScalarTy)(static_cast <bool> (ElementTy == ScalarTy) ? void (0) : __assert_fail ("ElementTy == ScalarTy", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2871, __extension__ __PRETTY_FUNCTION__)); | |||
2872 | ||||
2873 | unsigned BeginIndex = getIndex(NewBeginOffset); | |||
2874 | unsigned EndIndex = getIndex(NewEndOffset); | |||
2875 | assert(EndIndex > BeginIndex && "Empty vector!")(static_cast <bool> (EndIndex > BeginIndex && "Empty vector!") ? void (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2875, __extension__ __PRETTY_FUNCTION__)); | |||
2876 | unsigned NumElements = EndIndex - BeginIndex; | |||
2877 | assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&(static_cast <bool> (NumElements <= cast<FixedVectorType >(VecTy)->getNumElements() && "Too many elements!" ) ? void (0) : __assert_fail ("NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2878, __extension__ __PRETTY_FUNCTION__)) | |||
2878 | "Too many elements!")(static_cast <bool> (NumElements <= cast<FixedVectorType >(VecTy)->getNumElements() && "Too many elements!" ) ? void (0) : __assert_fail ("NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2878, __extension__ __PRETTY_FUNCTION__)); | |||
2879 | ||||
2880 | Value *Splat = getIntegerSplat( | |||
2881 | II.getValue(), DL.getTypeSizeInBits(ElementTy).getFixedSize() / 8); | |||
2882 | Splat = convertValue(DL, IRB, Splat, ElementTy); | |||
2883 | if (NumElements > 1) | |||
2884 | Splat = getVectorSplat(Splat, NumElements); | |||
2885 | ||||
2886 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2887 | NewAI.getAlign(), "oldload"); | |||
2888 | V = insertVector(IRB, Old, Splat, BeginIndex, "vec"); | |||
2889 | } else if (IntTy) { | |||
2890 | // If this is a memset on an alloca where we can widen stores, insert the | |||
2891 | // set integer. | |||
2892 | assert(!II.isVolatile())(static_cast <bool> (!II.isVolatile()) ? void (0) : __assert_fail ("!II.isVolatile()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2892, __extension__ __PRETTY_FUNCTION__)); | |||
2893 | ||||
2894 | uint64_t Size = NewEndOffset - NewBeginOffset; | |||
2895 | V = getIntegerSplat(II.getValue(), Size); | |||
2896 | ||||
2897 | if (IntTy && (BeginOffset != NewAllocaBeginOffset || | |||
2898 | EndOffset != NewAllocaBeginOffset)) { | |||
2899 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
2900 | NewAI.getAlign(), "oldload"); | |||
2901 | Old = convertValue(DL, IRB, Old, IntTy); | |||
2902 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
2903 | V = insertInteger(DL, IRB, Old, V, Offset, "insert"); | |||
2904 | } else { | |||
2905 | assert(V->getType() == IntTy &&(static_cast <bool> (V->getType() == IntTy && "Wrong type for an alloca wide integer!") ? void (0) : __assert_fail ("V->getType() == IntTy && \"Wrong type for an alloca wide integer!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2906, __extension__ __PRETTY_FUNCTION__)) | |||
2906 | "Wrong type for an alloca wide integer!")(static_cast <bool> (V->getType() == IntTy && "Wrong type for an alloca wide integer!") ? void (0) : __assert_fail ("V->getType() == IntTy && \"Wrong type for an alloca wide integer!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2906, __extension__ __PRETTY_FUNCTION__)); | |||
2907 | } | |||
2908 | V = convertValue(DL, IRB, V, AllocaTy); | |||
2909 | } else { | |||
2910 | // Established these invariants above. | |||
2911 | assert(NewBeginOffset == NewAllocaBeginOffset)(static_cast <bool> (NewBeginOffset == NewAllocaBeginOffset ) ? void (0) : __assert_fail ("NewBeginOffset == NewAllocaBeginOffset" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2911, __extension__ __PRETTY_FUNCTION__)); | |||
2912 | assert(NewEndOffset == NewAllocaEndOffset)(static_cast <bool> (NewEndOffset == NewAllocaEndOffset ) ? void (0) : __assert_fail ("NewEndOffset == NewAllocaEndOffset" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2912, __extension__ __PRETTY_FUNCTION__)); | |||
2913 | ||||
2914 | V = getIntegerSplat(II.getValue(), | |||
2915 | DL.getTypeSizeInBits(ScalarTy).getFixedSize() / 8); | |||
2916 | if (VectorType *AllocaVecTy = dyn_cast<VectorType>(AllocaTy)) | |||
2917 | V = getVectorSplat( | |||
2918 | V, cast<FixedVectorType>(AllocaVecTy)->getNumElements()); | |||
2919 | ||||
2920 | V = convertValue(DL, IRB, V, AllocaTy); | |||
2921 | } | |||
2922 | ||||
2923 | StoreInst *New = | |||
2924 | IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign(), II.isVolatile()); | |||
2925 | New->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access, | |||
2926 | LLVMContext::MD_access_group}); | |||
2927 | if (AATags) | |||
2928 | New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
2929 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||
2930 | return !II.isVolatile(); | |||
2931 | } | |||
2932 | ||||
2933 | bool visitMemTransferInst(MemTransferInst &II) { | |||
2934 | // Rewriting of memory transfer instructions can be a bit tricky. We break | |||
2935 | // them into two categories: split intrinsics and unsplit intrinsics. | |||
2936 | ||||
2937 | LLVM_DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | |||
| ||||
2938 | ||||
2939 | AAMDNodes AATags; | |||
2940 | II.getAAMetadata(AATags); | |||
2941 | ||||
2942 | bool IsDest = &II.getRawDestUse() == OldUse; | |||
2943 | assert((IsDest && II.getRawDest() == OldPtr) ||(static_cast <bool> ((IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr )) ? void (0) : __assert_fail ("(IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2944, __extension__ __PRETTY_FUNCTION__)) | |||
2944 | (!IsDest && II.getRawSource() == OldPtr))(static_cast <bool> ((IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr )) ? void (0) : __assert_fail ("(IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2944, __extension__ __PRETTY_FUNCTION__)); | |||
2945 | ||||
2946 | MaybeAlign SliceAlign = getSliceAlign(); | |||
2947 | ||||
2948 | // For unsplit intrinsics, we simply modify the source and destination | |||
2949 | // pointers in place. This isn't just an optimization, it is a matter of | |||
2950 | // correctness. With unsplit intrinsics we may be dealing with transfers | |||
2951 | // within a single alloca before SROA ran, or with transfers that have | |||
2952 | // a variable length. We may also be dealing with memmove instead of | |||
2953 | // memcpy, and so simply updating the pointers is the necessary for us to | |||
2954 | // update both source and dest of a single call. | |||
2955 | if (!IsSplittable) { | |||
2956 | Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||
| ||||
2957 | if (IsDest) { | |||
2958 | II.setDest(AdjustedPtr); | |||
2959 | II.setDestAlignment(SliceAlign); | |||
2960 | } | |||
2961 | else { | |||
2962 | II.setSource(AdjustedPtr); | |||
2963 | II.setSourceAlignment(SliceAlign); | |||
2964 | } | |||
2965 | ||||
2966 | LLVM_DEBUG(dbgs() << " to: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << II << "\n"; } } while (false); | |||
2967 | deleteIfTriviallyDead(OldPtr); | |||
2968 | return false; | |||
2969 | } | |||
2970 | // For split transfer intrinsics we have an incredibly useful assurance: | |||
2971 | // the source and destination do not reside within the same alloca, and at | |||
2972 | // least one of them does not escape. This means that we can replace | |||
2973 | // memmove with memcpy, and we don't need to worry about all manner of | |||
2974 | // downsides to splitting and transforming the operations. | |||
2975 | ||||
2976 | // If this doesn't map cleanly onto the alloca type, and that type isn't | |||
2977 | // a single value type, just emit a memcpy. | |||
2978 | bool EmitMemCpy = | |||
2979 | !VecTy && !IntTy && | |||
2980 | (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset || | |||
2981 | SliceSize != | |||
2982 | DL.getTypeStoreSize(NewAI.getAllocatedType()).getFixedSize() || | |||
2983 | !NewAI.getAllocatedType()->isSingleValueType()); | |||
2984 | ||||
2985 | // If we're just going to emit a memcpy, the alloca hasn't changed, and the | |||
2986 | // size hasn't been shrunk based on analysis of the viable range, this is | |||
2987 | // a no-op. | |||
2988 | if (EmitMemCpy && &OldAI == &NewAI) { | |||
2989 | // Ensure the start lines up. | |||
2990 | assert(NewBeginOffset == BeginOffset)(static_cast <bool> (NewBeginOffset == BeginOffset) ? void (0) : __assert_fail ("NewBeginOffset == BeginOffset", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 2990, __extension__ __PRETTY_FUNCTION__)); | |||
2991 | ||||
2992 | // Rewrite the size as needed. | |||
2993 | if (NewEndOffset != EndOffset) | |||
2994 | II.setLength(ConstantInt::get(II.getLength()->getType(), | |||
2995 | NewEndOffset - NewBeginOffset)); | |||
2996 | return false; | |||
2997 | } | |||
2998 | // Record this instruction for deletion. | |||
2999 | Pass.DeadInsts.push_back(&II); | |||
3000 | ||||
3001 | // Strip all inbounds GEPs and pointer casts to try to dig out any root | |||
3002 | // alloca that should be re-examined after rewriting this instruction. | |||
3003 | Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest(); | |||
3004 | if (AllocaInst *AI = | |||
3005 | dyn_cast<AllocaInst>(OtherPtr->stripInBoundsOffsets())) { | |||
3006 | assert(AI != &OldAI && AI != &NewAI &&(static_cast <bool> (AI != &OldAI && AI != & NewAI && "Splittable transfers cannot reach the same alloca on both ends." ) ? void (0) : __assert_fail ("AI != &OldAI && AI != &NewAI && \"Splittable transfers cannot reach the same alloca on both ends.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3007, __extension__ __PRETTY_FUNCTION__)) | |||
3007 | "Splittable transfers cannot reach the same alloca on both ends.")(static_cast <bool> (AI != &OldAI && AI != & NewAI && "Splittable transfers cannot reach the same alloca on both ends." ) ? void (0) : __assert_fail ("AI != &OldAI && AI != &NewAI && \"Splittable transfers cannot reach the same alloca on both ends.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3007, __extension__ __PRETTY_FUNCTION__)); | |||
3008 | Pass.Worklist.insert(AI); | |||
3009 | } | |||
3010 | ||||
3011 | Type *OtherPtrTy = OtherPtr->getType(); | |||
3012 | unsigned OtherAS = OtherPtrTy->getPointerAddressSpace(); | |||
3013 | ||||
3014 | // Compute the relative offset for the other pointer within the transfer. | |||
3015 | unsigned OffsetWidth = DL.getIndexSizeInBits(OtherAS); | |||
3016 | APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset); | |||
3017 | Align OtherAlign = | |||
3018 | (IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne(); | |||
3019 | OtherAlign = | |||
3020 | commonAlignment(OtherAlign, OtherOffset.zextOrTrunc(64).getZExtValue()); | |||
3021 | ||||
3022 | if (EmitMemCpy) { | |||
3023 | // Compute the other pointer, folding as much as possible to produce | |||
3024 | // a single, simple GEP in most cases. | |||
3025 | OtherPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy, | |||
3026 | OtherPtr->getName() + "."); | |||
3027 | ||||
3028 | Value *OurPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||
3029 | Type *SizeTy = II.getLength()->getType(); | |||
3030 | Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset); | |||
3031 | ||||
3032 | Value *DestPtr, *SrcPtr; | |||
3033 | MaybeAlign DestAlign, SrcAlign; | |||
3034 | // Note: IsDest is true iff we're copying into the new alloca slice | |||
3035 | if (IsDest) { | |||
3036 | DestPtr = OurPtr; | |||
3037 | DestAlign = SliceAlign; | |||
3038 | SrcPtr = OtherPtr; | |||
3039 | SrcAlign = OtherAlign; | |||
3040 | } else { | |||
3041 | DestPtr = OtherPtr; | |||
3042 | DestAlign = OtherAlign; | |||
3043 | SrcPtr = OurPtr; | |||
3044 | SrcAlign = SliceAlign; | |||
3045 | } | |||
3046 | CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign, | |||
3047 | Size, II.isVolatile()); | |||
3048 | if (AATags) | |||
3049 | New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
3050 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||
3051 | return false; | |||
3052 | } | |||
3053 | ||||
3054 | bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset && | |||
3055 | NewEndOffset == NewAllocaEndOffset; | |||
3056 | uint64_t Size = NewEndOffset - NewBeginOffset; | |||
3057 | unsigned BeginIndex = VecTy ? getIndex(NewBeginOffset) : 0; | |||
3058 | unsigned EndIndex = VecTy ? getIndex(NewEndOffset) : 0; | |||
3059 | unsigned NumElements = EndIndex - BeginIndex; | |||
3060 | IntegerType *SubIntTy = | |||
3061 | IntTy ? Type::getIntNTy(IntTy->getContext(), Size * 8) : nullptr; | |||
3062 | ||||
3063 | // Reset the other pointer type to match the register type we're going to | |||
3064 | // use, but using the address space of the original other pointer. | |||
3065 | Type *OtherTy; | |||
3066 | if (VecTy && !IsWholeAlloca) { | |||
3067 | if (NumElements == 1) | |||
3068 | OtherTy = VecTy->getElementType(); | |||
3069 | else | |||
3070 | OtherTy = FixedVectorType::get(VecTy->getElementType(), NumElements); | |||
3071 | } else if (IntTy && !IsWholeAlloca) { | |||
3072 | OtherTy = SubIntTy; | |||
3073 | } else { | |||
3074 | OtherTy = NewAllocaTy; | |||
3075 | } | |||
3076 | OtherPtrTy = OtherTy->getPointerTo(OtherAS); | |||
3077 | ||||
3078 | Value *SrcPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy, | |||
3079 | OtherPtr->getName() + "."); | |||
3080 | MaybeAlign SrcAlign = OtherAlign; | |||
3081 | Value *DstPtr = &NewAI; | |||
3082 | MaybeAlign DstAlign = SliceAlign; | |||
3083 | if (!IsDest) { | |||
3084 | std::swap(SrcPtr, DstPtr); | |||
3085 | std::swap(SrcAlign, DstAlign); | |||
3086 | } | |||
3087 | ||||
3088 | Value *Src; | |||
3089 | if (VecTy && !IsWholeAlloca && !IsDest) { | |||
3090 | Src = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3091 | NewAI.getAlign(), "load"); | |||
3092 | Src = extractVector(IRB, Src, BeginIndex, EndIndex, "vec"); | |||
3093 | } else if (IntTy && !IsWholeAlloca && !IsDest) { | |||
3094 | Src = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3095 | NewAI.getAlign(), "load"); | |||
3096 | Src = convertValue(DL, IRB, Src, IntTy); | |||
3097 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
3098 | Src = extractInteger(DL, IRB, Src, SubIntTy, Offset, "extract"); | |||
3099 | } else { | |||
3100 | LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign, | |||
3101 | II.isVolatile(), "copyload"); | |||
3102 | Load->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access, | |||
3103 | LLVMContext::MD_access_group}); | |||
3104 | if (AATags) | |||
3105 | Load->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
3106 | Src = Load; | |||
3107 | } | |||
3108 | ||||
3109 | if (VecTy && !IsWholeAlloca && IsDest) { | |||
3110 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3111 | NewAI.getAlign(), "oldload"); | |||
3112 | Src = insertVector(IRB, Old, Src, BeginIndex, "vec"); | |||
3113 | } else if (IntTy && !IsWholeAlloca && IsDest) { | |||
3114 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | |||
3115 | NewAI.getAlign(), "oldload"); | |||
3116 | Old = convertValue(DL, IRB, Old, IntTy); | |||
3117 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||
3118 | Src = insertInteger(DL, IRB, Old, Src, Offset, "insert"); | |||
3119 | Src = convertValue(DL, IRB, Src, NewAllocaTy); | |||
3120 | } | |||
3121 | ||||
3122 | StoreInst *Store = cast<StoreInst>( | |||
3123 | IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile())); | |||
3124 | Store->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access, | |||
3125 | LLVMContext::MD_access_group}); | |||
3126 | if (AATags) | |||
3127 | Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset)); | |||
3128 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||
3129 | return !II.isVolatile(); | |||
3130 | } | |||
3131 | ||||
3132 | bool visitIntrinsicInst(IntrinsicInst &II) { | |||
3133 | assert((II.isLifetimeStartOrEnd() || II.isDroppable()) &&(static_cast <bool> ((II.isLifetimeStartOrEnd() || II.isDroppable ()) && "Unexpected intrinsic!") ? void (0) : __assert_fail ("(II.isLifetimeStartOrEnd() || II.isDroppable()) && \"Unexpected intrinsic!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3134, __extension__ __PRETTY_FUNCTION__)) | |||
3134 | "Unexpected intrinsic!")(static_cast <bool> ((II.isLifetimeStartOrEnd() || II.isDroppable ()) && "Unexpected intrinsic!") ? void (0) : __assert_fail ("(II.isLifetimeStartOrEnd() || II.isDroppable()) && \"Unexpected intrinsic!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3134, __extension__ __PRETTY_FUNCTION__)); | |||
3135 | LLVM_DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | |||
3136 | ||||
3137 | // Record this instruction for deletion. | |||
3138 | Pass.DeadInsts.push_back(&II); | |||
3139 | ||||
3140 | if (II.isDroppable()) { | |||
3141 | assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume")(static_cast <bool> (II.getIntrinsicID() == Intrinsic:: assume && "Expected assume") ? void (0) : __assert_fail ("II.getIntrinsicID() == Intrinsic::assume && \"Expected assume\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3141, __extension__ __PRETTY_FUNCTION__)); | |||
3142 | // TODO For now we forget assumed information, this can be improved. | |||
3143 | OldPtr->dropDroppableUsesIn(II); | |||
3144 | return true; | |||
3145 | } | |||
3146 | ||||
3147 | assert(II.getArgOperand(1) == OldPtr)(static_cast <bool> (II.getArgOperand(1) == OldPtr) ? void (0) : __assert_fail ("II.getArgOperand(1) == OldPtr", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3147, __extension__ __PRETTY_FUNCTION__)); | |||
3148 | // Lifetime intrinsics are only promotable if they cover the whole alloca. | |||
3149 | // Therefore, we drop lifetime intrinsics which don't cover the whole | |||
3150 | // alloca. | |||
3151 | // (In theory, intrinsics which partially cover an alloca could be | |||
3152 | // promoted, but PromoteMemToReg doesn't handle that case.) | |||
3153 | // FIXME: Check whether the alloca is promotable before dropping the | |||
3154 | // lifetime intrinsics? | |||
3155 | if (NewBeginOffset != NewAllocaBeginOffset || | |||
3156 | NewEndOffset != NewAllocaEndOffset) | |||
3157 | return true; | |||
3158 | ||||
3159 | ConstantInt *Size = | |||
3160 | ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()), | |||
3161 | NewEndOffset - NewBeginOffset); | |||
3162 | // Lifetime intrinsics always expect an i8* so directly get such a pointer | |||
3163 | // for the new alloca slice. | |||
3164 | Type *PointerTy = IRB.getInt8PtrTy(OldPtr->getType()->getPointerAddressSpace()); | |||
3165 | Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy); | |||
3166 | Value *New; | |||
3167 | if (II.getIntrinsicID() == Intrinsic::lifetime_start) | |||
3168 | New = IRB.CreateLifetimeStart(Ptr, Size); | |||
3169 | else | |||
3170 | New = IRB.CreateLifetimeEnd(Ptr, Size); | |||
3171 | ||||
3172 | (void)New; | |||
3173 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||
3174 | ||||
3175 | return true; | |||
3176 | } | |||
3177 | ||||
3178 | void fixLoadStoreAlign(Instruction &Root) { | |||
3179 | // This algorithm implements the same visitor loop as | |||
3180 | // hasUnsafePHIOrSelectUse, and fixes the alignment of each load | |||
3181 | // or store found. | |||
3182 | SmallPtrSet<Instruction *, 4> Visited; | |||
3183 | SmallVector<Instruction *, 4> Uses; | |||
3184 | Visited.insert(&Root); | |||
3185 | Uses.push_back(&Root); | |||
3186 | do { | |||
3187 | Instruction *I = Uses.pop_back_val(); | |||
3188 | ||||
3189 | if (LoadInst *LI = dyn_cast<LoadInst>(I)) { | |||
3190 | LI->setAlignment(std::min(LI->getAlign(), getSliceAlign())); | |||
3191 | continue; | |||
3192 | } | |||
3193 | if (StoreInst *SI = dyn_cast<StoreInst>(I)) { | |||
3194 | SI->setAlignment(std::min(SI->getAlign(), getSliceAlign())); | |||
3195 | continue; | |||
3196 | } | |||
3197 | ||||
3198 | assert(isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) ||(static_cast <bool> (isa<BitCastInst>(I) || isa< AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst >(I) || isa<GetElementPtrInst>(I)) ? void (0) : __assert_fail ("isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst>(I) || isa<GetElementPtrInst>(I)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3200, __extension__ __PRETTY_FUNCTION__)) | |||
3199 | isa<PHINode>(I) || isa<SelectInst>(I) ||(static_cast <bool> (isa<BitCastInst>(I) || isa< AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst >(I) || isa<GetElementPtrInst>(I)) ? void (0) : __assert_fail ("isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst>(I) || isa<GetElementPtrInst>(I)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3200, __extension__ __PRETTY_FUNCTION__)) | |||
3200 | isa<GetElementPtrInst>(I))(static_cast <bool> (isa<BitCastInst>(I) || isa< AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst >(I) || isa<GetElementPtrInst>(I)) ? void (0) : __assert_fail ("isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst>(I) || isa<GetElementPtrInst>(I)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3200, __extension__ __PRETTY_FUNCTION__)); | |||
3201 | for (User *U : I->users()) | |||
3202 | if (Visited.insert(cast<Instruction>(U)).second) | |||
3203 | Uses.push_back(cast<Instruction>(U)); | |||
3204 | } while (!Uses.empty()); | |||
3205 | } | |||
3206 | ||||
3207 | bool visitPHINode(PHINode &PN) { | |||
3208 | LLVM_DEBUG(dbgs() << " original: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << PN << "\n"; } } while (false); | |||
3209 | assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable")(static_cast <bool> (BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable") ? void (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"PHIs are unsplittable\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3209, __extension__ __PRETTY_FUNCTION__)); | |||
3210 | assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable")(static_cast <bool> (EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable") ? void (0) : __assert_fail ("EndOffset <= NewAllocaEndOffset && \"PHIs are unsplittable\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3210, __extension__ __PRETTY_FUNCTION__)); | |||
3211 | ||||
3212 | // We would like to compute a new pointer in only one place, but have it be | |||
3213 | // as local as possible to the PHI. To do that, we re-use the location of | |||
3214 | // the old pointer, which necessarily must be in the right position to | |||
3215 | // dominate the PHI. | |||
3216 | IRBuilderBase::InsertPointGuard Guard(IRB); | |||
3217 | if (isa<PHINode>(OldPtr)) | |||
3218 | IRB.SetInsertPoint(&*OldPtr->getParent()->getFirstInsertionPt()); | |||
3219 | else | |||
3220 | IRB.SetInsertPoint(OldPtr); | |||
3221 | IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc()); | |||
3222 | ||||
3223 | Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||
3224 | // Replace the operands which were using the old pointer. | |||
3225 | std::replace(PN.op_begin(), PN.op_end(), cast<Value>(OldPtr), NewPtr); | |||
3226 | ||||
3227 | LLVM_DEBUG(dbgs() << " to: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << PN << "\n"; } } while (false); | |||
3228 | deleteIfTriviallyDead(OldPtr); | |||
3229 | ||||
3230 | // Fix the alignment of any loads or stores using this PHI node. | |||
3231 | fixLoadStoreAlign(PN); | |||
3232 | ||||
3233 | // PHIs can't be promoted on their own, but often can be speculated. We | |||
3234 | // check the speculation outside of the rewriter so that we see the | |||
3235 | // fully-rewritten alloca. | |||
3236 | PHIUsers.insert(&PN); | |||
3237 | return true; | |||
3238 | } | |||
3239 | ||||
3240 | bool visitSelectInst(SelectInst &SI) { | |||
3241 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||
3242 | assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&(static_cast <bool> ((SI.getTrueValue() == OldPtr || SI .getFalseValue() == OldPtr) && "Pointer isn't an operand!" ) ? void (0) : __assert_fail ("(SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && \"Pointer isn't an operand!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3243, __extension__ __PRETTY_FUNCTION__)) | |||
3243 | "Pointer isn't an operand!")(static_cast <bool> ((SI.getTrueValue() == OldPtr || SI .getFalseValue() == OldPtr) && "Pointer isn't an operand!" ) ? void (0) : __assert_fail ("(SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && \"Pointer isn't an operand!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3243, __extension__ __PRETTY_FUNCTION__)); | |||
3244 | assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable")(static_cast <bool> (BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable") ? void (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"Selects are unsplittable\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3244, __extension__ __PRETTY_FUNCTION__)); | |||
3245 | assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable")(static_cast <bool> (EndOffset <= NewAllocaEndOffset && "Selects are unsplittable") ? void (0) : __assert_fail ("EndOffset <= NewAllocaEndOffset && \"Selects are unsplittable\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3245, __extension__ __PRETTY_FUNCTION__)); | |||
3246 | ||||
3247 | Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||
3248 | // Replace the operands which were using the old pointer. | |||
3249 | if (SI.getOperand(1) == OldPtr) | |||
3250 | SI.setOperand(1, NewPtr); | |||
3251 | if (SI.getOperand(2) == OldPtr) | |||
3252 | SI.setOperand(2, NewPtr); | |||
3253 | ||||
3254 | LLVM_DEBUG(dbgs() << " to: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << SI << "\n"; } } while (false); | |||
3255 | deleteIfTriviallyDead(OldPtr); | |||
3256 | ||||
3257 | // Fix the alignment of any loads or stores using this select. | |||
3258 | fixLoadStoreAlign(SI); | |||
3259 | ||||
3260 | // Selects can't be promoted on their own, but often can be speculated. We | |||
3261 | // check the speculation outside of the rewriter so that we see the | |||
3262 | // fully-rewritten alloca. | |||
3263 | SelectUsers.insert(&SI); | |||
3264 | return true; | |||
3265 | } | |||
3266 | }; | |||
3267 | ||||
3268 | namespace { | |||
3269 | ||||
3270 | /// Visitor to rewrite aggregate loads and stores as scalar. | |||
3271 | /// | |||
3272 | /// This pass aggressively rewrites all aggregate loads and stores on | |||
3273 | /// a particular pointer (or any pointer derived from it which we can identify) | |||
3274 | /// with scalar loads and stores. | |||
3275 | class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> { | |||
3276 | // Befriend the base class so it can delegate to private visit methods. | |||
3277 | friend class InstVisitor<AggLoadStoreRewriter, bool>; | |||
3278 | ||||
3279 | /// Queue of pointer uses to analyze and potentially rewrite. | |||
3280 | SmallVector<Use *, 8> Queue; | |||
3281 | ||||
3282 | /// Set to prevent us from cycling with phi nodes and loops. | |||
3283 | SmallPtrSet<User *, 8> Visited; | |||
3284 | ||||
3285 | /// The current pointer use being rewritten. This is used to dig up the used | |||
3286 | /// value (as opposed to the user). | |||
3287 | Use *U = nullptr; | |||
3288 | ||||
3289 | /// Used to calculate offsets, and hence alignment, of subobjects. | |||
3290 | const DataLayout &DL; | |||
3291 | ||||
3292 | public: | |||
3293 | AggLoadStoreRewriter(const DataLayout &DL) : DL(DL) {} | |||
3294 | ||||
3295 | /// Rewrite loads and stores through a pointer and all pointers derived from | |||
3296 | /// it. | |||
3297 | bool rewrite(Instruction &I) { | |||
3298 | LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting FCA loads and stores...\n" ; } } while (false); | |||
3299 | enqueueUsers(I); | |||
3300 | bool Changed = false; | |||
3301 | while (!Queue.empty()) { | |||
3302 | U = Queue.pop_back_val(); | |||
3303 | Changed |= visit(cast<Instruction>(U->getUser())); | |||
3304 | } | |||
3305 | return Changed; | |||
3306 | } | |||
3307 | ||||
3308 | private: | |||
3309 | /// Enqueue all the users of the given instruction for further processing. | |||
3310 | /// This uses a set to de-duplicate users. | |||
3311 | void enqueueUsers(Instruction &I) { | |||
3312 | for (Use &U : I.uses()) | |||
3313 | if (Visited.insert(U.getUser()).second) | |||
3314 | Queue.push_back(&U); | |||
3315 | } | |||
3316 | ||||
3317 | // Conservative default is to not rewrite anything. | |||
3318 | bool visitInstruction(Instruction &I) { return false; } | |||
3319 | ||||
3320 | /// Generic recursive split emission class. | |||
3321 | template <typename Derived> class OpSplitter { | |||
3322 | protected: | |||
3323 | /// The builder used to form new instructions. | |||
3324 | IRBuilderTy IRB; | |||
3325 | ||||
3326 | /// The indices which to be used with insert- or extractvalue to select the | |||
3327 | /// appropriate value within the aggregate. | |||
3328 | SmallVector<unsigned, 4> Indices; | |||
3329 | ||||
3330 | /// The indices to a GEP instruction which will move Ptr to the correct slot | |||
3331 | /// within the aggregate. | |||
3332 | SmallVector<Value *, 4> GEPIndices; | |||
3333 | ||||
3334 | /// The base pointer of the original op, used as a base for GEPing the | |||
3335 | /// split operations. | |||
3336 | Value *Ptr; | |||
3337 | ||||
3338 | /// The base pointee type being GEPed into. | |||
3339 | Type *BaseTy; | |||
3340 | ||||
3341 | /// Known alignment of the base pointer. | |||
3342 | Align BaseAlign; | |||
3343 | ||||
3344 | /// To calculate offset of each component so we can correctly deduce | |||
3345 | /// alignments. | |||
3346 | const DataLayout &DL; | |||
3347 | ||||
3348 | /// Initialize the splitter with an insertion point, Ptr and start with a | |||
3349 | /// single zero GEP index. | |||
3350 | OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy, | |||
3351 | Align BaseAlign, const DataLayout &DL) | |||
3352 | : IRB(InsertionPoint), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr), | |||
3353 | BaseTy(BaseTy), BaseAlign(BaseAlign), DL(DL) {} | |||
3354 | ||||
3355 | public: | |||
3356 | /// Generic recursive split emission routine. | |||
3357 | /// | |||
3358 | /// This method recursively splits an aggregate op (load or store) into | |||
3359 | /// scalar or vector ops. It splits recursively until it hits a single value | |||
3360 | /// and emits that single value operation via the template argument. | |||
3361 | /// | |||
3362 | /// The logic of this routine relies on GEPs and insertvalue and | |||
3363 | /// extractvalue all operating with the same fundamental index list, merely | |||
3364 | /// formatted differently (GEPs need actual values). | |||
3365 | /// | |||
3366 | /// \param Ty The type being split recursively into smaller ops. | |||
3367 | /// \param Agg The aggregate value being built up or stored, depending on | |||
3368 | /// whether this is splitting a load or a store respectively. | |||
3369 | void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) { | |||
3370 | if (Ty->isSingleValueType()) { | |||
3371 | unsigned Offset = DL.getIndexedOffsetInType(BaseTy, GEPIndices); | |||
3372 | return static_cast<Derived *>(this)->emitFunc( | |||
3373 | Ty, Agg, commonAlignment(BaseAlign, Offset), Name); | |||
3374 | } | |||
3375 | ||||
3376 | if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { | |||
3377 | unsigned OldSize = Indices.size(); | |||
3378 | (void)OldSize; | |||
3379 | for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size; | |||
3380 | ++Idx) { | |||
3381 | assert(Indices.size() == OldSize && "Did not return to the old size")(static_cast <bool> (Indices.size() == OldSize && "Did not return to the old size") ? void (0) : __assert_fail ("Indices.size() == OldSize && \"Did not return to the old size\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3381, __extension__ __PRETTY_FUNCTION__)); | |||
3382 | Indices.push_back(Idx); | |||
3383 | GEPIndices.push_back(IRB.getInt32(Idx)); | |||
3384 | emitSplitOps(ATy->getElementType(), Agg, Name + "." + Twine(Idx)); | |||
3385 | GEPIndices.pop_back(); | |||
3386 | Indices.pop_back(); | |||
3387 | } | |||
3388 | return; | |||
3389 | } | |||
3390 | ||||
3391 | if (StructType *STy = dyn_cast<StructType>(Ty)) { | |||
3392 | unsigned OldSize = Indices.size(); | |||
3393 | (void)OldSize; | |||
3394 | for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size; | |||
3395 | ++Idx) { | |||
3396 | assert(Indices.size() == OldSize && "Did not return to the old size")(static_cast <bool> (Indices.size() == OldSize && "Did not return to the old size") ? void (0) : __assert_fail ("Indices.size() == OldSize && \"Did not return to the old size\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3396, __extension__ __PRETTY_FUNCTION__)); | |||
3397 | Indices.push_back(Idx); | |||
3398 | GEPIndices.push_back(IRB.getInt32(Idx)); | |||
3399 | emitSplitOps(STy->getElementType(Idx), Agg, Name + "." + Twine(Idx)); | |||
3400 | GEPIndices.pop_back(); | |||
3401 | Indices.pop_back(); | |||
3402 | } | |||
3403 | return; | |||
3404 | } | |||
3405 | ||||
3406 | llvm_unreachable("Only arrays and structs are aggregate loadable types")::llvm::llvm_unreachable_internal("Only arrays and structs are aggregate loadable types" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3406); | |||
3407 | } | |||
3408 | }; | |||
3409 | ||||
3410 | struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> { | |||
3411 | AAMDNodes AATags; | |||
3412 | ||||
3413 | LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy, | |||
3414 | AAMDNodes AATags, Align BaseAlign, const DataLayout &DL) | |||
3415 | : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, | |||
3416 | DL), | |||
3417 | AATags(AATags) {} | |||
3418 | ||||
3419 | /// Emit a leaf load of a single value. This is called at the leaves of the | |||
3420 | /// recursive emission to actually load values. | |||
3421 | void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) { | |||
3422 | assert(Ty->isSingleValueType())(static_cast <bool> (Ty->isSingleValueType()) ? void (0) : __assert_fail ("Ty->isSingleValueType()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3422, __extension__ __PRETTY_FUNCTION__)); | |||
3423 | // Load the single value and insert it using the indices. | |||
3424 | Value *GEP = | |||
3425 | IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep"); | |||
3426 | LoadInst *Load = | |||
3427 | IRB.CreateAlignedLoad(Ty, GEP, Alignment, Name + ".load"); | |||
3428 | ||||
3429 | APInt Offset( | |||
3430 | DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0); | |||
3431 | if (AATags && | |||
3432 | GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset)) | |||
3433 | Load->setAAMetadata(AATags.shift(Offset.getZExtValue())); | |||
3434 | ||||
3435 | Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert"); | |||
3436 | LLVM_DEBUG(dbgs() << " to: " << *Load << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Load << "\n"; } } while (false); | |||
3437 | } | |||
3438 | }; | |||
3439 | ||||
3440 | bool visitLoadInst(LoadInst &LI) { | |||
3441 | assert(LI.getPointerOperand() == *U)(static_cast <bool> (LI.getPointerOperand() == *U) ? void (0) : __assert_fail ("LI.getPointerOperand() == *U", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3441, __extension__ __PRETTY_FUNCTION__)); | |||
3442 | if (!LI.isSimple() || LI.getType()->isSingleValueType()) | |||
3443 | return false; | |||
3444 | ||||
3445 | // We have an aggregate being loaded, split it apart. | |||
3446 | LLVM_DEBUG(dbgs() << " original: " << LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << LI << "\n"; } } while (false); | |||
3447 | AAMDNodes AATags; | |||
3448 | LI.getAAMetadata(AATags); | |||
3449 | LoadOpSplitter Splitter(&LI, *U, LI.getType(), AATags, | |||
3450 | getAdjustedAlignment(&LI, 0), DL); | |||
3451 | Value *V = UndefValue::get(LI.getType()); | |||
3452 | Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca"); | |||
3453 | Visited.erase(&LI); | |||
3454 | LI.replaceAllUsesWith(V); | |||
3455 | LI.eraseFromParent(); | |||
3456 | return true; | |||
3457 | } | |||
3458 | ||||
3459 | struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> { | |||
3460 | StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy, | |||
3461 | AAMDNodes AATags, Align BaseAlign, const DataLayout &DL) | |||
3462 | : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, | |||
3463 | DL), | |||
3464 | AATags(AATags) {} | |||
3465 | AAMDNodes AATags; | |||
3466 | /// Emit a leaf store of a single value. This is called at the leaves of the | |||
3467 | /// recursive emission to actually produce stores. | |||
3468 | void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) { | |||
3469 | assert(Ty->isSingleValueType())(static_cast <bool> (Ty->isSingleValueType()) ? void (0) : __assert_fail ("Ty->isSingleValueType()", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3469, __extension__ __PRETTY_FUNCTION__)); | |||
3470 | // Extract the single value and store it using the indices. | |||
3471 | // | |||
3472 | // The gep and extractvalue values are factored out of the CreateStore | |||
3473 | // call to make the output independent of the argument evaluation order. | |||
3474 | Value *ExtractValue = | |||
3475 | IRB.CreateExtractValue(Agg, Indices, Name + ".extract"); | |||
3476 | Value *InBoundsGEP = | |||
3477 | IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep"); | |||
3478 | StoreInst *Store = | |||
3479 | IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Alignment); | |||
3480 | ||||
3481 | APInt Offset( | |||
3482 | DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0); | |||
3483 | if (AATags && | |||
3484 | GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset)) | |||
3485 | Store->setAAMetadata(AATags.shift(Offset.getZExtValue())); | |||
3486 | ||||
3487 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||
3488 | } | |||
3489 | }; | |||
3490 | ||||
3491 | bool visitStoreInst(StoreInst &SI) { | |||
3492 | if (!SI.isSimple() || SI.getPointerOperand() != *U) | |||
3493 | return false; | |||
3494 | Value *V = SI.getValueOperand(); | |||
3495 | if (V->getType()->isSingleValueType()) | |||
3496 | return false; | |||
3497 | ||||
3498 | // We have an aggregate being stored, split it apart. | |||
3499 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||
3500 | AAMDNodes AATags; | |||
3501 | SI.getAAMetadata(AATags); | |||
3502 | StoreOpSplitter Splitter(&SI, *U, V->getType(), AATags, | |||
3503 | getAdjustedAlignment(&SI, 0), DL); | |||
3504 | Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca"); | |||
3505 | Visited.erase(&SI); | |||
3506 | SI.eraseFromParent(); | |||
3507 | return true; | |||
3508 | } | |||
3509 | ||||
3510 | bool visitBitCastInst(BitCastInst &BC) { | |||
3511 | enqueueUsers(BC); | |||
3512 | return false; | |||
3513 | } | |||
3514 | ||||
3515 | bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) { | |||
3516 | enqueueUsers(ASC); | |||
3517 | return false; | |||
3518 | } | |||
3519 | ||||
3520 | // Fold gep (select cond, ptr1, ptr2) => select cond, gep(ptr1), gep(ptr2) | |||
3521 | bool foldGEPSelect(GetElementPtrInst &GEPI) { | |||
3522 | if (!GEPI.hasAllConstantIndices()) | |||
3523 | return false; | |||
3524 | ||||
3525 | SelectInst *Sel = cast<SelectInst>(GEPI.getPointerOperand()); | |||
3526 | ||||
3527 | LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(select) -> select(gep):" << "\n original: " << *Sel << "\n " << GEPI; } } while (false) | |||
3528 | << "\n original: " << *Seldo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(select) -> select(gep):" << "\n original: " << *Sel << "\n " << GEPI; } } while (false) | |||
3529 | << "\n " << GEPI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(select) -> select(gep):" << "\n original: " << *Sel << "\n " << GEPI; } } while (false); | |||
3530 | ||||
3531 | IRBuilderTy Builder(&GEPI); | |||
3532 | SmallVector<Value *, 4> Index(GEPI.indices()); | |||
3533 | bool IsInBounds = GEPI.isInBounds(); | |||
3534 | ||||
3535 | Type *Ty = GEPI.getSourceElementType(); | |||
3536 | Value *True = Sel->getTrueValue(); | |||
3537 | Value *NTrue = | |||
3538 | IsInBounds | |||
3539 | ? Builder.CreateInBoundsGEP(Ty, True, Index, | |||
3540 | True->getName() + ".sroa.gep") | |||
3541 | : Builder.CreateGEP(Ty, True, Index, True->getName() + ".sroa.gep"); | |||
3542 | ||||
3543 | Value *False = Sel->getFalseValue(); | |||
3544 | ||||
3545 | Value *NFalse = | |||
3546 | IsInBounds | |||
3547 | ? Builder.CreateInBoundsGEP(Ty, False, Index, | |||
3548 | False->getName() + ".sroa.gep") | |||
3549 | : Builder.CreateGEP(Ty, False, Index, | |||
3550 | False->getName() + ".sroa.gep"); | |||
3551 | ||||
3552 | Value *NSel = Builder.CreateSelect(Sel->getCondition(), NTrue, NFalse, | |||
3553 | Sel->getName() + ".sroa.sel"); | |||
3554 | Visited.erase(&GEPI); | |||
3555 | GEPI.replaceAllUsesWith(NSel); | |||
3556 | GEPI.eraseFromParent(); | |||
3557 | Instruction *NSelI = cast<Instruction>(NSel); | |||
3558 | Visited.insert(NSelI); | |||
3559 | enqueueUsers(*NSelI); | |||
3560 | ||||
3561 | LLVM_DEBUG(dbgs() << "\n to: " << *NTruedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n to: " << *NTrue << "\n " << *NFalse << "\n " << *NSel << '\n'; } } while (false) | |||
3562 | << "\n " << *NFalsedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n to: " << *NTrue << "\n " << *NFalse << "\n " << *NSel << '\n'; } } while (false) | |||
3563 | << "\n " << *NSel << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n to: " << *NTrue << "\n " << *NFalse << "\n " << *NSel << '\n'; } } while (false); | |||
3564 | ||||
3565 | return true; | |||
3566 | } | |||
3567 | ||||
3568 | // Fold gep (phi ptr1, ptr2) => phi gep(ptr1), gep(ptr2) | |||
3569 | bool foldGEPPhi(GetElementPtrInst &GEPI) { | |||
3570 | if (!GEPI.hasAllConstantIndices()) | |||
3571 | return false; | |||
3572 | ||||
3573 | PHINode *PHI = cast<PHINode>(GEPI.getPointerOperand()); | |||
3574 | if (GEPI.getParent() != PHI->getParent() || | |||
3575 | llvm::any_of(PHI->incoming_values(), [](Value *In) | |||
3576 | { Instruction *I = dyn_cast<Instruction>(In); | |||
3577 | return !I || isa<GetElementPtrInst>(I) || isa<PHINode>(I) || | |||
3578 | succ_empty(I->getParent()) || | |||
3579 | !I->getParent()->isLegalToHoistInto(); | |||
3580 | })) | |||
3581 | return false; | |||
3582 | ||||
3583 | LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(phi) -> phi(gep):" << "\n original: " << *PHI << "\n " << GEPI << "\n to: "; } } while (false) | |||
3584 | << "\n original: " << *PHIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(phi) -> phi(gep):" << "\n original: " << *PHI << "\n " << GEPI << "\n to: "; } } while (false) | |||
3585 | << "\n " << GEPIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(phi) -> phi(gep):" << "\n original: " << *PHI << "\n " << GEPI << "\n to: "; } } while (false) | |||
3586 | << "\n to: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting gep(phi) -> phi(gep):" << "\n original: " << *PHI << "\n " << GEPI << "\n to: "; } } while (false); | |||
3587 | ||||
3588 | SmallVector<Value *, 4> Index(GEPI.indices()); | |||
3589 | bool IsInBounds = GEPI.isInBounds(); | |||
3590 | IRBuilderTy PHIBuilder(GEPI.getParent()->getFirstNonPHI()); | |||
3591 | PHINode *NewPN = PHIBuilder.CreatePHI(GEPI.getType(), | |||
3592 | PHI->getNumIncomingValues(), | |||
3593 | PHI->getName() + ".sroa.phi"); | |||
3594 | for (unsigned I = 0, E = PHI->getNumIncomingValues(); I != E; ++I) { | |||
3595 | BasicBlock *B = PHI->getIncomingBlock(I); | |||
3596 | Value *NewVal = nullptr; | |||
3597 | int Idx = NewPN->getBasicBlockIndex(B); | |||
3598 | if (Idx >= 0) { | |||
3599 | NewVal = NewPN->getIncomingValue(Idx); | |||
3600 | } else { | |||
3601 | Instruction *In = cast<Instruction>(PHI->getIncomingValue(I)); | |||
3602 | ||||
3603 | IRBuilderTy B(In->getParent(), std::next(In->getIterator())); | |||
3604 | Type *Ty = GEPI.getSourceElementType(); | |||
3605 | NewVal = IsInBounds | |||
3606 | ? B.CreateInBoundsGEP(Ty, In, Index, In->getName() + ".sroa.gep") | |||
3607 | : B.CreateGEP(Ty, In, Index, In->getName() + ".sroa.gep"); | |||
3608 | } | |||
3609 | NewPN->addIncoming(NewVal, B); | |||
3610 | } | |||
3611 | ||||
3612 | Visited.erase(&GEPI); | |||
3613 | GEPI.replaceAllUsesWith(NewPN); | |||
3614 | GEPI.eraseFromParent(); | |||
3615 | Visited.insert(NewPN); | |||
3616 | enqueueUsers(*NewPN); | |||
3617 | ||||
3618 | LLVM_DEBUG(for (Value *In : NewPN->incoming_values())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { for (Value *In : NewPN->incoming_values()) dbgs () << "\n " << *In; dbgs() << "\n " << *NewPN << '\n'; } } while (false) | |||
3619 | dbgs() << "\n " << *In;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { for (Value *In : NewPN->incoming_values()) dbgs () << "\n " << *In; dbgs() << "\n " << *NewPN << '\n'; } } while (false) | |||
3620 | dbgs() << "\n " << *NewPN << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { for (Value *In : NewPN->incoming_values()) dbgs () << "\n " << *In; dbgs() << "\n " << *NewPN << '\n'; } } while (false); | |||
3621 | ||||
3622 | return true; | |||
3623 | } | |||
3624 | ||||
3625 | bool visitGetElementPtrInst(GetElementPtrInst &GEPI) { | |||
3626 | if (isa<SelectInst>(GEPI.getPointerOperand()) && | |||
3627 | foldGEPSelect(GEPI)) | |||
3628 | return true; | |||
3629 | ||||
3630 | if (isa<PHINode>(GEPI.getPointerOperand()) && | |||
3631 | foldGEPPhi(GEPI)) | |||
3632 | return true; | |||
3633 | ||||
3634 | enqueueUsers(GEPI); | |||
3635 | return false; | |||
3636 | } | |||
3637 | ||||
3638 | bool visitPHINode(PHINode &PN) { | |||
3639 | enqueueUsers(PN); | |||
3640 | return false; | |||
3641 | } | |||
3642 | ||||
3643 | bool visitSelectInst(SelectInst &SI) { | |||
3644 | enqueueUsers(SI); | |||
3645 | return false; | |||
3646 | } | |||
3647 | }; | |||
3648 | ||||
3649 | } // end anonymous namespace | |||
3650 | ||||
3651 | /// Strip aggregate type wrapping. | |||
3652 | /// | |||
3653 | /// This removes no-op aggregate types wrapping an underlying type. It will | |||
3654 | /// strip as many layers of types as it can without changing either the type | |||
3655 | /// size or the allocated size. | |||
3656 | static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) { | |||
3657 | if (Ty->isSingleValueType()) | |||
3658 | return Ty; | |||
3659 | ||||
3660 | uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedSize(); | |||
3661 | uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedSize(); | |||
3662 | ||||
3663 | Type *InnerTy; | |||
3664 | if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) { | |||
3665 | InnerTy = ArrTy->getElementType(); | |||
3666 | } else if (StructType *STy = dyn_cast<StructType>(Ty)) { | |||
3667 | const StructLayout *SL = DL.getStructLayout(STy); | |||
3668 | unsigned Index = SL->getElementContainingOffset(0); | |||
3669 | InnerTy = STy->getElementType(Index); | |||
3670 | } else { | |||
3671 | return Ty; | |||
3672 | } | |||
3673 | ||||
3674 | if (AllocSize > DL.getTypeAllocSize(InnerTy).getFixedSize() || | |||
3675 | TypeSize > DL.getTypeSizeInBits(InnerTy).getFixedSize()) | |||
3676 | return Ty; | |||
3677 | ||||
3678 | return stripAggregateTypeWrapping(DL, InnerTy); | |||
3679 | } | |||
3680 | ||||
3681 | /// Try to find a partition of the aggregate type passed in for a given | |||
3682 | /// offset and size. | |||
3683 | /// | |||
3684 | /// This recurses through the aggregate type and tries to compute a subtype | |||
3685 | /// based on the offset and size. When the offset and size span a sub-section | |||
3686 | /// of an array, it will even compute a new array type for that sub-section, | |||
3687 | /// and the same for structs. | |||
3688 | /// | |||
3689 | /// Note that this routine is very strict and tries to find a partition of the | |||
3690 | /// type which produces the *exact* right offset and size. It is not forgiving | |||
3691 | /// when the size or offset cause either end of type-based partition to be off. | |||
3692 | /// Also, this is a best-effort routine. It is reasonable to give up and not | |||
3693 | /// return a type if necessary. | |||
3694 | static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset, | |||
3695 | uint64_t Size) { | |||
3696 | if (Offset == 0 && DL.getTypeAllocSize(Ty).getFixedSize() == Size) | |||
3697 | return stripAggregateTypeWrapping(DL, Ty); | |||
3698 | if (Offset > DL.getTypeAllocSize(Ty).getFixedSize() || | |||
3699 | (DL.getTypeAllocSize(Ty).getFixedSize() - Offset) < Size) | |||
3700 | return nullptr; | |||
3701 | ||||
3702 | if (isa<ArrayType>(Ty) || isa<VectorType>(Ty)) { | |||
3703 | Type *ElementTy; | |||
3704 | uint64_t TyNumElements; | |||
3705 | if (auto *AT = dyn_cast<ArrayType>(Ty)) { | |||
3706 | ElementTy = AT->getElementType(); | |||
3707 | TyNumElements = AT->getNumElements(); | |||
3708 | } else { | |||
3709 | // FIXME: This isn't right for vectors with non-byte-sized or | |||
3710 | // non-power-of-two sized elements. | |||
3711 | auto *VT = cast<FixedVectorType>(Ty); | |||
3712 | ElementTy = VT->getElementType(); | |||
3713 | TyNumElements = VT->getNumElements(); | |||
3714 | } | |||
3715 | uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedSize(); | |||
3716 | uint64_t NumSkippedElements = Offset / ElementSize; | |||
3717 | if (NumSkippedElements >= TyNumElements) | |||
3718 | return nullptr; | |||
3719 | Offset -= NumSkippedElements * ElementSize; | |||
3720 | ||||
3721 | // First check if we need to recurse. | |||
3722 | if (Offset > 0 || Size < ElementSize) { | |||
3723 | // Bail if the partition ends in a different array element. | |||
3724 | if ((Offset + Size) > ElementSize) | |||
3725 | return nullptr; | |||
3726 | // Recurse through the element type trying to peel off offset bytes. | |||
3727 | return getTypePartition(DL, ElementTy, Offset, Size); | |||
3728 | } | |||
3729 | assert(Offset == 0)(static_cast <bool> (Offset == 0) ? void (0) : __assert_fail ("Offset == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3729, __extension__ __PRETTY_FUNCTION__)); | |||
3730 | ||||
3731 | if (Size == ElementSize) | |||
3732 | return stripAggregateTypeWrapping(DL, ElementTy); | |||
3733 | assert(Size > ElementSize)(static_cast <bool> (Size > ElementSize) ? void (0) : __assert_fail ("Size > ElementSize", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3733, __extension__ __PRETTY_FUNCTION__)); | |||
3734 | uint64_t NumElements = Size / ElementSize; | |||
3735 | if (NumElements * ElementSize != Size) | |||
3736 | return nullptr; | |||
3737 | return ArrayType::get(ElementTy, NumElements); | |||
3738 | } | |||
3739 | ||||
3740 | StructType *STy = dyn_cast<StructType>(Ty); | |||
3741 | if (!STy) | |||
3742 | return nullptr; | |||
3743 | ||||
3744 | const StructLayout *SL = DL.getStructLayout(STy); | |||
3745 | if (Offset >= SL->getSizeInBytes()) | |||
3746 | return nullptr; | |||
3747 | uint64_t EndOffset = Offset + Size; | |||
3748 | if (EndOffset > SL->getSizeInBytes()) | |||
3749 | return nullptr; | |||
3750 | ||||
3751 | unsigned Index = SL->getElementContainingOffset(Offset); | |||
3752 | Offset -= SL->getElementOffset(Index); | |||
3753 | ||||
3754 | Type *ElementTy = STy->getElementType(Index); | |||
3755 | uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedSize(); | |||
3756 | if (Offset >= ElementSize) | |||
3757 | return nullptr; // The offset points into alignment padding. | |||
3758 | ||||
3759 | // See if any partition must be contained by the element. | |||
3760 | if (Offset > 0 || Size < ElementSize) { | |||
3761 | if ((Offset + Size) > ElementSize) | |||
3762 | return nullptr; | |||
3763 | return getTypePartition(DL, ElementTy, Offset, Size); | |||
3764 | } | |||
3765 | assert(Offset == 0)(static_cast <bool> (Offset == 0) ? void (0) : __assert_fail ("Offset == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3765, __extension__ __PRETTY_FUNCTION__)); | |||
3766 | ||||
3767 | if (Size == ElementSize) | |||
3768 | return stripAggregateTypeWrapping(DL, ElementTy); | |||
3769 | ||||
3770 | StructType::element_iterator EI = STy->element_begin() + Index, | |||
3771 | EE = STy->element_end(); | |||
3772 | if (EndOffset < SL->getSizeInBytes()) { | |||
3773 | unsigned EndIndex = SL->getElementContainingOffset(EndOffset); | |||
3774 | if (Index == EndIndex) | |||
3775 | return nullptr; // Within a single element and its padding. | |||
3776 | ||||
3777 | // Don't try to form "natural" types if the elements don't line up with the | |||
3778 | // expected size. | |||
3779 | // FIXME: We could potentially recurse down through the last element in the | |||
3780 | // sub-struct to find a natural end point. | |||
3781 | if (SL->getElementOffset(EndIndex) != EndOffset) | |||
3782 | return nullptr; | |||
3783 | ||||
3784 | assert(Index < EndIndex)(static_cast <bool> (Index < EndIndex) ? void (0) : __assert_fail ("Index < EndIndex", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3784, __extension__ __PRETTY_FUNCTION__)); | |||
3785 | EE = STy->element_begin() + EndIndex; | |||
3786 | } | |||
3787 | ||||
3788 | // Try to build up a sub-structure. | |||
3789 | StructType *SubTy = | |||
3790 | StructType::get(STy->getContext(), makeArrayRef(EI, EE), STy->isPacked()); | |||
3791 | const StructLayout *SubSL = DL.getStructLayout(SubTy); | |||
3792 | if (Size != SubSL->getSizeInBytes()) | |||
3793 | return nullptr; // The sub-struct doesn't have quite the size needed. | |||
3794 | ||||
3795 | return SubTy; | |||
3796 | } | |||
3797 | ||||
3798 | /// Pre-split loads and stores to simplify rewriting. | |||
3799 | /// | |||
3800 | /// We want to break up the splittable load+store pairs as much as | |||
3801 | /// possible. This is important to do as a preprocessing step, as once we | |||
3802 | /// start rewriting the accesses to partitions of the alloca we lose the | |||
3803 | /// necessary information to correctly split apart paired loads and stores | |||
3804 | /// which both point into this alloca. The case to consider is something like | |||
3805 | /// the following: | |||
3806 | /// | |||
3807 | /// %a = alloca [12 x i8] | |||
3808 | /// %gep1 = getelementptr [12 x i8]* %a, i32 0, i32 0 | |||
3809 | /// %gep2 = getelementptr [12 x i8]* %a, i32 0, i32 4 | |||
3810 | /// %gep3 = getelementptr [12 x i8]* %a, i32 0, i32 8 | |||
3811 | /// %iptr1 = bitcast i8* %gep1 to i64* | |||
3812 | /// %iptr2 = bitcast i8* %gep2 to i64* | |||
3813 | /// %fptr1 = bitcast i8* %gep1 to float* | |||
3814 | /// %fptr2 = bitcast i8* %gep2 to float* | |||
3815 | /// %fptr3 = bitcast i8* %gep3 to float* | |||
3816 | /// store float 0.0, float* %fptr1 | |||
3817 | /// store float 1.0, float* %fptr2 | |||
3818 | /// %v = load i64* %iptr1 | |||
3819 | /// store i64 %v, i64* %iptr2 | |||
3820 | /// %f1 = load float* %fptr2 | |||
3821 | /// %f2 = load float* %fptr3 | |||
3822 | /// | |||
3823 | /// Here we want to form 3 partitions of the alloca, each 4 bytes large, and | |||
3824 | /// promote everything so we recover the 2 SSA values that should have been | |||
3825 | /// there all along. | |||
3826 | /// | |||
3827 | /// \returns true if any changes are made. | |||
3828 | bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { | |||
3829 | LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Pre-splitting loads and stores\n" ; } } while (false); | |||
3830 | ||||
3831 | // Track the loads and stores which are candidates for pre-splitting here, in | |||
3832 | // the order they first appear during the partition scan. These give stable | |||
3833 | // iteration order and a basis for tracking which loads and stores we | |||
3834 | // actually split. | |||
3835 | SmallVector<LoadInst *, 4> Loads; | |||
3836 | SmallVector<StoreInst *, 4> Stores; | |||
3837 | ||||
3838 | // We need to accumulate the splits required of each load or store where we | |||
3839 | // can find them via a direct lookup. This is important to cross-check loads | |||
3840 | // and stores against each other. We also track the slice so that we can kill | |||
3841 | // all the slices that end up split. | |||
3842 | struct SplitOffsets { | |||
3843 | Slice *S; | |||
3844 | std::vector<uint64_t> Splits; | |||
3845 | }; | |||
3846 | SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap; | |||
3847 | ||||
3848 | // Track loads out of this alloca which cannot, for any reason, be pre-split. | |||
3849 | // This is important as we also cannot pre-split stores of those loads! | |||
3850 | // FIXME: This is all pretty gross. It means that we can be more aggressive | |||
3851 | // in pre-splitting when the load feeding the store happens to come from | |||
3852 | // a separate alloca. Put another way, the effectiveness of SROA would be | |||
3853 | // decreased by a frontend which just concatenated all of its local allocas | |||
3854 | // into one big flat alloca. But defeating such patterns is exactly the job | |||
3855 | // SROA is tasked with! Sadly, to not have this discrepancy we would have | |||
3856 | // change store pre-splitting to actually force pre-splitting of the load | |||
3857 | // that feeds it *and all stores*. That makes pre-splitting much harder, but | |||
3858 | // maybe it would make it more principled? | |||
3859 | SmallPtrSet<LoadInst *, 8> UnsplittableLoads; | |||
3860 | ||||
3861 | LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Searching for candidate loads and stores\n" ; } } while (false); | |||
3862 | for (auto &P : AS.partitions()) { | |||
3863 | for (Slice &S : P) { | |||
3864 | Instruction *I = cast<Instruction>(S.getUse()->getUser()); | |||
3865 | if (!S.isSplittable() || S.endOffset() <= P.endOffset()) { | |||
3866 | // If this is a load we have to track that it can't participate in any | |||
3867 | // pre-splitting. If this is a store of a load we have to track that | |||
3868 | // that load also can't participate in any pre-splitting. | |||
3869 | if (auto *LI = dyn_cast<LoadInst>(I)) | |||
3870 | UnsplittableLoads.insert(LI); | |||
3871 | else if (auto *SI = dyn_cast<StoreInst>(I)) | |||
3872 | if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand())) | |||
3873 | UnsplittableLoads.insert(LI); | |||
3874 | continue; | |||
3875 | } | |||
3876 | assert(P.endOffset() > S.beginOffset() &&(static_cast <bool> (P.endOffset() > S.beginOffset() && "Empty or backwards partition!") ? void (0) : __assert_fail ("P.endOffset() > S.beginOffset() && \"Empty or backwards partition!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3877, __extension__ __PRETTY_FUNCTION__)) | |||
3877 | "Empty or backwards partition!")(static_cast <bool> (P.endOffset() > S.beginOffset() && "Empty or backwards partition!") ? void (0) : __assert_fail ("P.endOffset() > S.beginOffset() && \"Empty or backwards partition!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3877, __extension__ __PRETTY_FUNCTION__)); | |||
3878 | ||||
3879 | // Determine if this is a pre-splittable slice. | |||
3880 | if (auto *LI = dyn_cast<LoadInst>(I)) { | |||
3881 | assert(!LI->isVolatile() && "Cannot split volatile loads!")(static_cast <bool> (!LI->isVolatile() && "Cannot split volatile loads!" ) ? void (0) : __assert_fail ("!LI->isVolatile() && \"Cannot split volatile loads!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3881, __extension__ __PRETTY_FUNCTION__)); | |||
3882 | ||||
3883 | // The load must be used exclusively to store into other pointers for | |||
3884 | // us to be able to arbitrarily pre-split it. The stores must also be | |||
3885 | // simple to avoid changing semantics. | |||
3886 | auto IsLoadSimplyStored = [](LoadInst *LI) { | |||
3887 | for (User *LU : LI->users()) { | |||
3888 | auto *SI = dyn_cast<StoreInst>(LU); | |||
3889 | if (!SI || !SI->isSimple()) | |||
3890 | return false; | |||
3891 | } | |||
3892 | return true; | |||
3893 | }; | |||
3894 | if (!IsLoadSimplyStored(LI)) { | |||
3895 | UnsplittableLoads.insert(LI); | |||
3896 | continue; | |||
3897 | } | |||
3898 | ||||
3899 | Loads.push_back(LI); | |||
3900 | } else if (auto *SI = dyn_cast<StoreInst>(I)) { | |||
3901 | if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex())) | |||
3902 | // Skip stores *of* pointers. FIXME: This shouldn't even be possible! | |||
3903 | continue; | |||
3904 | auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand()); | |||
3905 | if (!StoredLoad || !StoredLoad->isSimple()) | |||
3906 | continue; | |||
3907 | assert(!SI->isVolatile() && "Cannot split volatile stores!")(static_cast <bool> (!SI->isVolatile() && "Cannot split volatile stores!" ) ? void (0) : __assert_fail ("!SI->isVolatile() && \"Cannot split volatile stores!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3907, __extension__ __PRETTY_FUNCTION__)); | |||
3908 | ||||
3909 | Stores.push_back(SI); | |||
3910 | } else { | |||
3911 | // Other uses cannot be pre-split. | |||
3912 | continue; | |||
3913 | } | |||
3914 | ||||
3915 | // Record the initial split. | |||
3916 | LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Candidate: " << *I << "\n"; } } while (false); | |||
3917 | auto &Offsets = SplitOffsetsMap[I]; | |||
3918 | assert(Offsets.Splits.empty() &&(static_cast <bool> (Offsets.Splits.empty() && "Should not have splits the first time we see an instruction!" ) ? void (0) : __assert_fail ("Offsets.Splits.empty() && \"Should not have splits the first time we see an instruction!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3919, __extension__ __PRETTY_FUNCTION__)) | |||
3919 | "Should not have splits the first time we see an instruction!")(static_cast <bool> (Offsets.Splits.empty() && "Should not have splits the first time we see an instruction!" ) ? void (0) : __assert_fail ("Offsets.Splits.empty() && \"Should not have splits the first time we see an instruction!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3919, __extension__ __PRETTY_FUNCTION__)); | |||
3920 | Offsets.S = &S; | |||
3921 | Offsets.Splits.push_back(P.endOffset() - S.beginOffset()); | |||
3922 | } | |||
3923 | ||||
3924 | // Now scan the already split slices, and add a split for any of them which | |||
3925 | // we're going to pre-split. | |||
3926 | for (Slice *S : P.splitSliceTails()) { | |||
3927 | auto SplitOffsetsMapI = | |||
3928 | SplitOffsetsMap.find(cast<Instruction>(S->getUse()->getUser())); | |||
3929 | if (SplitOffsetsMapI == SplitOffsetsMap.end()) | |||
3930 | continue; | |||
3931 | auto &Offsets = SplitOffsetsMapI->second; | |||
3932 | ||||
3933 | assert(Offsets.S == S && "Found a mismatched slice!")(static_cast <bool> (Offsets.S == S && "Found a mismatched slice!" ) ? void (0) : __assert_fail ("Offsets.S == S && \"Found a mismatched slice!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3933, __extension__ __PRETTY_FUNCTION__)); | |||
3934 | assert(!Offsets.Splits.empty() &&(static_cast <bool> (!Offsets.Splits.empty() && "Cannot have an empty set of splits on the second partition!" ) ? void (0) : __assert_fail ("!Offsets.Splits.empty() && \"Cannot have an empty set of splits on the second partition!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3935, __extension__ __PRETTY_FUNCTION__)) | |||
3935 | "Cannot have an empty set of splits on the second partition!")(static_cast <bool> (!Offsets.Splits.empty() && "Cannot have an empty set of splits on the second partition!" ) ? void (0) : __assert_fail ("!Offsets.Splits.empty() && \"Cannot have an empty set of splits on the second partition!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3935, __extension__ __PRETTY_FUNCTION__)); | |||
3936 | assert(Offsets.Splits.back() ==(static_cast <bool> (Offsets.Splits.back() == P.beginOffset () - Offsets.S->beginOffset() && "Previous split does not end where this one begins!" ) ? void (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3938, __extension__ __PRETTY_FUNCTION__)) | |||
3937 | P.beginOffset() - Offsets.S->beginOffset() &&(static_cast <bool> (Offsets.Splits.back() == P.beginOffset () - Offsets.S->beginOffset() && "Previous split does not end where this one begins!" ) ? void (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3938, __extension__ __PRETTY_FUNCTION__)) | |||
3938 | "Previous split does not end where this one begins!")(static_cast <bool> (Offsets.Splits.back() == P.beginOffset () - Offsets.S->beginOffset() && "Previous split does not end where this one begins!" ) ? void (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 3938, __extension__ __PRETTY_FUNCTION__)); | |||
3939 | ||||
3940 | // Record each split. The last partition's end isn't needed as the size | |||
3941 | // of the slice dictates that. | |||
3942 | if (S->endOffset() > P.endOffset()) | |||
3943 | Offsets.Splits.push_back(P.endOffset() - Offsets.S->beginOffset()); | |||
3944 | } | |||
3945 | } | |||
3946 | ||||
3947 | // We may have split loads where some of their stores are split stores. For | |||
3948 | // such loads and stores, we can only pre-split them if their splits exactly | |||
3949 | // match relative to their starting offset. We have to verify this prior to | |||
3950 | // any rewriting. | |||
3951 | llvm::erase_if(Stores, [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) { | |||
3952 | // Lookup the load we are storing in our map of split | |||
3953 | // offsets. | |||
3954 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | |||
3955 | // If it was completely unsplittable, then we're done, | |||
3956 | // and this store can't be pre-split. | |||
3957 | if (UnsplittableLoads.count(LI)) | |||
3958 | return true; | |||
3959 | ||||
3960 | auto LoadOffsetsI = SplitOffsetsMap.find(LI); | |||
3961 | if (LoadOffsetsI == SplitOffsetsMap.end()) | |||
3962 | return false; // Unrelated loads are definitely safe. | |||
3963 | auto &LoadOffsets = LoadOffsetsI->second; | |||
3964 | ||||
3965 | // Now lookup the store's offsets. | |||
3966 | auto &StoreOffsets = SplitOffsetsMap[SI]; | |||
3967 | ||||
3968 | // If the relative offsets of each split in the load and | |||
3969 | // store match exactly, then we can split them and we | |||
3970 | // don't need to remove them here. | |||
3971 | if (LoadOffsets.Splits == StoreOffsets.Splits) | |||
3972 | return false; | |||
3973 | ||||
3974 | LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | |||
3975 | << " " << *LI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | |||
3976 | << " " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false); | |||
3977 | ||||
3978 | // We've found a store and load that we need to split | |||
3979 | // with mismatched relative splits. Just give up on them | |||
3980 | // and remove both instructions from our list of | |||
3981 | // candidates. | |||
3982 | UnsplittableLoads.insert(LI); | |||
3983 | return true; | |||
3984 | }); | |||
3985 | // Now we have to go *back* through all the stores, because a later store may | |||
3986 | // have caused an earlier store's load to become unsplittable and if it is | |||
3987 | // unsplittable for the later store, then we can't rely on it being split in | |||
3988 | // the earlier store either. | |||
3989 | llvm::erase_if(Stores, [&UnsplittableLoads](StoreInst *SI) { | |||
3990 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | |||
3991 | return UnsplittableLoads.count(LI); | |||
3992 | }); | |||
3993 | // Once we've established all the loads that can't be split for some reason, | |||
3994 | // filter any that made it into our list out. | |||
3995 | llvm::erase_if(Loads, [&UnsplittableLoads](LoadInst *LI) { | |||
3996 | return UnsplittableLoads.count(LI); | |||
3997 | }); | |||
3998 | ||||
3999 | // If no loads or stores are left, there is no pre-splitting to be done for | |||
4000 | // this alloca. | |||
4001 | if (Loads.empty() && Stores.empty()) | |||
4002 | return false; | |||
4003 | ||||
4004 | // From here on, we can't fail and will be building new accesses, so rig up | |||
4005 | // an IR builder. | |||
4006 | IRBuilderTy IRB(&AI); | |||
4007 | ||||
4008 | // Collect the new slices which we will merge into the alloca slices. | |||
4009 | SmallVector<Slice, 4> NewSlices; | |||
4010 | ||||
4011 | // Track any allocas we end up splitting loads and stores for so we iterate | |||
4012 | // on them. | |||
4013 | SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas; | |||
4014 | ||||
4015 | // At this point, we have collected all of the loads and stores we can | |||
4016 | // pre-split, and the specific splits needed for them. We actually do the | |||
4017 | // splitting in a specific order in order to handle when one of the loads in | |||
4018 | // the value operand to one of the stores. | |||
4019 | // | |||
4020 | // First, we rewrite all of the split loads, and just accumulate each split | |||
4021 | // load in a parallel structure. We also build the slices for them and append | |||
4022 | // them to the alloca slices. | |||
4023 | SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap; | |||
4024 | std::vector<LoadInst *> SplitLoads; | |||
4025 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||
4026 | for (LoadInst *LI : Loads) { | |||
4027 | SplitLoads.clear(); | |||
4028 | ||||
4029 | IntegerType *Ty = cast<IntegerType>(LI->getType()); | |||
4030 | assert(Ty->getBitWidth() % 8 == 0)(static_cast <bool> (Ty->getBitWidth() % 8 == 0) ? void (0) : __assert_fail ("Ty->getBitWidth() % 8 == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4030, __extension__ __PRETTY_FUNCTION__)); | |||
4031 | uint64_t LoadSize = Ty->getBitWidth() / 8; | |||
4032 | assert(LoadSize > 0 && "Cannot have a zero-sized integer load!")(static_cast <bool> (LoadSize > 0 && "Cannot have a zero-sized integer load!" ) ? void (0) : __assert_fail ("LoadSize > 0 && \"Cannot have a zero-sized integer load!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4032, __extension__ __PRETTY_FUNCTION__)); | |||
4033 | ||||
4034 | auto &Offsets = SplitOffsetsMap[LI]; | |||
4035 | assert(LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&(static_cast <bool> (LoadSize == Offsets.S->endOffset () - Offsets.S->beginOffset() && "Slice size should always match load size exactly!" ) ? void (0) : __assert_fail ("LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4036, __extension__ __PRETTY_FUNCTION__)) | |||
4036 | "Slice size should always match load size exactly!")(static_cast <bool> (LoadSize == Offsets.S->endOffset () - Offsets.S->beginOffset() && "Slice size should always match load size exactly!" ) ? void (0) : __assert_fail ("LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4036, __extension__ __PRETTY_FUNCTION__)); | |||
4037 | uint64_t BaseOffset = Offsets.S->beginOffset(); | |||
4038 | assert(BaseOffset + LoadSize > BaseOffset &&(static_cast <bool> (BaseOffset + LoadSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? void (0) : __assert_fail ("BaseOffset + LoadSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4039, __extension__ __PRETTY_FUNCTION__)) | |||
4039 | "Cannot represent alloca access size using 64-bit integers!")(static_cast <bool> (BaseOffset + LoadSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? void (0) : __assert_fail ("BaseOffset + LoadSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4039, __extension__ __PRETTY_FUNCTION__)); | |||
4040 | ||||
4041 | Instruction *BasePtr = cast<Instruction>(LI->getPointerOperand()); | |||
4042 | IRB.SetInsertPoint(LI); | |||
4043 | ||||
4044 | LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting load: " << *LI << "\n"; } } while (false); | |||
4045 | ||||
4046 | uint64_t PartOffset = 0, PartSize = Offsets.Splits.front(); | |||
4047 | int Idx = 0, Size = Offsets.Splits.size(); | |||
4048 | for (;;) { | |||
4049 | auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8); | |||
4050 | auto AS = LI->getPointerAddressSpace(); | |||
4051 | auto *PartPtrTy = PartTy->getPointerTo(AS); | |||
4052 | LoadInst *PLoad = IRB.CreateAlignedLoad( | |||
4053 | PartTy, | |||
4054 | getAdjustedPtr(IRB, DL, BasePtr, | |||
4055 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | |||
4056 | PartPtrTy, BasePtr->getName() + "."), | |||
4057 | getAdjustedAlignment(LI, PartOffset), | |||
4058 | /*IsVolatile*/ false, LI->getName()); | |||
4059 | PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access, | |||
4060 | LLVMContext::MD_access_group}); | |||
4061 | ||||
4062 | // Append this load onto the list of split loads so we can find it later | |||
4063 | // to rewrite the stores. | |||
4064 | SplitLoads.push_back(PLoad); | |||
4065 | ||||
4066 | // Now build a new slice for the alloca. | |||
4067 | NewSlices.push_back( | |||
4068 | Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, | |||
4069 | &PLoad->getOperandUse(PLoad->getPointerOperandIndex()), | |||
4070 | /*IsSplittable*/ false)); | |||
4071 | LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false) | |||
4072 | << ", " << NewSlices.back().endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false) | |||
4073 | << "): " << *PLoad << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false); | |||
4074 | ||||
4075 | // See if we've handled all the splits. | |||
4076 | if (Idx >= Size) | |||
4077 | break; | |||
4078 | ||||
4079 | // Setup the next partition. | |||
4080 | PartOffset = Offsets.Splits[Idx]; | |||
4081 | ++Idx; | |||
4082 | PartSize = (Idx < Size ? Offsets.Splits[Idx] : LoadSize) - PartOffset; | |||
4083 | } | |||
4084 | ||||
4085 | // Now that we have the split loads, do the slow walk over all uses of the | |||
4086 | // load and rewrite them as split stores, or save the split loads to use | |||
4087 | // below if the store is going to be split there anyways. | |||
4088 | bool DeferredStores = false; | |||
4089 | for (User *LU : LI->users()) { | |||
4090 | StoreInst *SI = cast<StoreInst>(LU); | |||
4091 | if (!Stores.empty() && SplitOffsetsMap.count(SI)) { | |||
4092 | DeferredStores = true; | |||
4093 | LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Deferred splitting of store: " << *SI << "\n"; } } while (false) | |||
4094 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Deferred splitting of store: " << *SI << "\n"; } } while (false); | |||
4095 | continue; | |||
4096 | } | |||
4097 | ||||
4098 | Value *StoreBasePtr = SI->getPointerOperand(); | |||
4099 | IRB.SetInsertPoint(SI); | |||
4100 | ||||
4101 | LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting store of load: " << *SI << "\n"; } } while (false); | |||
4102 | ||||
4103 | for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) { | |||
4104 | LoadInst *PLoad = SplitLoads[Idx]; | |||
4105 | uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1]; | |||
4106 | auto *PartPtrTy = | |||
4107 | PLoad->getType()->getPointerTo(SI->getPointerAddressSpace()); | |||
4108 | ||||
4109 | auto AS = SI->getPointerAddressSpace(); | |||
4110 | StoreInst *PStore = IRB.CreateAlignedStore( | |||
4111 | PLoad, | |||
4112 | getAdjustedPtr(IRB, DL, StoreBasePtr, | |||
4113 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | |||
4114 | PartPtrTy, StoreBasePtr->getName() + "."), | |||
4115 | getAdjustedAlignment(SI, PartOffset), | |||
4116 | /*IsVolatile*/ false); | |||
4117 | PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
4118 | LLVMContext::MD_access_group}); | |||
4119 | LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " +" << PartOffset << ":" << *PStore << "\n"; } } while (false); | |||
4120 | } | |||
4121 | ||||
4122 | // We want to immediately iterate on any allocas impacted by splitting | |||
4123 | // this store, and we have to track any promotable alloca (indicated by | |||
4124 | // a direct store) as needing to be resplit because it is no longer | |||
4125 | // promotable. | |||
4126 | if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(StoreBasePtr)) { | |||
4127 | ResplitPromotableAllocas.insert(OtherAI); | |||
4128 | Worklist.insert(OtherAI); | |||
4129 | } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>( | |||
4130 | StoreBasePtr->stripInBoundsOffsets())) { | |||
4131 | Worklist.insert(OtherAI); | |||
4132 | } | |||
4133 | ||||
4134 | // Mark the original store as dead. | |||
4135 | DeadInsts.push_back(SI); | |||
4136 | } | |||
4137 | ||||
4138 | // Save the split loads if there are deferred stores among the users. | |||
4139 | if (DeferredStores) | |||
4140 | SplitLoadsMap.insert(std::make_pair(LI, std::move(SplitLoads))); | |||
4141 | ||||
4142 | // Mark the original load as dead and kill the original slice. | |||
4143 | DeadInsts.push_back(LI); | |||
4144 | Offsets.S->kill(); | |||
4145 | } | |||
4146 | ||||
4147 | // Second, we rewrite all of the split stores. At this point, we know that | |||
4148 | // all loads from this alloca have been split already. For stores of such | |||
4149 | // loads, we can simply look up the pre-existing split loads. For stores of | |||
4150 | // other loads, we split those loads first and then write split stores of | |||
4151 | // them. | |||
4152 | for (StoreInst *SI : Stores) { | |||
4153 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | |||
4154 | IntegerType *Ty = cast<IntegerType>(LI->getType()); | |||
4155 | assert(Ty->getBitWidth() % 8 == 0)(static_cast <bool> (Ty->getBitWidth() % 8 == 0) ? void (0) : __assert_fail ("Ty->getBitWidth() % 8 == 0", "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4155, __extension__ __PRETTY_FUNCTION__)); | |||
4156 | uint64_t StoreSize = Ty->getBitWidth() / 8; | |||
4157 | assert(StoreSize > 0 && "Cannot have a zero-sized integer store!")(static_cast <bool> (StoreSize > 0 && "Cannot have a zero-sized integer store!" ) ? void (0) : __assert_fail ("StoreSize > 0 && \"Cannot have a zero-sized integer store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4157, __extension__ __PRETTY_FUNCTION__)); | |||
4158 | ||||
4159 | auto &Offsets = SplitOffsetsMap[SI]; | |||
4160 | assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&(static_cast <bool> (StoreSize == Offsets.S->endOffset () - Offsets.S->beginOffset() && "Slice size should always match load size exactly!" ) ? void (0) : __assert_fail ("StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4161, __extension__ __PRETTY_FUNCTION__)) | |||
4161 | "Slice size should always match load size exactly!")(static_cast <bool> (StoreSize == Offsets.S->endOffset () - Offsets.S->beginOffset() && "Slice size should always match load size exactly!" ) ? void (0) : __assert_fail ("StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4161, __extension__ __PRETTY_FUNCTION__)); | |||
4162 | uint64_t BaseOffset = Offsets.S->beginOffset(); | |||
4163 | assert(BaseOffset + StoreSize > BaseOffset &&(static_cast <bool> (BaseOffset + StoreSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? void (0) : __assert_fail ("BaseOffset + StoreSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4164, __extension__ __PRETTY_FUNCTION__)) | |||
4164 | "Cannot represent alloca access size using 64-bit integers!")(static_cast <bool> (BaseOffset + StoreSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? void (0) : __assert_fail ("BaseOffset + StoreSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4164, __extension__ __PRETTY_FUNCTION__)); | |||
4165 | ||||
4166 | Value *LoadBasePtr = LI->getPointerOperand(); | |||
4167 | Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand()); | |||
4168 | ||||
4169 | LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting store: " << *SI << "\n"; } } while (false); | |||
4170 | ||||
4171 | // Check whether we have an already split load. | |||
4172 | auto SplitLoadsMapI = SplitLoadsMap.find(LI); | |||
4173 | std::vector<LoadInst *> *SplitLoads = nullptr; | |||
4174 | if (SplitLoadsMapI != SplitLoadsMap.end()) { | |||
4175 | SplitLoads = &SplitLoadsMapI->second; | |||
4176 | assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&(static_cast <bool> (SplitLoads->size() == Offsets.Splits .size() + 1 && "Too few split loads for the number of splits in the store!" ) ? void (0) : __assert_fail ("SplitLoads->size() == Offsets.Splits.size() + 1 && \"Too few split loads for the number of splits in the store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4177, __extension__ __PRETTY_FUNCTION__)) | |||
4177 | "Too few split loads for the number of splits in the store!")(static_cast <bool> (SplitLoads->size() == Offsets.Splits .size() + 1 && "Too few split loads for the number of splits in the store!" ) ? void (0) : __assert_fail ("SplitLoads->size() == Offsets.Splits.size() + 1 && \"Too few split loads for the number of splits in the store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4177, __extension__ __PRETTY_FUNCTION__)); | |||
4178 | } else { | |||
4179 | LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " of load: " << *LI << "\n"; } } while (false); | |||
4180 | } | |||
4181 | ||||
4182 | uint64_t PartOffset = 0, PartSize = Offsets.Splits.front(); | |||
4183 | int Idx = 0, Size = Offsets.Splits.size(); | |||
4184 | for (;;) { | |||
4185 | auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8); | |||
4186 | auto *LoadPartPtrTy = PartTy->getPointerTo(LI->getPointerAddressSpace()); | |||
4187 | auto *StorePartPtrTy = PartTy->getPointerTo(SI->getPointerAddressSpace()); | |||
4188 | ||||
4189 | // Either lookup a split load or create one. | |||
4190 | LoadInst *PLoad; | |||
4191 | if (SplitLoads) { | |||
4192 | PLoad = (*SplitLoads)[Idx]; | |||
4193 | } else { | |||
4194 | IRB.SetInsertPoint(LI); | |||
4195 | auto AS = LI->getPointerAddressSpace(); | |||
4196 | PLoad = IRB.CreateAlignedLoad( | |||
4197 | PartTy, | |||
4198 | getAdjustedPtr(IRB, DL, LoadBasePtr, | |||
4199 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | |||
4200 | LoadPartPtrTy, LoadBasePtr->getName() + "."), | |||
4201 | getAdjustedAlignment(LI, PartOffset), | |||
4202 | /*IsVolatile*/ false, LI->getName()); | |||
4203 | PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access, | |||
4204 | LLVMContext::MD_access_group}); | |||
4205 | } | |||
4206 | ||||
4207 | // And store this partition. | |||
4208 | IRB.SetInsertPoint(SI); | |||
4209 | auto AS = SI->getPointerAddressSpace(); | |||
4210 | StoreInst *PStore = IRB.CreateAlignedStore( | |||
4211 | PLoad, | |||
4212 | getAdjustedPtr(IRB, DL, StoreBasePtr, | |||
4213 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | |||
4214 | StorePartPtrTy, StoreBasePtr->getName() + "."), | |||
4215 | getAdjustedAlignment(SI, PartOffset), | |||
4216 | /*IsVolatile*/ false); | |||
4217 | PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access, | |||
4218 | LLVMContext::MD_access_group}); | |||
4219 | ||||
4220 | // Now build a new slice for the alloca. | |||
4221 | NewSlices.push_back( | |||
4222 | Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, | |||
4223 | &PStore->getOperandUse(PStore->getPointerOperandIndex()), | |||
4224 | /*IsSplittable*/ false)); | |||
4225 | LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false) | |||
4226 | << ", " << NewSlices.back().endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false) | |||
4227 | << "): " << *PStore << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false); | |||
4228 | if (!SplitLoads) { | |||
4229 | LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " of split load: " << * PLoad << "\n"; } } while (false); | |||
4230 | } | |||
4231 | ||||
4232 | // See if we've finished all the splits. | |||
4233 | if (Idx >= Size) | |||
4234 | break; | |||
4235 | ||||
4236 | // Setup the next partition. | |||
4237 | PartOffset = Offsets.Splits[Idx]; | |||
4238 | ++Idx; | |||
4239 | PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset; | |||
4240 | } | |||
4241 | ||||
4242 | // We want to immediately iterate on any allocas impacted by splitting | |||
4243 | // this load, which is only relevant if it isn't a load of this alloca and | |||
4244 | // thus we didn't already split the loads above. We also have to keep track | |||
4245 | // of any promotable allocas we split loads on as they can no longer be | |||
4246 | // promoted. | |||
4247 | if (!SplitLoads) { | |||
4248 | if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(LoadBasePtr)) { | |||
4249 | assert(OtherAI != &AI && "We can't re-split our own alloca!")(static_cast <bool> (OtherAI != &AI && "We can't re-split our own alloca!" ) ? void (0) : __assert_fail ("OtherAI != &AI && \"We can't re-split our own alloca!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4249, __extension__ __PRETTY_FUNCTION__)); | |||
4250 | ResplitPromotableAllocas.insert(OtherAI); | |||
4251 | Worklist.insert(OtherAI); | |||
4252 | } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>( | |||
4253 | LoadBasePtr->stripInBoundsOffsets())) { | |||
4254 | assert(OtherAI != &AI && "We can't re-split our own alloca!")(static_cast <bool> (OtherAI != &AI && "We can't re-split our own alloca!" ) ? void (0) : __assert_fail ("OtherAI != &AI && \"We can't re-split our own alloca!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4254, __extension__ __PRETTY_FUNCTION__)); | |||
4255 | Worklist.insert(OtherAI); | |||
4256 | } | |||
4257 | } | |||
4258 | ||||
4259 | // Mark the original store as dead now that we've split it up and kill its | |||
4260 | // slice. Note that we leave the original load in place unless this store | |||
4261 | // was its only use. It may in turn be split up if it is an alloca load | |||
4262 | // for some other alloca, but it may be a normal load. This may introduce | |||
4263 | // redundant loads, but where those can be merged the rest of the optimizer | |||
4264 | // should handle the merging, and this uncovers SSA splits which is more | |||
4265 | // important. In practice, the original loads will almost always be fully | |||
4266 | // split and removed eventually, and the splits will be merged by any | |||
4267 | // trivial CSE, including instcombine. | |||
4268 | if (LI->hasOneUse()) { | |||
4269 | assert(*LI->user_begin() == SI && "Single use isn't this store!")(static_cast <bool> (*LI->user_begin() == SI && "Single use isn't this store!") ? void (0) : __assert_fail ( "*LI->user_begin() == SI && \"Single use isn't this store!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4269, __extension__ __PRETTY_FUNCTION__)); | |||
4270 | DeadInsts.push_back(LI); | |||
4271 | } | |||
4272 | DeadInsts.push_back(SI); | |||
4273 | Offsets.S->kill(); | |||
4274 | } | |||
4275 | ||||
4276 | // Remove the killed slices that have ben pre-split. | |||
4277 | llvm::erase_if(AS, [](const Slice &S) { return S.isDead(); }); | |||
4278 | ||||
4279 | // Insert our new slices. This will sort and merge them into the sorted | |||
4280 | // sequence. | |||
4281 | AS.insert(NewSlices); | |||
4282 | ||||
4283 | LLVM_DEBUG(dbgs() << " Pre-split slices:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Pre-split slices:\n"; } } while (false); | |||
4284 | #ifndef NDEBUG | |||
4285 | for (auto I = AS.begin(), E = AS.end(); I != E; ++I) | |||
4286 | LLVM_DEBUG(AS.print(dbgs(), I, " "))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.print(dbgs(), I, " "); } } while (false); | |||
4287 | #endif | |||
4288 | ||||
4289 | // Finally, don't try to promote any allocas that new require re-splitting. | |||
4290 | // They have already been added to the worklist above. | |||
4291 | llvm::erase_if(PromotableAllocas, [&](AllocaInst *AI) { | |||
4292 | return ResplitPromotableAllocas.count(AI); | |||
4293 | }); | |||
4294 | ||||
4295 | return true; | |||
4296 | } | |||
4297 | ||||
4298 | /// Rewrite an alloca partition's users. | |||
4299 | /// | |||
4300 | /// This routine drives both of the rewriting goals of the SROA pass. It tries | |||
4301 | /// to rewrite uses of an alloca partition to be conducive for SSA value | |||
4302 | /// promotion. If the partition needs a new, more refined alloca, this will | |||
4303 | /// build that new alloca, preserving as much type information as possible, and | |||
4304 | /// rewrite the uses of the old alloca to point at the new one and have the | |||
4305 | /// appropriate new offsets. It also evaluates how successful the rewrite was | |||
4306 | /// at enabling promotion and if it was successful queues the alloca to be | |||
4307 | /// promoted. | |||
4308 | AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, | |||
4309 | Partition &P) { | |||
4310 | // Try to compute a friendly type for this partition of the alloca. This | |||
4311 | // won't always succeed, in which case we fall back to a legal integer type | |||
4312 | // or an i8 array of an appropriate size. | |||
4313 | Type *SliceTy = nullptr; | |||
4314 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||
4315 | std::pair<Type *, IntegerType *> CommonUseTy = | |||
4316 | findCommonType(P.begin(), P.end(), P.endOffset()); | |||
4317 | // Do all uses operate on the same type? | |||
4318 | if (CommonUseTy.first) | |||
4319 | if (DL.getTypeAllocSize(CommonUseTy.first).getFixedSize() >= P.size()) | |||
4320 | SliceTy = CommonUseTy.first; | |||
4321 | // If not, can we find an appropriate subtype in the original allocated type? | |||
4322 | if (!SliceTy) | |||
4323 | if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(), | |||
4324 | P.beginOffset(), P.size())) | |||
4325 | SliceTy = TypePartitionTy; | |||
4326 | // If still not, can we use the largest bitwidth integer type used? | |||
4327 | if (!SliceTy && CommonUseTy.second) | |||
4328 | if (DL.getTypeAllocSize(CommonUseTy.second).getFixedSize() >= P.size()) | |||
4329 | SliceTy = CommonUseTy.second; | |||
4330 | if ((!SliceTy || (SliceTy->isArrayTy() && | |||
4331 | SliceTy->getArrayElementType()->isIntegerTy())) && | |||
4332 | DL.isLegalInteger(P.size() * 8)) | |||
4333 | SliceTy = Type::getIntNTy(*C, P.size() * 8); | |||
4334 | if (!SliceTy) | |||
4335 | SliceTy = ArrayType::get(Type::getInt8Ty(*C), P.size()); | |||
4336 | assert(DL.getTypeAllocSize(SliceTy).getFixedSize() >= P.size())(static_cast <bool> (DL.getTypeAllocSize(SliceTy).getFixedSize () >= P.size()) ? void (0) : __assert_fail ("DL.getTypeAllocSize(SliceTy).getFixedSize() >= P.size()" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4336, __extension__ __PRETTY_FUNCTION__)); | |||
4337 | ||||
4338 | bool IsIntegerPromotable = isIntegerWideningViable(P, SliceTy, DL); | |||
4339 | ||||
4340 | VectorType *VecTy = | |||
4341 | IsIntegerPromotable ? nullptr : isVectorPromotionViable(P, DL); | |||
4342 | if (VecTy) | |||
4343 | SliceTy = VecTy; | |||
4344 | ||||
4345 | // Check for the case where we're going to rewrite to a new alloca of the | |||
4346 | // exact same type as the original, and with the same access offsets. In that | |||
4347 | // case, re-use the existing alloca, but still run through the rewriter to | |||
4348 | // perform phi and select speculation. | |||
4349 | // P.beginOffset() can be non-zero even with the same type in a case with | |||
4350 | // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll). | |||
4351 | AllocaInst *NewAI; | |||
4352 | if (SliceTy == AI.getAllocatedType() && P.beginOffset() == 0) { | |||
4353 | NewAI = &AI; | |||
4354 | // FIXME: We should be able to bail at this point with "nothing changed". | |||
4355 | // FIXME: We might want to defer PHI speculation until after here. | |||
4356 | // FIXME: return nullptr; | |||
4357 | } else { | |||
4358 | // Make sure the alignment is compatible with P.beginOffset(). | |||
4359 | const Align Alignment = commonAlignment(AI.getAlign(), P.beginOffset()); | |||
4360 | // If we will get at least this much alignment from the type alone, leave | |||
4361 | // the alloca's alignment unconstrained. | |||
4362 | const bool IsUnconstrained = Alignment <= DL.getABITypeAlign(SliceTy); | |||
4363 | NewAI = new AllocaInst( | |||
4364 | SliceTy, AI.getType()->getAddressSpace(), nullptr, | |||
4365 | IsUnconstrained ? DL.getPrefTypeAlign(SliceTy) : Alignment, | |||
4366 | AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()), &AI); | |||
4367 | // Copy the old AI debug location over to the new one. | |||
4368 | NewAI->setDebugLoc(AI.getDebugLoc()); | |||
4369 | ++NumNewAllocas; | |||
4370 | } | |||
4371 | ||||
4372 | LLVM_DEBUG(dbgs() << "Rewriting alloca partition "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false) | |||
4373 | << "[" << P.beginOffset() << "," << P.endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false) | |||
4374 | << ") to: " << *NewAI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false); | |||
4375 | ||||
4376 | // Track the high watermark on the worklist as it is only relevant for | |||
4377 | // promoted allocas. We will reset it to this point if the alloca is not in | |||
4378 | // fact scheduled for promotion. | |||
4379 | unsigned PPWOldSize = PostPromotionWorklist.size(); | |||
4380 | unsigned NumUses = 0; | |||
4381 | SmallSetVector<PHINode *, 8> PHIUsers; | |||
4382 | SmallSetVector<SelectInst *, 8> SelectUsers; | |||
4383 | ||||
4384 | AllocaSliceRewriter Rewriter(DL, AS, *this, AI, *NewAI, P.beginOffset(), | |||
4385 | P.endOffset(), IsIntegerPromotable, VecTy, | |||
4386 | PHIUsers, SelectUsers); | |||
4387 | bool Promotable = true; | |||
4388 | for (Slice *S : P.splitSliceTails()) { | |||
4389 | Promotable &= Rewriter.visit(S); | |||
4390 | ++NumUses; | |||
4391 | } | |||
4392 | for (Slice &S : P) { | |||
4393 | Promotable &= Rewriter.visit(&S); | |||
4394 | ++NumUses; | |||
4395 | } | |||
4396 | ||||
4397 | NumAllocaPartitionUses += NumUses; | |||
4398 | MaxUsesPerAllocaPartition.updateMax(NumUses); | |||
4399 | ||||
4400 | // Now that we've processed all the slices in the new partition, check if any | |||
4401 | // PHIs or Selects would block promotion. | |||
4402 | for (PHINode *PHI : PHIUsers) | |||
4403 | if (!isSafePHIToSpeculate(*PHI)) { | |||
4404 | Promotable = false; | |||
4405 | PHIUsers.clear(); | |||
4406 | SelectUsers.clear(); | |||
4407 | break; | |||
4408 | } | |||
4409 | ||||
4410 | for (SelectInst *Sel : SelectUsers) | |||
4411 | if (!isSafeSelectToSpeculate(*Sel)) { | |||
4412 | Promotable = false; | |||
4413 | PHIUsers.clear(); | |||
4414 | SelectUsers.clear(); | |||
4415 | break; | |||
4416 | } | |||
4417 | ||||
4418 | if (Promotable) { | |||
4419 | for (Use *U : AS.getDeadUsesIfPromotable()) { | |||
4420 | auto *OldInst = dyn_cast<Instruction>(U->get()); | |||
4421 | Value::dropDroppableUse(*U); | |||
4422 | if (OldInst) | |||
4423 | if (isInstructionTriviallyDead(OldInst)) | |||
4424 | DeadInsts.push_back(OldInst); | |||
4425 | } | |||
4426 | if (PHIUsers.empty() && SelectUsers.empty()) { | |||
4427 | // Promote the alloca. | |||
4428 | PromotableAllocas.push_back(NewAI); | |||
4429 | } else { | |||
4430 | // If we have either PHIs or Selects to speculate, add them to those | |||
4431 | // worklists and re-queue the new alloca so that we promote in on the | |||
4432 | // next iteration. | |||
4433 | for (PHINode *PHIUser : PHIUsers) | |||
4434 | SpeculatablePHIs.insert(PHIUser); | |||
4435 | for (SelectInst *SelectUser : SelectUsers) | |||
4436 | SpeculatableSelects.insert(SelectUser); | |||
4437 | Worklist.insert(NewAI); | |||
4438 | } | |||
4439 | } else { | |||
4440 | // Drop any post-promotion work items if promotion didn't happen. | |||
4441 | while (PostPromotionWorklist.size() > PPWOldSize) | |||
4442 | PostPromotionWorklist.pop_back(); | |||
4443 | ||||
4444 | // We couldn't promote and we didn't create a new partition, nothing | |||
4445 | // happened. | |||
4446 | if (NewAI == &AI) | |||
4447 | return nullptr; | |||
4448 | ||||
4449 | // If we can't promote the alloca, iterate on it to check for new | |||
4450 | // refinements exposed by splitting the current alloca. Don't iterate on an | |||
4451 | // alloca which didn't actually change and didn't get promoted. | |||
4452 | Worklist.insert(NewAI); | |||
4453 | } | |||
4454 | ||||
4455 | return NewAI; | |||
4456 | } | |||
4457 | ||||
4458 | /// Walks the slices of an alloca and form partitions based on them, | |||
4459 | /// rewriting each of their uses. | |||
4460 | bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { | |||
4461 | if (AS.begin() == AS.end()) | |||
4462 | return false; | |||
4463 | ||||
4464 | unsigned NumPartitions = 0; | |||
4465 | bool Changed = false; | |||
4466 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||
4467 | ||||
4468 | // First try to pre-split loads and stores. | |||
4469 | Changed |= presplitLoadsAndStores(AI, AS); | |||
4470 | ||||
4471 | // Now that we have identified any pre-splitting opportunities, | |||
4472 | // mark loads and stores unsplittable except for the following case. | |||
4473 | // We leave a slice splittable if all other slices are disjoint or fully | |||
4474 | // included in the slice, such as whole-alloca loads and stores. | |||
4475 | // If we fail to split these during pre-splitting, we want to force them | |||
4476 | // to be rewritten into a partition. | |||
4477 | bool IsSorted = true; | |||
4478 | ||||
4479 | uint64_t AllocaSize = | |||
4480 | DL.getTypeAllocSize(AI.getAllocatedType()).getFixedSize(); | |||
4481 | const uint64_t MaxBitVectorSize = 1024; | |||
4482 | if (AllocaSize <= MaxBitVectorSize) { | |||
4483 | // If a byte boundary is included in any load or store, a slice starting or | |||
4484 | // ending at the boundary is not splittable. | |||
4485 | SmallBitVector SplittableOffset(AllocaSize + 1, true); | |||
4486 | for (Slice &S : AS) | |||
4487 | for (unsigned O = S.beginOffset() + 1; | |||
4488 | O < S.endOffset() && O < AllocaSize; O++) | |||
4489 | SplittableOffset.reset(O); | |||
4490 | ||||
4491 | for (Slice &S : AS) { | |||
4492 | if (!S.isSplittable()) | |||
4493 | continue; | |||
4494 | ||||
4495 | if ((S.beginOffset() > AllocaSize || SplittableOffset[S.beginOffset()]) && | |||
4496 | (S.endOffset() > AllocaSize || SplittableOffset[S.endOffset()])) | |||
4497 | continue; | |||
4498 | ||||
4499 | if (isa<LoadInst>(S.getUse()->getUser()) || | |||
4500 | isa<StoreInst>(S.getUse()->getUser())) { | |||
4501 | S.makeUnsplittable(); | |||
4502 | IsSorted = false; | |||
4503 | } | |||
4504 | } | |||
4505 | } | |||
4506 | else { | |||
4507 | // We only allow whole-alloca splittable loads and stores | |||
4508 | // for a large alloca to avoid creating too large BitVector. | |||
4509 | for (Slice &S : AS) { | |||
4510 | if (!S.isSplittable()) | |||
4511 | continue; | |||
4512 | ||||
4513 | if (S.beginOffset() == 0 && S.endOffset() >= AllocaSize) | |||
4514 | continue; | |||
4515 | ||||
4516 | if (isa<LoadInst>(S.getUse()->getUser()) || | |||
4517 | isa<StoreInst>(S.getUse()->getUser())) { | |||
4518 | S.makeUnsplittable(); | |||
4519 | IsSorted = false; | |||
4520 | } | |||
4521 | } | |||
4522 | } | |||
4523 | ||||
4524 | if (!IsSorted) | |||
4525 | llvm::sort(AS); | |||
4526 | ||||
4527 | /// Describes the allocas introduced by rewritePartition in order to migrate | |||
4528 | /// the debug info. | |||
4529 | struct Fragment { | |||
4530 | AllocaInst *Alloca; | |||
4531 | uint64_t Offset; | |||
4532 | uint64_t Size; | |||
4533 | Fragment(AllocaInst *AI, uint64_t O, uint64_t S) | |||
4534 | : Alloca(AI), Offset(O), Size(S) {} | |||
4535 | }; | |||
4536 | SmallVector<Fragment, 4> Fragments; | |||
4537 | ||||
4538 | // Rewrite each partition. | |||
4539 | for (auto &P : AS.partitions()) { | |||
4540 | if (AllocaInst *NewAI = rewritePartition(AI, AS, P)) { | |||
4541 | Changed = true; | |||
4542 | if (NewAI != &AI) { | |||
4543 | uint64_t SizeOfByte = 8; | |||
4544 | uint64_t AllocaSize = | |||
4545 | DL.getTypeSizeInBits(NewAI->getAllocatedType()).getFixedSize(); | |||
4546 | // Don't include any padding. | |||
4547 | uint64_t Size = std::min(AllocaSize, P.size() * SizeOfByte); | |||
4548 | Fragments.push_back(Fragment(NewAI, P.beginOffset() * SizeOfByte, Size)); | |||
4549 | } | |||
4550 | } | |||
4551 | ++NumPartitions; | |||
4552 | } | |||
4553 | ||||
4554 | NumAllocaPartitions += NumPartitions; | |||
4555 | MaxPartitionsPerAlloca.updateMax(NumPartitions); | |||
4556 | ||||
4557 | // Migrate debug information from the old alloca to the new alloca(s) | |||
4558 | // and the individual partitions. | |||
4559 | TinyPtrVector<DbgVariableIntrinsic *> DbgDeclares = FindDbgAddrUses(&AI); | |||
4560 | for (DbgVariableIntrinsic *DbgDeclare : DbgDeclares) { | |||
4561 | auto *Expr = DbgDeclare->getExpression(); | |||
4562 | DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false); | |||
4563 | uint64_t AllocaSize = | |||
4564 | DL.getTypeSizeInBits(AI.getAllocatedType()).getFixedSize(); | |||
4565 | for (auto Fragment : Fragments) { | |||
4566 | // Create a fragment expression describing the new partition or reuse AI's | |||
4567 | // expression if there is only one partition. | |||
4568 | auto *FragmentExpr = Expr; | |||
4569 | if (Fragment.Size < AllocaSize || Expr->isFragment()) { | |||
4570 | // If this alloca is already a scalar replacement of a larger aggregate, | |||
4571 | // Fragment.Offset describes the offset inside the scalar. | |||
4572 | auto ExprFragment = Expr->getFragmentInfo(); | |||
4573 | uint64_t Offset = ExprFragment ? ExprFragment->OffsetInBits : 0; | |||
4574 | uint64_t Start = Offset + Fragment.Offset; | |||
4575 | uint64_t Size = Fragment.Size; | |||
4576 | if (ExprFragment) { | |||
4577 | uint64_t AbsEnd = | |||
4578 | ExprFragment->OffsetInBits + ExprFragment->SizeInBits; | |||
4579 | if (Start >= AbsEnd) | |||
4580 | // No need to describe a SROAed padding. | |||
4581 | continue; | |||
4582 | Size = std::min(Size, AbsEnd - Start); | |||
4583 | } | |||
4584 | // The new, smaller fragment is stenciled out from the old fragment. | |||
4585 | if (auto OrigFragment = FragmentExpr->getFragmentInfo()) { | |||
4586 | assert(Start >= OrigFragment->OffsetInBits &&(static_cast <bool> (Start >= OrigFragment->OffsetInBits && "new fragment is outside of original fragment") ? void (0) : __assert_fail ("Start >= OrigFragment->OffsetInBits && \"new fragment is outside of original fragment\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4587, __extension__ __PRETTY_FUNCTION__)) | |||
4587 | "new fragment is outside of original fragment")(static_cast <bool> (Start >= OrigFragment->OffsetInBits && "new fragment is outside of original fragment") ? void (0) : __assert_fail ("Start >= OrigFragment->OffsetInBits && \"new fragment is outside of original fragment\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/Transforms/Scalar/SROA.cpp" , 4587, __extension__ __PRETTY_FUNCTION__)); | |||
4588 | Start -= OrigFragment->OffsetInBits; | |||
4589 | } | |||
4590 | ||||
4591 | // The alloca may be larger than the variable. | |||
4592 | auto VarSize = DbgDeclare->getVariable()->getSizeInBits(); | |||
4593 | if (VarSize) { | |||
4594 | if (Size > *VarSize) | |||
4595 | Size = *VarSize; | |||
4596 | if (Size == 0 || Start + Size > *VarSize) | |||
4597 | continue; | |||
4598 | } | |||
4599 | ||||
4600 | // Avoid creating a fragment expression that covers the entire variable. | |||
4601 | if (!VarSize || *VarSize != Size) { | |||
4602 | if (auto E = | |||
4603 | DIExpression::createFragmentExpression(Expr, Start, Size)) | |||
4604 | FragmentExpr = *E; | |||
4605 | else | |||
4606 | continue; | |||
4607 | } | |||
4608 | } | |||
4609 | ||||
4610 | // Remove any existing intrinsics on the new alloca describing | |||
4611 | // the variable fragment. | |||
4612 | for (DbgVariableIntrinsic *OldDII : FindDbgAddrUses(Fragment.Alloca)) { | |||
4613 | auto SameVariableFragment = [](const DbgVariableIntrinsic *LHS, | |||
4614 | const DbgVariableIntrinsic *RHS) { | |||
4615 | return LHS->getVariable() == RHS->getVariable() && | |||
4616 | LHS->getDebugLoc()->getInlinedAt() == | |||
4617 | RHS->getDebugLoc()->getInlinedAt(); | |||
4618 | }; | |||
4619 | if (SameVariableFragment(OldDII, DbgDeclare)) | |||
4620 | OldDII->eraseFromParent(); | |||
4621 | } | |||
4622 | ||||
4623 | DIB.insertDeclare(Fragment.Alloca, DbgDeclare->getVariable(), FragmentExpr, | |||
4624 | DbgDeclare->getDebugLoc(), &AI); | |||
4625 | } | |||
4626 | } | |||
4627 | return Changed; | |||
4628 | } | |||
4629 | ||||
4630 | /// Clobber a use with undef, deleting the used value if it becomes dead. | |||
4631 | void SROA::clobberUse(Use &U) { | |||
4632 | Value *OldV = U; | |||
4633 | // Replace the use with an undef value. | |||
4634 | U = UndefValue::get(OldV->getType()); | |||
4635 | ||||
4636 | // Check for this making an instruction dead. We have to garbage collect | |||
4637 | // all the dead instructions to ensure the uses of any alloca end up being | |||
4638 | // minimal. | |||
4639 | if (Instruction *OldI = dyn_cast<Instruction>(OldV)) | |||
4640 | if (isInstructionTriviallyDead(OldI)) { | |||
4641 | DeadInsts.push_back(OldI); | |||
4642 | } | |||
4643 | } | |||
4644 | ||||
4645 | /// Analyze an alloca for SROA. | |||
4646 | /// | |||
4647 | /// This analyzes the alloca to ensure we can reason about it, builds | |||
4648 | /// the slices of the alloca, and then hands it off to be split and | |||
4649 | /// rewritten as needed. | |||
4650 | bool SROA::runOnAlloca(AllocaInst &AI) { | |||
4651 | LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "SROA alloca: " << AI << "\n"; } } while (false); | |||
4652 | ++NumAllocasAnalyzed; | |||
4653 | ||||
4654 | // Special case dead allocas, as they're trivial. | |||
4655 | if (AI.use_empty()) { | |||
4656 | AI.eraseFromParent(); | |||
4657 | return true; | |||
4658 | } | |||
4659 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||
4660 | ||||
4661 | // Skip alloca forms that this analysis can't handle. | |||
4662 | auto *AT = AI.getAllocatedType(); | |||
4663 | if (AI.isArrayAllocation() || !AT->isSized() || isa<ScalableVectorType>(AT) || | |||
4664 | DL.getTypeAllocSize(AT).getFixedSize() == 0) | |||
4665 | return false; | |||
4666 | ||||
4667 | bool Changed = false; | |||
4668 | ||||
4669 | // First, split any FCA loads and stores touching this alloca to promote | |||
4670 | // better splitting and promotion opportunities. | |||
4671 | AggLoadStoreRewriter AggRewriter(DL); | |||
4672 | Changed |= AggRewriter.rewrite(AI); | |||
4673 | ||||
4674 | // Build the slices using a recursive instruction-visiting builder. | |||
4675 | AllocaSlices AS(DL, AI); | |||
4676 | LLVM_DEBUG(AS.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.print(dbgs()); } } while (false); | |||
4677 | if (AS.isEscaped()) | |||
4678 | return Changed; | |||
4679 | ||||
4680 | // Delete all the dead users of this alloca before splitting and rewriting it. | |||
4681 | for (Instruction *DeadUser : AS.getDeadUsers()) { | |||
4682 | // Free up everything used by this instruction. | |||
4683 | for (Use &DeadOp : DeadUser->operands()) | |||
4684 | clobberUse(DeadOp); | |||
4685 | ||||
4686 | // Now replace the uses of this instruction. | |||
4687 | DeadUser->replaceAllUsesWith(UndefValue::get(DeadUser->getType())); | |||
4688 | ||||
4689 | // And mark it for deletion. | |||
4690 | DeadInsts.push_back(DeadUser); | |||
4691 | Changed = true; | |||
4692 | } | |||
4693 | for (Use *DeadOp : AS.getDeadOperands()) { | |||
4694 | clobberUse(*DeadOp); | |||
4695 | Changed = true; | |||
4696 | } | |||
4697 | ||||
4698 | // No slices to split. Leave the dead alloca for a later pass to clean up. | |||
4699 | if (AS.begin() == AS.end()) | |||
4700 | return Changed; | |||
4701 | ||||
4702 | Changed |= splitAlloca(AI, AS); | |||
4703 | ||||
4704 | LLVM_DEBUG(dbgs() << " Speculating PHIs\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Speculating PHIs\n"; } } while (false); | |||
4705 | while (!SpeculatablePHIs.empty()) | |||
4706 | speculatePHINodeLoads(*SpeculatablePHIs.pop_back_val()); | |||
4707 | ||||
4708 | LLVM_DEBUG(dbgs() << " Speculating Selects\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Speculating Selects\n"; } } while (false); | |||
4709 | while (!SpeculatableSelects.empty()) | |||
4710 | speculateSelectInstLoads(*SpeculatableSelects.pop_back_val()); | |||
4711 | ||||
4712 | return Changed; | |||
4713 | } | |||
4714 | ||||
4715 | /// Delete the dead instructions accumulated in this run. | |||
4716 | /// | |||
4717 | /// Recursively deletes the dead instructions we've accumulated. This is done | |||
4718 | /// at the very end to maximize locality of the recursive delete and to | |||
4719 | /// minimize the problems of invalidated instruction pointers as such pointers | |||
4720 | /// are used heavily in the intermediate stages of the algorithm. | |||
4721 | /// | |||
4722 | /// We also record the alloca instructions deleted here so that they aren't | |||
4723 | /// subsequently handed to mem2reg to promote. | |||
4724 | bool SROA::deleteDeadInstructions( | |||
4725 | SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) { | |||
4726 | bool Changed = false; | |||
4727 | while (!DeadInsts.empty()) { | |||
4728 | Instruction *I = dyn_cast_or_null<Instruction>(DeadInsts.pop_back_val()); | |||
4729 | if (!I) continue; | |||
4730 | LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Deleting dead instruction: " << *I << "\n"; } } while (false); | |||
4731 | ||||
4732 | // If the instruction is an alloca, find the possible dbg.declare connected | |||
4733 | // to it, and remove it too. We must do this before calling RAUW or we will | |||
4734 | // not be able to find it. | |||
4735 | if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) { | |||
4736 | DeletedAllocas.insert(AI); | |||
4737 | for (DbgVariableIntrinsic *OldDII : FindDbgAddrUses(AI)) | |||
4738 | OldDII->eraseFromParent(); | |||
4739 | } | |||
4740 | ||||
4741 | I->replaceAllUsesWith(UndefValue::get(I->getType())); | |||
4742 | ||||
4743 | for (Use &Operand : I->operands()) | |||
4744 | if (Instruction *U = dyn_cast<Instruction>(Operand)) { | |||
4745 | // Zero out the operand and see if it becomes trivially dead. | |||
4746 | Operand = nullptr; | |||
4747 | if (isInstructionTriviallyDead(U)) | |||
4748 | DeadInsts.push_back(U); | |||
4749 | } | |||
4750 | ||||
4751 | ++NumDeleted; | |||
4752 | I->eraseFromParent(); | |||
4753 | Changed = true; | |||
4754 | } | |||
4755 | return Changed; | |||
4756 | } | |||
4757 | ||||
4758 | /// Promote the allocas, using the best available technique. | |||
4759 | /// | |||
4760 | /// This attempts to promote whatever allocas have been identified as viable in | |||
4761 | /// the PromotableAllocas list. If that list is empty, there is nothing to do. | |||
4762 | /// This function returns whether any promotion occurred. | |||
4763 | bool SROA::promoteAllocas(Function &F) { | |||
4764 | if (PromotableAllocas.empty()) | |||
4765 | return false; | |||
4766 | ||||
4767 | NumPromoted += PromotableAllocas.size(); | |||
4768 | ||||
4769 | LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Promoting allocas with mem2reg...\n" ; } } while (false); | |||
4770 | PromoteMemToReg(PromotableAllocas, *DT, AC); | |||
4771 | PromotableAllocas.clear(); | |||
4772 | return true; | |||
4773 | } | |||
4774 | ||||
4775 | PreservedAnalyses SROA::runImpl(Function &F, DominatorTree &RunDT, | |||
4776 | AssumptionCache &RunAC) { | |||
4777 | LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "SROA function: " << F.getName () << "\n"; } } while (false); | |||
4778 | C = &F.getContext(); | |||
4779 | DT = &RunDT; | |||
4780 | AC = &RunAC; | |||
4781 | ||||
4782 | BasicBlock &EntryBB = F.getEntryBlock(); | |||
4783 | for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(EntryBB.end()); | |||
4784 | I != E; ++I) { | |||
4785 | if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) { | |||
4786 | if (isa<ScalableVectorType>(AI->getAllocatedType())) { | |||
4787 | if (isAllocaPromotable(AI)) | |||
4788 | PromotableAllocas.push_back(AI); | |||
4789 | } else { | |||
4790 | Worklist.insert(AI); | |||
4791 | } | |||
4792 | } | |||
4793 | } | |||
4794 | ||||
4795 | bool Changed = false; | |||
4796 | // A set of deleted alloca instruction pointers which should be removed from | |||
4797 | // the list of promotable allocas. | |||
4798 | SmallPtrSet<AllocaInst *, 4> DeletedAllocas; | |||
4799 | ||||
4800 | do { | |||
4801 | while (!Worklist.empty()) { | |||
4802 | Changed |= runOnAlloca(*Worklist.pop_back_val()); | |||
4803 | Changed |= deleteDeadInstructions(DeletedAllocas); | |||
4804 | ||||
4805 | // Remove the deleted allocas from various lists so that we don't try to | |||
4806 | // continue processing them. | |||
4807 | if (!DeletedAllocas.empty()) { | |||
4808 | auto IsInSet = [&](AllocaInst *AI) { return DeletedAllocas.count(AI); }; | |||
4809 | Worklist.remove_if(IsInSet); | |||
4810 | PostPromotionWorklist.remove_if(IsInSet); | |||
4811 | llvm::erase_if(PromotableAllocas, IsInSet); | |||
4812 | DeletedAllocas.clear(); | |||
4813 | } | |||
4814 | } | |||
4815 | ||||
4816 | Changed |= promoteAllocas(F); | |||
4817 | ||||
4818 | Worklist = PostPromotionWorklist; | |||
4819 | PostPromotionWorklist.clear(); | |||
4820 | } while (!Worklist.empty()); | |||
4821 | ||||
4822 | if (!Changed) | |||
4823 | return PreservedAnalyses::all(); | |||
4824 | ||||
4825 | PreservedAnalyses PA; | |||
4826 | PA.preserveSet<CFGAnalyses>(); | |||
4827 | return PA; | |||
4828 | } | |||
4829 | ||||
4830 | PreservedAnalyses SROA::run(Function &F, FunctionAnalysisManager &AM) { | |||
4831 | return runImpl(F, AM.getResult<DominatorTreeAnalysis>(F), | |||
4832 | AM.getResult<AssumptionAnalysis>(F)); | |||
4833 | } | |||
4834 | ||||
4835 | /// A legacy pass for the legacy pass manager that wraps the \c SROA pass. | |||
4836 | /// | |||
4837 | /// This is in the llvm namespace purely to allow it to be a friend of the \c | |||
4838 | /// SROA pass. | |||
4839 | class llvm::sroa::SROALegacyPass : public FunctionPass { | |||
4840 | /// The SROA implementation. | |||
4841 | SROA Impl; | |||
4842 | ||||
4843 | public: | |||
4844 | static char ID; | |||
4845 | ||||
4846 | SROALegacyPass() : FunctionPass(ID) { | |||
4847 | initializeSROALegacyPassPass(*PassRegistry::getPassRegistry()); | |||
4848 | } | |||
4849 | ||||
4850 | bool runOnFunction(Function &F) override { | |||
4851 | if (skipFunction(F)) | |||
4852 | return false; | |||
4853 | ||||
4854 | auto PA = Impl.runImpl( | |||
4855 | F, getAnalysis<DominatorTreeWrapperPass>().getDomTree(), | |||
4856 | getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F)); | |||
4857 | return !PA.areAllPreserved(); | |||
4858 | } | |||
4859 | ||||
4860 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
4861 | AU.addRequired<AssumptionCacheTracker>(); | |||
4862 | AU.addRequired<DominatorTreeWrapperPass>(); | |||
4863 | AU.addPreserved<GlobalsAAWrapperPass>(); | |||
4864 | AU.setPreservesCFG(); | |||
4865 | } | |||
4866 | ||||
4867 | StringRef getPassName() const override { return "SROA"; } | |||
4868 | }; | |||
4869 | ||||
4870 | char SROALegacyPass::ID = 0; | |||
4871 | ||||
4872 | FunctionPass *llvm::createSROAPass() { return new SROALegacyPass(); } | |||
4873 | ||||
4874 | INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",static void *initializeSROALegacyPassPassOnce(PassRegistry & Registry) { | |||
4875 | "Scalar Replacement Of Aggregates", false, false)static void *initializeSROALegacyPassPassOnce(PassRegistry & Registry) { | |||
4876 | INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry); | |||
4877 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); | |||
4878 | INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",PassInfo *PI = new PassInfo( "Scalar Replacement Of Aggregates" , "sroa", &SROALegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <SROALegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeSROALegacyPassPassFlag ; void llvm::initializeSROALegacyPassPass(PassRegistry &Registry ) { llvm::call_once(InitializeSROALegacyPassPassFlag, initializeSROALegacyPassPassOnce , std::ref(Registry)); } | |||
4879 | false, false)PassInfo *PI = new PassInfo( "Scalar Replacement Of Aggregates" , "sroa", &SROALegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <SROALegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeSROALegacyPassPassFlag ; void llvm::initializeSROALegacyPassPass(PassRegistry &Registry ) { llvm::call_once(InitializeSROALegacyPassPassFlag, initializeSROALegacyPassPassOnce , std::ref(Registry)); } |