| File: | lib/Transforms/Scalar/SROA.cpp |
| Warning: | line 2453, column 5 Use of memory after it is freed |
| 1 | //===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===// | |||||||||
| 2 | // | |||||||||
| 3 | // The LLVM Compiler Infrastructure | |||||||||
| 4 | // | |||||||||
| 5 | // This file is distributed under the University of Illinois Open Source | |||||||||
| 6 | // License. See LICENSE.TXT for details. | |||||||||
| 7 | // | |||||||||
| 8 | //===----------------------------------------------------------------------===// | |||||||||
| 9 | /// \file | |||||||||
| 10 | /// This transformation implements the well known scalar replacement of | |||||||||
| 11 | /// aggregates transformation. It tries to identify promotable elements of an | |||||||||
| 12 | /// aggregate alloca, and promote them to registers. It will also try to | |||||||||
| 13 | /// convert uses of an element (or set of elements) of an alloca into a vector | |||||||||
| 14 | /// or bitfield-style integer scalar if appropriate. | |||||||||
| 15 | /// | |||||||||
| 16 | /// It works to do this with minimal slicing of the alloca so that regions | |||||||||
| 17 | /// which are merely transferred in and out of external memory remain unchanged | |||||||||
| 18 | /// and are not decomposed to scalar code. | |||||||||
| 19 | /// | |||||||||
| 20 | /// Because this also performs alloca promotion, it can be thought of as also | |||||||||
| 21 | /// serving the purpose of SSA formation. The algorithm iterates on the | |||||||||
| 22 | /// function until all opportunities for promotion have been realized. | |||||||||
| 23 | /// | |||||||||
| 24 | //===----------------------------------------------------------------------===// | |||||||||
| 25 | ||||||||||
| 26 | #include "llvm/Transforms/Scalar/SROA.h" | |||||||||
| 27 | #include "llvm/ADT/STLExtras.h" | |||||||||
| 28 | #include "llvm/ADT/SetVector.h" | |||||||||
| 29 | #include "llvm/ADT/SmallVector.h" | |||||||||
| 30 | #include "llvm/ADT/Statistic.h" | |||||||||
| 31 | #include "llvm/Analysis/AssumptionCache.h" | |||||||||
| 32 | #include "llvm/Analysis/GlobalsModRef.h" | |||||||||
| 33 | #include "llvm/Analysis/Loads.h" | |||||||||
| 34 | #include "llvm/Analysis/PtrUseVisitor.h" | |||||||||
| 35 | #include "llvm/Analysis/ValueTracking.h" | |||||||||
| 36 | #include "llvm/IR/Constants.h" | |||||||||
| 37 | #include "llvm/IR/DIBuilder.h" | |||||||||
| 38 | #include "llvm/IR/DataLayout.h" | |||||||||
| 39 | #include "llvm/IR/DebugInfo.h" | |||||||||
| 40 | #include "llvm/IR/DerivedTypes.h" | |||||||||
| 41 | #include "llvm/IR/IRBuilder.h" | |||||||||
| 42 | #include "llvm/IR/InstVisitor.h" | |||||||||
| 43 | #include "llvm/IR/Instructions.h" | |||||||||
| 44 | #include "llvm/IR/IntrinsicInst.h" | |||||||||
| 45 | #include "llvm/IR/LLVMContext.h" | |||||||||
| 46 | #include "llvm/IR/Operator.h" | |||||||||
| 47 | #include "llvm/Pass.h" | |||||||||
| 48 | #include "llvm/Support/Chrono.h" | |||||||||
| 49 | #include "llvm/Support/CommandLine.h" | |||||||||
| 50 | #include "llvm/Support/Compiler.h" | |||||||||
| 51 | #include "llvm/Support/Debug.h" | |||||||||
| 52 | #include "llvm/Support/ErrorHandling.h" | |||||||||
| 53 | #include "llvm/Support/MathExtras.h" | |||||||||
| 54 | #include "llvm/Support/raw_ostream.h" | |||||||||
| 55 | #include "llvm/Transforms/Scalar.h" | |||||||||
| 56 | #include "llvm/Transforms/Utils/Local.h" | |||||||||
| 57 | #include "llvm/Transforms/Utils/PromoteMemToReg.h" | |||||||||
| 58 | ||||||||||
| 59 | #ifndef NDEBUG | |||||||||
| 60 | // We only use this for a debug check. | |||||||||
| 61 | #include <random> | |||||||||
| 62 | #endif | |||||||||
| 63 | ||||||||||
| 64 | using namespace llvm; | |||||||||
| 65 | using namespace llvm::sroa; | |||||||||
| 66 | ||||||||||
| 67 | #define DEBUG_TYPE"sroa" "sroa" | |||||||||
| 68 | ||||||||||
| 69 | STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement")static llvm::Statistic NumAllocasAnalyzed = {"sroa", "NumAllocasAnalyzed" , "Number of allocas analyzed for replacement", {0}, false}; | |||||||||
| 70 | STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed")static llvm::Statistic NumAllocaPartitions = {"sroa", "NumAllocaPartitions" , "Number of alloca partitions formed", {0}, false}; | |||||||||
| 71 | STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca")static llvm::Statistic MaxPartitionsPerAlloca = {"sroa", "MaxPartitionsPerAlloca" , "Maximum number of partitions per alloca", {0}, false}; | |||||||||
| 72 | STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten")static llvm::Statistic NumAllocaPartitionUses = {"sroa", "NumAllocaPartitionUses" , "Number of alloca partition uses rewritten", {0}, false}; | |||||||||
| 73 | STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition")static llvm::Statistic MaxUsesPerAllocaPartition = {"sroa", "MaxUsesPerAllocaPartition" , "Maximum number of uses of a partition", {0}, false}; | |||||||||
| 74 | STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced")static llvm::Statistic NumNewAllocas = {"sroa", "NumNewAllocas" , "Number of new, smaller allocas introduced", {0}, false}; | |||||||||
| 75 | STATISTIC(NumPromoted, "Number of allocas promoted to SSA values")static llvm::Statistic NumPromoted = {"sroa", "NumPromoted", "Number of allocas promoted to SSA values" , {0}, false}; | |||||||||
| 76 | STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion")static llvm::Statistic NumLoadsSpeculated = {"sroa", "NumLoadsSpeculated" , "Number of loads speculated to allow promotion", {0}, false }; | |||||||||
| 77 | STATISTIC(NumDeleted, "Number of instructions deleted")static llvm::Statistic NumDeleted = {"sroa", "NumDeleted", "Number of instructions deleted" , {0}, false}; | |||||||||
| 78 | STATISTIC(NumVectorized, "Number of vectorized aggregates")static llvm::Statistic NumVectorized = {"sroa", "NumVectorized" , "Number of vectorized aggregates", {0}, false}; | |||||||||
| 79 | ||||||||||
| 80 | /// Hidden option to enable randomly shuffling the slices to help uncover | |||||||||
| 81 | /// instability in their order. | |||||||||
| 82 | static cl::opt<bool> SROARandomShuffleSlices("sroa-random-shuffle-slices", | |||||||||
| 83 | cl::init(false), cl::Hidden); | |||||||||
| 84 | ||||||||||
| 85 | /// Hidden option to experiment with completely strict handling of inbounds | |||||||||
| 86 | /// GEPs. | |||||||||
| 87 | static cl::opt<bool> SROAStrictInbounds("sroa-strict-inbounds", cl::init(false), | |||||||||
| 88 | cl::Hidden); | |||||||||
| 89 | ||||||||||
| 90 | namespace { | |||||||||
| 91 | /// \brief A custom IRBuilder inserter which prefixes all names, but only in | |||||||||
| 92 | /// Assert builds. | |||||||||
| 93 | class IRBuilderPrefixedInserter : public IRBuilderDefaultInserter { | |||||||||
| 94 | std::string Prefix; | |||||||||
| 95 | const Twine getNameWithPrefix(const Twine &Name) const { | |||||||||
| 96 | return Name.isTriviallyEmpty() ? Name : Prefix + Name; | |||||||||
| 97 | } | |||||||||
| 98 | ||||||||||
| 99 | public: | |||||||||
| 100 | void SetNamePrefix(const Twine &P) { Prefix = P.str(); } | |||||||||
| 101 | ||||||||||
| 102 | protected: | |||||||||
| 103 | void InsertHelper(Instruction *I, const Twine &Name, BasicBlock *BB, | |||||||||
| 104 | BasicBlock::iterator InsertPt) const { | |||||||||
| 105 | IRBuilderDefaultInserter::InsertHelper(I, getNameWithPrefix(Name), BB, | |||||||||
| 106 | InsertPt); | |||||||||
| 107 | } | |||||||||
| 108 | }; | |||||||||
| 109 | ||||||||||
| 110 | /// \brief Provide a typedef for IRBuilder that drops names in release builds. | |||||||||
| 111 | using IRBuilderTy = llvm::IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>; | |||||||||
| 112 | } | |||||||||
| 113 | ||||||||||
| 114 | namespace { | |||||||||
| 115 | /// \brief A used slice of an alloca. | |||||||||
| 116 | /// | |||||||||
| 117 | /// This structure represents a slice of an alloca used by some instruction. It | |||||||||
| 118 | /// stores both the begin and end offsets of this use, a pointer to the use | |||||||||
| 119 | /// itself, and a flag indicating whether we can classify the use as splittable | |||||||||
| 120 | /// or not when forming partitions of the alloca. | |||||||||
| 121 | class Slice { | |||||||||
| 122 | /// \brief The beginning offset of the range. | |||||||||
| 123 | uint64_t BeginOffset; | |||||||||
| 124 | ||||||||||
| 125 | /// \brief The ending offset, not included in the range. | |||||||||
| 126 | uint64_t EndOffset; | |||||||||
| 127 | ||||||||||
| 128 | /// \brief Storage for both the use of this slice and whether it can be | |||||||||
| 129 | /// split. | |||||||||
| 130 | PointerIntPair<Use *, 1, bool> UseAndIsSplittable; | |||||||||
| 131 | ||||||||||
| 132 | public: | |||||||||
| 133 | Slice() : BeginOffset(), EndOffset() {} | |||||||||
| 134 | Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable) | |||||||||
| 135 | : BeginOffset(BeginOffset), EndOffset(EndOffset), | |||||||||
| 136 | UseAndIsSplittable(U, IsSplittable) {} | |||||||||
| 137 | ||||||||||
| 138 | uint64_t beginOffset() const { return BeginOffset; } | |||||||||
| 139 | uint64_t endOffset() const { return EndOffset; } | |||||||||
| 140 | ||||||||||
| 141 | bool isSplittable() const { return UseAndIsSplittable.getInt(); } | |||||||||
| 142 | void makeUnsplittable() { UseAndIsSplittable.setInt(false); } | |||||||||
| 143 | ||||||||||
| 144 | Use *getUse() const { return UseAndIsSplittable.getPointer(); } | |||||||||
| 145 | ||||||||||
| 146 | bool isDead() const { return getUse() == nullptr; } | |||||||||
| 147 | void kill() { UseAndIsSplittable.setPointer(nullptr); } | |||||||||
| 148 | ||||||||||
| 149 | /// \brief Support for ordering ranges. | |||||||||
| 150 | /// | |||||||||
| 151 | /// This provides an ordering over ranges such that start offsets are | |||||||||
| 152 | /// always increasing, and within equal start offsets, the end offsets are | |||||||||
| 153 | /// decreasing. Thus the spanning range comes first in a cluster with the | |||||||||
| 154 | /// same start position. | |||||||||
| 155 | bool operator<(const Slice &RHS) const { | |||||||||
| 156 | if (beginOffset() < RHS.beginOffset()) | |||||||||
| 157 | return true; | |||||||||
| 158 | if (beginOffset() > RHS.beginOffset()) | |||||||||
| 159 | return false; | |||||||||
| 160 | if (isSplittable() != RHS.isSplittable()) | |||||||||
| 161 | return !isSplittable(); | |||||||||
| 162 | if (endOffset() > RHS.endOffset()) | |||||||||
| 163 | return true; | |||||||||
| 164 | return false; | |||||||||
| 165 | } | |||||||||
| 166 | ||||||||||
| 167 | /// \brief Support comparison with a single offset to allow binary searches. | |||||||||
| 168 | friend LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__)) bool operator<(const Slice &LHS, | |||||||||
| 169 | uint64_t RHSOffset) { | |||||||||
| 170 | return LHS.beginOffset() < RHSOffset; | |||||||||
| 171 | } | |||||||||
| 172 | friend LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__)) bool operator<(uint64_t LHSOffset, | |||||||||
| 173 | const Slice &RHS) { | |||||||||
| 174 | return LHSOffset < RHS.beginOffset(); | |||||||||
| 175 | } | |||||||||
| 176 | ||||||||||
| 177 | bool operator==(const Slice &RHS) const { | |||||||||
| 178 | return isSplittable() == RHS.isSplittable() && | |||||||||
| 179 | beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset(); | |||||||||
| 180 | } | |||||||||
| 181 | bool operator!=(const Slice &RHS) const { return !operator==(RHS); } | |||||||||
| 182 | }; | |||||||||
| 183 | } // end anonymous namespace | |||||||||
| 184 | ||||||||||
| 185 | namespace llvm { | |||||||||
| 186 | template <typename T> struct isPodLike; | |||||||||
| 187 | template <> struct isPodLike<Slice> { static const bool value = true; }; | |||||||||
| 188 | } | |||||||||
| 189 | ||||||||||
| 190 | /// \brief Representation of the alloca slices. | |||||||||
| 191 | /// | |||||||||
| 192 | /// This class represents the slices of an alloca which are formed by its | |||||||||
| 193 | /// various uses. If a pointer escapes, we can't fully build a representation | |||||||||
| 194 | /// for the slices used and we reflect that in this structure. The uses are | |||||||||
| 195 | /// stored, sorted by increasing beginning offset and with unsplittable slices | |||||||||
| 196 | /// starting at a particular offset before splittable slices. | |||||||||
| 197 | class llvm::sroa::AllocaSlices { | |||||||||
| 198 | public: | |||||||||
| 199 | /// \brief Construct the slices of a particular alloca. | |||||||||
| 200 | AllocaSlices(const DataLayout &DL, AllocaInst &AI); | |||||||||
| 201 | ||||||||||
| 202 | /// \brief Test whether a pointer to the allocation escapes our analysis. | |||||||||
| 203 | /// | |||||||||
| 204 | /// If this is true, the slices are never fully built and should be | |||||||||
| 205 | /// ignored. | |||||||||
| 206 | bool isEscaped() const { return PointerEscapingInstr; } | |||||||||
| 207 | ||||||||||
| 208 | /// \brief Support for iterating over the slices. | |||||||||
| 209 | /// @{ | |||||||||
| 210 | typedef SmallVectorImpl<Slice>::iterator iterator; | |||||||||
| 211 | typedef iterator_range<iterator> range; | |||||||||
| 212 | iterator begin() { return Slices.begin(); } | |||||||||
| 213 | iterator end() { return Slices.end(); } | |||||||||
| 214 | ||||||||||
| 215 | typedef SmallVectorImpl<Slice>::const_iterator const_iterator; | |||||||||
| 216 | typedef iterator_range<const_iterator> const_range; | |||||||||
| 217 | const_iterator begin() const { return Slices.begin(); } | |||||||||
| 218 | const_iterator end() const { return Slices.end(); } | |||||||||
| 219 | /// @} | |||||||||
| 220 | ||||||||||
| 221 | /// \brief Erase a range of slices. | |||||||||
| 222 | void erase(iterator Start, iterator Stop) { Slices.erase(Start, Stop); } | |||||||||
| 223 | ||||||||||
| 224 | /// \brief Insert new slices for this alloca. | |||||||||
| 225 | /// | |||||||||
| 226 | /// This moves the slices into the alloca's slices collection, and re-sorts | |||||||||
| 227 | /// everything so that the usual ordering properties of the alloca's slices | |||||||||
| 228 | /// hold. | |||||||||
| 229 | void insert(ArrayRef<Slice> NewSlices) { | |||||||||
| 230 | int OldSize = Slices.size(); | |||||||||
| 231 | Slices.append(NewSlices.begin(), NewSlices.end()); | |||||||||
| 232 | auto SliceI = Slices.begin() + OldSize; | |||||||||
| 233 | std::sort(SliceI, Slices.end()); | |||||||||
| 234 | std::inplace_merge(Slices.begin(), SliceI, Slices.end()); | |||||||||
| 235 | } | |||||||||
| 236 | ||||||||||
| 237 | // Forward declare the iterator and range accessor for walking the | |||||||||
| 238 | // partitions. | |||||||||
| 239 | class partition_iterator; | |||||||||
| 240 | iterator_range<partition_iterator> partitions(); | |||||||||
| 241 | ||||||||||
| 242 | /// \brief Access the dead users for this alloca. | |||||||||
| 243 | ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; } | |||||||||
| 244 | ||||||||||
| 245 | /// \brief Access the dead operands referring to this alloca. | |||||||||
| 246 | /// | |||||||||
| 247 | /// These are operands which have cannot actually be used to refer to the | |||||||||
| 248 | /// alloca as they are outside its range and the user doesn't correct for | |||||||||
| 249 | /// that. These mostly consist of PHI node inputs and the like which we just | |||||||||
| 250 | /// need to replace with undef. | |||||||||
| 251 | ArrayRef<Use *> getDeadOperands() const { return DeadOperands; } | |||||||||
| 252 | ||||||||||
| 253 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||||||||
| 254 | void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const; | |||||||||
| 255 | void printSlice(raw_ostream &OS, const_iterator I, | |||||||||
| 256 | StringRef Indent = " ") const; | |||||||||
| 257 | void printUse(raw_ostream &OS, const_iterator I, | |||||||||
| 258 | StringRef Indent = " ") const; | |||||||||
| 259 | void print(raw_ostream &OS) const; | |||||||||
| 260 | void dump(const_iterator I) const; | |||||||||
| 261 | void dump() const; | |||||||||
| 262 | #endif | |||||||||
| 263 | ||||||||||
| 264 | private: | |||||||||
| 265 | template <typename DerivedT, typename RetT = void> class BuilderBase; | |||||||||
| 266 | class SliceBuilder; | |||||||||
| 267 | friend class AllocaSlices::SliceBuilder; | |||||||||
| 268 | ||||||||||
| 269 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||||||||
| 270 | /// \brief Handle to alloca instruction to simplify method interfaces. | |||||||||
| 271 | AllocaInst &AI; | |||||||||
| 272 | #endif | |||||||||
| 273 | ||||||||||
| 274 | /// \brief The instruction responsible for this alloca not having a known set | |||||||||
| 275 | /// of slices. | |||||||||
| 276 | /// | |||||||||
| 277 | /// When an instruction (potentially) escapes the pointer to the alloca, we | |||||||||
| 278 | /// store a pointer to that here and abort trying to form slices of the | |||||||||
| 279 | /// alloca. This will be null if the alloca slices are analyzed successfully. | |||||||||
| 280 | Instruction *PointerEscapingInstr; | |||||||||
| 281 | ||||||||||
| 282 | /// \brief The slices of the alloca. | |||||||||
| 283 | /// | |||||||||
| 284 | /// We store a vector of the slices formed by uses of the alloca here. This | |||||||||
| 285 | /// vector is sorted by increasing begin offset, and then the unsplittable | |||||||||
| 286 | /// slices before the splittable ones. See the Slice inner class for more | |||||||||
| 287 | /// details. | |||||||||
| 288 | SmallVector<Slice, 8> Slices; | |||||||||
| 289 | ||||||||||
| 290 | /// \brief Instructions which will become dead if we rewrite the alloca. | |||||||||
| 291 | /// | |||||||||
| 292 | /// Note that these are not separated by slice. This is because we expect an | |||||||||
| 293 | /// alloca to be completely rewritten or not rewritten at all. If rewritten, | |||||||||
| 294 | /// all these instructions can simply be removed and replaced with undef as | |||||||||
| 295 | /// they come from outside of the allocated space. | |||||||||
| 296 | SmallVector<Instruction *, 8> DeadUsers; | |||||||||
| 297 | ||||||||||
| 298 | /// \brief Operands which will become dead if we rewrite the alloca. | |||||||||
| 299 | /// | |||||||||
| 300 | /// These are operands that in their particular use can be replaced with | |||||||||
| 301 | /// undef when we rewrite the alloca. These show up in out-of-bounds inputs | |||||||||
| 302 | /// to PHI nodes and the like. They aren't entirely dead (there might be | |||||||||
| 303 | /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we | |||||||||
| 304 | /// want to swap this particular input for undef to simplify the use lists of | |||||||||
| 305 | /// the alloca. | |||||||||
| 306 | SmallVector<Use *, 8> DeadOperands; | |||||||||
| 307 | }; | |||||||||
| 308 | ||||||||||
| 309 | /// \brief A partition of the slices. | |||||||||
| 310 | /// | |||||||||
| 311 | /// An ephemeral representation for a range of slices which can be viewed as | |||||||||
| 312 | /// a partition of the alloca. This range represents a span of the alloca's | |||||||||
| 313 | /// memory which cannot be split, and provides access to all of the slices | |||||||||
| 314 | /// overlapping some part of the partition. | |||||||||
| 315 | /// | |||||||||
| 316 | /// Objects of this type are produced by traversing the alloca's slices, but | |||||||||
| 317 | /// are only ephemeral and not persistent. | |||||||||
| 318 | class llvm::sroa::Partition { | |||||||||
| 319 | private: | |||||||||
| 320 | friend class AllocaSlices; | |||||||||
| 321 | friend class AllocaSlices::partition_iterator; | |||||||||
| 322 | ||||||||||
| 323 | typedef AllocaSlices::iterator iterator; | |||||||||
| 324 | ||||||||||
| 325 | /// \brief The beginning and ending offsets of the alloca for this | |||||||||
| 326 | /// partition. | |||||||||
| 327 | uint64_t BeginOffset, EndOffset; | |||||||||
| 328 | ||||||||||
| 329 | /// \brief The start end end iterators of this partition. | |||||||||
| 330 | iterator SI, SJ; | |||||||||
| 331 | ||||||||||
| 332 | /// \brief A collection of split slice tails overlapping the partition. | |||||||||
| 333 | SmallVector<Slice *, 4> SplitTails; | |||||||||
| 334 | ||||||||||
| 335 | /// \brief Raw constructor builds an empty partition starting and ending at | |||||||||
| 336 | /// the given iterator. | |||||||||
| 337 | Partition(iterator SI) : SI(SI), SJ(SI) {} | |||||||||
| 338 | ||||||||||
| 339 | public: | |||||||||
| 340 | /// \brief The start offset of this partition. | |||||||||
| 341 | /// | |||||||||
| 342 | /// All of the contained slices start at or after this offset. | |||||||||
| 343 | uint64_t beginOffset() const { return BeginOffset; } | |||||||||
| 344 | ||||||||||
| 345 | /// \brief The end offset of this partition. | |||||||||
| 346 | /// | |||||||||
| 347 | /// All of the contained slices end at or before this offset. | |||||||||
| 348 | uint64_t endOffset() const { return EndOffset; } | |||||||||
| 349 | ||||||||||
| 350 | /// \brief The size of the partition. | |||||||||
| 351 | /// | |||||||||
| 352 | /// Note that this can never be zero. | |||||||||
| 353 | uint64_t size() const { | |||||||||
| 354 | assert(BeginOffset < EndOffset && "Partitions must span some bytes!")((BeginOffset < EndOffset && "Partitions must span some bytes!" ) ? static_cast<void> (0) : __assert_fail ("BeginOffset < EndOffset && \"Partitions must span some bytes!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 354, __PRETTY_FUNCTION__)); | |||||||||
| 355 | return EndOffset - BeginOffset; | |||||||||
| 356 | } | |||||||||
| 357 | ||||||||||
| 358 | /// \brief Test whether this partition contains no slices, and merely spans | |||||||||
| 359 | /// a region occupied by split slices. | |||||||||
| 360 | bool empty() const { return SI == SJ; } | |||||||||
| 361 | ||||||||||
| 362 | /// \name Iterate slices that start within the partition. | |||||||||
| 363 | /// These may be splittable or unsplittable. They have a begin offset >= the | |||||||||
| 364 | /// partition begin offset. | |||||||||
| 365 | /// @{ | |||||||||
| 366 | // FIXME: We should probably define a "concat_iterator" helper and use that | |||||||||
| 367 | // to stitch together pointee_iterators over the split tails and the | |||||||||
| 368 | // contiguous iterators of the partition. That would give a much nicer | |||||||||
| 369 | // interface here. We could then additionally expose filtered iterators for | |||||||||
| 370 | // split, unsplit, and unsplittable splices based on the usage patterns. | |||||||||
| 371 | iterator begin() const { return SI; } | |||||||||
| 372 | iterator end() const { return SJ; } | |||||||||
| 373 | /// @} | |||||||||
| 374 | ||||||||||
| 375 | /// \brief Get the sequence of split slice tails. | |||||||||
| 376 | /// | |||||||||
| 377 | /// These tails are of slices which start before this partition but are | |||||||||
| 378 | /// split and overlap into the partition. We accumulate these while forming | |||||||||
| 379 | /// partitions. | |||||||||
| 380 | ArrayRef<Slice *> splitSliceTails() const { return SplitTails; } | |||||||||
| 381 | }; | |||||||||
| 382 | ||||||||||
| 383 | /// \brief An iterator over partitions of the alloca's slices. | |||||||||
| 384 | /// | |||||||||
| 385 | /// This iterator implements the core algorithm for partitioning the alloca's | |||||||||
| 386 | /// slices. It is a forward iterator as we don't support backtracking for | |||||||||
| 387 | /// efficiency reasons, and re-use a single storage area to maintain the | |||||||||
| 388 | /// current set of split slices. | |||||||||
| 389 | /// | |||||||||
| 390 | /// It is templated on the slice iterator type to use so that it can operate | |||||||||
| 391 | /// with either const or non-const slice iterators. | |||||||||
| 392 | class AllocaSlices::partition_iterator | |||||||||
| 393 | : public iterator_facade_base<partition_iterator, std::forward_iterator_tag, | |||||||||
| 394 | Partition> { | |||||||||
| 395 | friend class AllocaSlices; | |||||||||
| 396 | ||||||||||
| 397 | /// \brief Most of the state for walking the partitions is held in a class | |||||||||
| 398 | /// with a nice interface for examining them. | |||||||||
| 399 | Partition P; | |||||||||
| 400 | ||||||||||
| 401 | /// \brief We need to keep the end of the slices to know when to stop. | |||||||||
| 402 | AllocaSlices::iterator SE; | |||||||||
| 403 | ||||||||||
| 404 | /// \brief We also need to keep track of the maximum split end offset seen. | |||||||||
| 405 | /// FIXME: Do we really? | |||||||||
| 406 | uint64_t MaxSplitSliceEndOffset; | |||||||||
| 407 | ||||||||||
| 408 | /// \brief Sets the partition to be empty at given iterator, and sets the | |||||||||
| 409 | /// end iterator. | |||||||||
| 410 | partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE) | |||||||||
| 411 | : P(SI), SE(SE), MaxSplitSliceEndOffset(0) { | |||||||||
| 412 | // If not already at the end, advance our state to form the initial | |||||||||
| 413 | // partition. | |||||||||
| 414 | if (SI != SE) | |||||||||
| 415 | advance(); | |||||||||
| 416 | } | |||||||||
| 417 | ||||||||||
| 418 | /// \brief Advance the iterator to the next partition. | |||||||||
| 419 | /// | |||||||||
| 420 | /// Requires that the iterator not be at the end of the slices. | |||||||||
| 421 | void advance() { | |||||||||
| 422 | assert((P.SI != SE || !P.SplitTails.empty()) &&(((P.SI != SE || !P.SplitTails.empty()) && "Cannot advance past the end of the slices!" ) ? static_cast<void> (0) : __assert_fail ("(P.SI != SE || !P.SplitTails.empty()) && \"Cannot advance past the end of the slices!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 423, __PRETTY_FUNCTION__)) | |||||||||
| 423 | "Cannot advance past the end of the slices!")(((P.SI != SE || !P.SplitTails.empty()) && "Cannot advance past the end of the slices!" ) ? static_cast<void> (0) : __assert_fail ("(P.SI != SE || !P.SplitTails.empty()) && \"Cannot advance past the end of the slices!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 423, __PRETTY_FUNCTION__)); | |||||||||
| 424 | ||||||||||
| 425 | // Clear out any split uses which have ended. | |||||||||
| 426 | if (!P.SplitTails.empty()) { | |||||||||
| 427 | if (P.EndOffset >= MaxSplitSliceEndOffset) { | |||||||||
| 428 | // If we've finished all splits, this is easy. | |||||||||
| 429 | P.SplitTails.clear(); | |||||||||
| 430 | MaxSplitSliceEndOffset = 0; | |||||||||
| 431 | } else { | |||||||||
| 432 | // Remove the uses which have ended in the prior partition. This | |||||||||
| 433 | // cannot change the max split slice end because we just checked that | |||||||||
| 434 | // the prior partition ended prior to that max. | |||||||||
| 435 | P.SplitTails.erase( | |||||||||
| 436 | remove_if(P.SplitTails, | |||||||||
| 437 | [&](Slice *S) { return S->endOffset() <= P.EndOffset; }), | |||||||||
| 438 | P.SplitTails.end()); | |||||||||
| 439 | assert(any_of(P.SplitTails,((any_of(P.SplitTails, [&](Slice *S) { return S->endOffset () == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 443, __PRETTY_FUNCTION__)) | |||||||||
| 440 | [&](Slice *S) {((any_of(P.SplitTails, [&](Slice *S) { return S->endOffset () == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 443, __PRETTY_FUNCTION__)) | |||||||||
| 441 | return S->endOffset() == MaxSplitSliceEndOffset;((any_of(P.SplitTails, [&](Slice *S) { return S->endOffset () == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 443, __PRETTY_FUNCTION__)) | |||||||||
| 442 | }) &&((any_of(P.SplitTails, [&](Slice *S) { return S->endOffset () == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 443, __PRETTY_FUNCTION__)) | |||||||||
| 443 | "Could not find the current max split slice offset!")((any_of(P.SplitTails, [&](Slice *S) { return S->endOffset () == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 443, __PRETTY_FUNCTION__)); | |||||||||
| 444 | assert(all_of(P.SplitTails,((all_of(P.SplitTails, [&](Slice *S) { return S->endOffset () <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 448, __PRETTY_FUNCTION__)) | |||||||||
| 445 | [&](Slice *S) {((all_of(P.SplitTails, [&](Slice *S) { return S->endOffset () <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 448, __PRETTY_FUNCTION__)) | |||||||||
| 446 | return S->endOffset() <= MaxSplitSliceEndOffset;((all_of(P.SplitTails, [&](Slice *S) { return S->endOffset () <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 448, __PRETTY_FUNCTION__)) | |||||||||
| 447 | }) &&((all_of(P.SplitTails, [&](Slice *S) { return S->endOffset () <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 448, __PRETTY_FUNCTION__)) | |||||||||
| 448 | "Max split slice end offset is not actually the max!")((all_of(P.SplitTails, [&](Slice *S) { return S->endOffset () <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 448, __PRETTY_FUNCTION__)); | |||||||||
| 449 | } | |||||||||
| 450 | } | |||||||||
| 451 | ||||||||||
| 452 | // If P.SI is already at the end, then we've cleared the split tail and | |||||||||
| 453 | // now have an end iterator. | |||||||||
| 454 | if (P.SI == SE) { | |||||||||
| 455 | assert(P.SplitTails.empty() && "Failed to clear the split slices!")((P.SplitTails.empty() && "Failed to clear the split slices!" ) ? static_cast<void> (0) : __assert_fail ("P.SplitTails.empty() && \"Failed to clear the split slices!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 455, __PRETTY_FUNCTION__)); | |||||||||
| 456 | return; | |||||||||
| 457 | } | |||||||||
| 458 | ||||||||||
| 459 | // If we had a non-empty partition previously, set up the state for | |||||||||
| 460 | // subsequent partitions. | |||||||||
| 461 | if (P.SI != P.SJ) { | |||||||||
| 462 | // Accumulate all the splittable slices which started in the old | |||||||||
| 463 | // partition into the split list. | |||||||||
| 464 | for (Slice &S : P) | |||||||||
| 465 | if (S.isSplittable() && S.endOffset() > P.EndOffset) { | |||||||||
| 466 | P.SplitTails.push_back(&S); | |||||||||
| 467 | MaxSplitSliceEndOffset = | |||||||||
| 468 | std::max(S.endOffset(), MaxSplitSliceEndOffset); | |||||||||
| 469 | } | |||||||||
| 470 | ||||||||||
| 471 | // Start from the end of the previous partition. | |||||||||
| 472 | P.SI = P.SJ; | |||||||||
| 473 | ||||||||||
| 474 | // If P.SI is now at the end, we at most have a tail of split slices. | |||||||||
| 475 | if (P.SI == SE) { | |||||||||
| 476 | P.BeginOffset = P.EndOffset; | |||||||||
| 477 | P.EndOffset = MaxSplitSliceEndOffset; | |||||||||
| 478 | return; | |||||||||
| 479 | } | |||||||||
| 480 | ||||||||||
| 481 | // If the we have split slices and the next slice is after a gap and is | |||||||||
| 482 | // not splittable immediately form an empty partition for the split | |||||||||
| 483 | // slices up until the next slice begins. | |||||||||
| 484 | if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset && | |||||||||
| 485 | !P.SI->isSplittable()) { | |||||||||
| 486 | P.BeginOffset = P.EndOffset; | |||||||||
| 487 | P.EndOffset = P.SI->beginOffset(); | |||||||||
| 488 | return; | |||||||||
| 489 | } | |||||||||
| 490 | } | |||||||||
| 491 | ||||||||||
| 492 | // OK, we need to consume new slices. Set the end offset based on the | |||||||||
| 493 | // current slice, and step SJ past it. The beginning offset of the | |||||||||
| 494 | // partition is the beginning offset of the next slice unless we have | |||||||||
| 495 | // pre-existing split slices that are continuing, in which case we begin | |||||||||
| 496 | // at the prior end offset. | |||||||||
| 497 | P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset; | |||||||||
| 498 | P.EndOffset = P.SI->endOffset(); | |||||||||
| 499 | ++P.SJ; | |||||||||
| 500 | ||||||||||
| 501 | // There are two strategies to form a partition based on whether the | |||||||||
| 502 | // partition starts with an unsplittable slice or a splittable slice. | |||||||||
| 503 | if (!P.SI->isSplittable()) { | |||||||||
| 504 | // When we're forming an unsplittable region, it must always start at | |||||||||
| 505 | // the first slice and will extend through its end. | |||||||||
| 506 | assert(P.BeginOffset == P.SI->beginOffset())((P.BeginOffset == P.SI->beginOffset()) ? static_cast<void > (0) : __assert_fail ("P.BeginOffset == P.SI->beginOffset()" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 506, __PRETTY_FUNCTION__)); | |||||||||
| 507 | ||||||||||
| 508 | // Form a partition including all of the overlapping slices with this | |||||||||
| 509 | // unsplittable slice. | |||||||||
| 510 | while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) { | |||||||||
| 511 | if (!P.SJ->isSplittable()) | |||||||||
| 512 | P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset()); | |||||||||
| 513 | ++P.SJ; | |||||||||
| 514 | } | |||||||||
| 515 | ||||||||||
| 516 | // We have a partition across a set of overlapping unsplittable | |||||||||
| 517 | // partitions. | |||||||||
| 518 | return; | |||||||||
| 519 | } | |||||||||
| 520 | ||||||||||
| 521 | // If we're starting with a splittable slice, then we need to form | |||||||||
| 522 | // a synthetic partition spanning it and any other overlapping splittable | |||||||||
| 523 | // splices. | |||||||||
| 524 | assert(P.SI->isSplittable() && "Forming a splittable partition!")((P.SI->isSplittable() && "Forming a splittable partition!" ) ? static_cast<void> (0) : __assert_fail ("P.SI->isSplittable() && \"Forming a splittable partition!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 524, __PRETTY_FUNCTION__)); | |||||||||
| 525 | ||||||||||
| 526 | // Collect all of the overlapping splittable slices. | |||||||||
| 527 | while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset && | |||||||||
| 528 | P.SJ->isSplittable()) { | |||||||||
| 529 | P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset()); | |||||||||
| 530 | ++P.SJ; | |||||||||
| 531 | } | |||||||||
| 532 | ||||||||||
| 533 | // Back upiP.EndOffset if we ended the span early when encountering an | |||||||||
| 534 | // unsplittable slice. This synthesizes the early end offset of | |||||||||
| 535 | // a partition spanning only splittable slices. | |||||||||
| 536 | if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) { | |||||||||
| 537 | assert(!P.SJ->isSplittable())((!P.SJ->isSplittable()) ? static_cast<void> (0) : __assert_fail ("!P.SJ->isSplittable()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 537, __PRETTY_FUNCTION__)); | |||||||||
| 538 | P.EndOffset = P.SJ->beginOffset(); | |||||||||
| 539 | } | |||||||||
| 540 | } | |||||||||
| 541 | ||||||||||
| 542 | public: | |||||||||
| 543 | bool operator==(const partition_iterator &RHS) const { | |||||||||
| 544 | assert(SE == RHS.SE &&((SE == RHS.SE && "End iterators don't match between compared partition iterators!" ) ? static_cast<void> (0) : __assert_fail ("SE == RHS.SE && \"End iterators don't match between compared partition iterators!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 545, __PRETTY_FUNCTION__)) | |||||||||
| 545 | "End iterators don't match between compared partition iterators!")((SE == RHS.SE && "End iterators don't match between compared partition iterators!" ) ? static_cast<void> (0) : __assert_fail ("SE == RHS.SE && \"End iterators don't match between compared partition iterators!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 545, __PRETTY_FUNCTION__)); | |||||||||
| 546 | ||||||||||
| 547 | // The observed positions of partitions is marked by the P.SI iterator and | |||||||||
| 548 | // the emptiness of the split slices. The latter is only relevant when | |||||||||
| 549 | // P.SI == SE, as the end iterator will additionally have an empty split | |||||||||
| 550 | // slices list, but the prior may have the same P.SI and a tail of split | |||||||||
| 551 | // slices. | |||||||||
| 552 | if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) { | |||||||||
| 553 | assert(P.SJ == RHS.P.SJ &&((P.SJ == RHS.P.SJ && "Same set of slices formed two different sized partitions!" ) ? static_cast<void> (0) : __assert_fail ("P.SJ == RHS.P.SJ && \"Same set of slices formed two different sized partitions!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 554, __PRETTY_FUNCTION__)) | |||||||||
| 554 | "Same set of slices formed two different sized partitions!")((P.SJ == RHS.P.SJ && "Same set of slices formed two different sized partitions!" ) ? static_cast<void> (0) : __assert_fail ("P.SJ == RHS.P.SJ && \"Same set of slices formed two different sized partitions!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 554, __PRETTY_FUNCTION__)); | |||||||||
| 555 | assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&((P.SplitTails.size() == RHS.P.SplitTails.size() && "Same slice position with differently sized non-empty split " "slice tails!") ? static_cast<void> (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 557, __PRETTY_FUNCTION__)) | |||||||||
| 556 | "Same slice position with differently sized non-empty split "((P.SplitTails.size() == RHS.P.SplitTails.size() && "Same slice position with differently sized non-empty split " "slice tails!") ? static_cast<void> (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 557, __PRETTY_FUNCTION__)) | |||||||||
| 557 | "slice tails!")((P.SplitTails.size() == RHS.P.SplitTails.size() && "Same slice position with differently sized non-empty split " "slice tails!") ? static_cast<void> (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 557, __PRETTY_FUNCTION__)); | |||||||||
| 558 | return true; | |||||||||
| 559 | } | |||||||||
| 560 | return false; | |||||||||
| 561 | } | |||||||||
| 562 | ||||||||||
| 563 | partition_iterator &operator++() { | |||||||||
| 564 | advance(); | |||||||||
| 565 | return *this; | |||||||||
| 566 | } | |||||||||
| 567 | ||||||||||
| 568 | Partition &operator*() { return P; } | |||||||||
| 569 | }; | |||||||||
| 570 | ||||||||||
| 571 | /// \brief A forward range over the partitions of the alloca's slices. | |||||||||
| 572 | /// | |||||||||
| 573 | /// This accesses an iterator range over the partitions of the alloca's | |||||||||
| 574 | /// slices. It computes these partitions on the fly based on the overlapping | |||||||||
| 575 | /// offsets of the slices and the ability to split them. It will visit "empty" | |||||||||
| 576 | /// partitions to cover regions of the alloca only accessed via split | |||||||||
| 577 | /// slices. | |||||||||
| 578 | iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() { | |||||||||
| 579 | return make_range(partition_iterator(begin(), end()), | |||||||||
| 580 | partition_iterator(end(), end())); | |||||||||
| 581 | } | |||||||||
| 582 | ||||||||||
| 583 | static Value *foldSelectInst(SelectInst &SI) { | |||||||||
| 584 | // If the condition being selected on is a constant or the same value is | |||||||||
| 585 | // being selected between, fold the select. Yes this does (rarely) happen | |||||||||
| 586 | // early on. | |||||||||
| 587 | if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition())) | |||||||||
| 588 | return SI.getOperand(1 + CI->isZero()); | |||||||||
| 589 | if (SI.getOperand(1) == SI.getOperand(2)) | |||||||||
| 590 | return SI.getOperand(1); | |||||||||
| 591 | ||||||||||
| 592 | return nullptr; | |||||||||
| 593 | } | |||||||||
| 594 | ||||||||||
| 595 | /// \brief A helper that folds a PHI node or a select. | |||||||||
| 596 | static Value *foldPHINodeOrSelectInst(Instruction &I) { | |||||||||
| 597 | if (PHINode *PN = dyn_cast<PHINode>(&I)) { | |||||||||
| 598 | // If PN merges together the same value, return that value. | |||||||||
| 599 | return PN->hasConstantValue(); | |||||||||
| 600 | } | |||||||||
| 601 | return foldSelectInst(cast<SelectInst>(I)); | |||||||||
| 602 | } | |||||||||
| 603 | ||||||||||
| 604 | /// \brief Builder for the alloca slices. | |||||||||
| 605 | /// | |||||||||
| 606 | /// This class builds a set of alloca slices by recursively visiting the uses | |||||||||
| 607 | /// of an alloca and making a slice for each load and store at each offset. | |||||||||
| 608 | class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> { | |||||||||
| 609 | friend class PtrUseVisitor<SliceBuilder>; | |||||||||
| 610 | friend class InstVisitor<SliceBuilder>; | |||||||||
| 611 | typedef PtrUseVisitor<SliceBuilder> Base; | |||||||||
| 612 | ||||||||||
| 613 | const uint64_t AllocSize; | |||||||||
| 614 | AllocaSlices &AS; | |||||||||
| 615 | ||||||||||
| 616 | SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap; | |||||||||
| 617 | SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes; | |||||||||
| 618 | ||||||||||
| 619 | /// \brief Set to de-duplicate dead instructions found in the use walk. | |||||||||
| 620 | SmallPtrSet<Instruction *, 4> VisitedDeadInsts; | |||||||||
| 621 | ||||||||||
| 622 | public: | |||||||||
| 623 | SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS) | |||||||||
| 624 | : PtrUseVisitor<SliceBuilder>(DL), | |||||||||
| 625 | AllocSize(DL.getTypeAllocSize(AI.getAllocatedType())), AS(AS) {} | |||||||||
| 626 | ||||||||||
| 627 | private: | |||||||||
| 628 | void markAsDead(Instruction &I) { | |||||||||
| 629 | if (VisitedDeadInsts.insert(&I).second) | |||||||||
| 630 | AS.DeadUsers.push_back(&I); | |||||||||
| 631 | } | |||||||||
| 632 | ||||||||||
| 633 | void insertUse(Instruction &I, const APInt &Offset, uint64_t Size, | |||||||||
| 634 | bool IsSplittable = false) { | |||||||||
| 635 | // Completely skip uses which have a zero size or start either before or | |||||||||
| 636 | // past the end of the allocation. | |||||||||
| 637 | if (Size == 0 || Offset.uge(AllocSize)) { | |||||||||
| 638 | DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offsetdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||||||||
| 639 | << " which has zero size or starts outside of the "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||||||||
| 640 | << AllocSize << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||||||||
| 641 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||||||||
| 642 | << " use: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false); | |||||||||
| 643 | return markAsDead(I); | |||||||||
| 644 | } | |||||||||
| 645 | ||||||||||
| 646 | uint64_t BeginOffset = Offset.getZExtValue(); | |||||||||
| 647 | uint64_t EndOffset = BeginOffset + Size; | |||||||||
| 648 | ||||||||||
| 649 | // Clamp the end offset to the end of the allocation. Note that this is | |||||||||
| 650 | // formulated to handle even the case where "BeginOffset + Size" overflows. | |||||||||
| 651 | // This may appear superficially to be something we could ignore entirely, | |||||||||
| 652 | // but that is not so! There may be widened loads or PHI-node uses where | |||||||||
| 653 | // some instructions are dead but not others. We can't completely ignore | |||||||||
| 654 | // them, and so have to record at least the information here. | |||||||||
| 655 | assert(AllocSize >= BeginOffset)((AllocSize >= BeginOffset) ? static_cast<void> (0) : __assert_fail ("AllocSize >= BeginOffset", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 655, __PRETTY_FUNCTION__)); // Established above. | |||||||||
| 656 | if (Size > AllocSize - BeginOffset) { | |||||||||
| 657 | DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offsetdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||||||||
| 658 | << " to remain within the " << AllocSize << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||||||||
| 659 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | |||||||||
| 660 | << " use: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false); | |||||||||
| 661 | EndOffset = AllocSize; | |||||||||
| 662 | } | |||||||||
| 663 | ||||||||||
| 664 | AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable)); | |||||||||
| 665 | } | |||||||||
| 666 | ||||||||||
| 667 | void visitBitCastInst(BitCastInst &BC) { | |||||||||
| 668 | if (BC.use_empty()) | |||||||||
| 669 | return markAsDead(BC); | |||||||||
| 670 | ||||||||||
| 671 | return Base::visitBitCastInst(BC); | |||||||||
| 672 | } | |||||||||
| 673 | ||||||||||
| 674 | void visitGetElementPtrInst(GetElementPtrInst &GEPI) { | |||||||||
| 675 | if (GEPI.use_empty()) | |||||||||
| 676 | return markAsDead(GEPI); | |||||||||
| 677 | ||||||||||
| 678 | if (SROAStrictInbounds && GEPI.isInBounds()) { | |||||||||
| 679 | // FIXME: This is a manually un-factored variant of the basic code inside | |||||||||
| 680 | // of GEPs with checking of the inbounds invariant specified in the | |||||||||
| 681 | // langref in a very strict sense. If we ever want to enable | |||||||||
| 682 | // SROAStrictInbounds, this code should be factored cleanly into | |||||||||
| 683 | // PtrUseVisitor, but it is easier to experiment with SROAStrictInbounds | |||||||||
| 684 | // by writing out the code here where we have the underlying allocation | |||||||||
| 685 | // size readily available. | |||||||||
| 686 | APInt GEPOffset = Offset; | |||||||||
| 687 | const DataLayout &DL = GEPI.getModule()->getDataLayout(); | |||||||||
| 688 | for (gep_type_iterator GTI = gep_type_begin(GEPI), | |||||||||
| 689 | GTE = gep_type_end(GEPI); | |||||||||
| 690 | GTI != GTE; ++GTI) { | |||||||||
| 691 | ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand()); | |||||||||
| 692 | if (!OpC) | |||||||||
| 693 | break; | |||||||||
| 694 | ||||||||||
| 695 | // Handle a struct index, which adds its field offset to the pointer. | |||||||||
| 696 | if (StructType *STy = GTI.getStructTypeOrNull()) { | |||||||||
| 697 | unsigned ElementIdx = OpC->getZExtValue(); | |||||||||
| 698 | const StructLayout *SL = DL.getStructLayout(STy); | |||||||||
| 699 | GEPOffset += | |||||||||
| 700 | APInt(Offset.getBitWidth(), SL->getElementOffset(ElementIdx)); | |||||||||
| 701 | } else { | |||||||||
| 702 | // For array or vector indices, scale the index by the size of the | |||||||||
| 703 | // type. | |||||||||
| 704 | APInt Index = OpC->getValue().sextOrTrunc(Offset.getBitWidth()); | |||||||||
| 705 | GEPOffset += Index * APInt(Offset.getBitWidth(), | |||||||||
| 706 | DL.getTypeAllocSize(GTI.getIndexedType())); | |||||||||
| 707 | } | |||||||||
| 708 | ||||||||||
| 709 | // If this index has computed an intermediate pointer which is not | |||||||||
| 710 | // inbounds, then the result of the GEP is a poison value and we can | |||||||||
| 711 | // delete it and all uses. | |||||||||
| 712 | if (GEPOffset.ugt(AllocSize)) | |||||||||
| 713 | return markAsDead(GEPI); | |||||||||
| 714 | } | |||||||||
| 715 | } | |||||||||
| 716 | ||||||||||
| 717 | return Base::visitGetElementPtrInst(GEPI); | |||||||||
| 718 | } | |||||||||
| 719 | ||||||||||
| 720 | void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset, | |||||||||
| 721 | uint64_t Size, bool IsVolatile) { | |||||||||
| 722 | // We allow splitting of non-volatile loads and stores where the type is an | |||||||||
| 723 | // integer type. These may be used to implement 'memcpy' or other "transfer | |||||||||
| 724 | // of bits" patterns. | |||||||||
| 725 | bool IsSplittable = Ty->isIntegerTy() && !IsVolatile; | |||||||||
| 726 | ||||||||||
| 727 | insertUse(I, Offset, Size, IsSplittable); | |||||||||
| 728 | } | |||||||||
| 729 | ||||||||||
| 730 | void visitLoadInst(LoadInst &LI) { | |||||||||
| 731 | assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&(((!LI.isSimple() || LI.getType()->isSingleValueType()) && "All simple FCA loads should have been pre-split") ? static_cast <void> (0) : __assert_fail ("(!LI.isSimple() || LI.getType()->isSingleValueType()) && \"All simple FCA loads should have been pre-split\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 732, __PRETTY_FUNCTION__)) | |||||||||
| 732 | "All simple FCA loads should have been pre-split")(((!LI.isSimple() || LI.getType()->isSingleValueType()) && "All simple FCA loads should have been pre-split") ? static_cast <void> (0) : __assert_fail ("(!LI.isSimple() || LI.getType()->isSingleValueType()) && \"All simple FCA loads should have been pre-split\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 732, __PRETTY_FUNCTION__)); | |||||||||
| 733 | ||||||||||
| 734 | if (!IsOffsetKnown) | |||||||||
| 735 | return PI.setAborted(&LI); | |||||||||
| 736 | ||||||||||
| 737 | const DataLayout &DL = LI.getModule()->getDataLayout(); | |||||||||
| 738 | uint64_t Size = DL.getTypeStoreSize(LI.getType()); | |||||||||
| 739 | return handleLoadOrStore(LI.getType(), LI, Offset, Size, LI.isVolatile()); | |||||||||
| 740 | } | |||||||||
| 741 | ||||||||||
| 742 | void visitStoreInst(StoreInst &SI) { | |||||||||
| 743 | Value *ValOp = SI.getValueOperand(); | |||||||||
| 744 | if (ValOp == *U) | |||||||||
| 745 | return PI.setEscapedAndAborted(&SI); | |||||||||
| 746 | if (!IsOffsetKnown) | |||||||||
| 747 | return PI.setAborted(&SI); | |||||||||
| 748 | ||||||||||
| 749 | const DataLayout &DL = SI.getModule()->getDataLayout(); | |||||||||
| 750 | uint64_t Size = DL.getTypeStoreSize(ValOp->getType()); | |||||||||
| 751 | ||||||||||
| 752 | // If this memory access can be shown to *statically* extend outside the | |||||||||
| 753 | // bounds of of the allocation, it's behavior is undefined, so simply | |||||||||
| 754 | // ignore it. Note that this is more strict than the generic clamping | |||||||||
| 755 | // behavior of insertUse. We also try to handle cases which might run the | |||||||||
| 756 | // risk of overflow. | |||||||||
| 757 | // FIXME: We should instead consider the pointer to have escaped if this | |||||||||
| 758 | // function is being instrumented for addressing bugs or race conditions. | |||||||||
| 759 | if (Size > AllocSize || Offset.ugt(AllocSize - Size)) { | |||||||||
| 760 | DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offsetdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||||||||
| 761 | << " which extends past the end of the " << AllocSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||||||||
| 762 | << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||||||||
| 763 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | |||||||||
| 764 | << " use: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false); | |||||||||
| 765 | return markAsDead(SI); | |||||||||
| 766 | } | |||||||||
| 767 | ||||||||||
| 768 | assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&(((!SI.isSimple() || ValOp->getType()->isSingleValueType ()) && "All simple FCA stores should have been pre-split" ) ? static_cast<void> (0) : __assert_fail ("(!SI.isSimple() || ValOp->getType()->isSingleValueType()) && \"All simple FCA stores should have been pre-split\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 769, __PRETTY_FUNCTION__)) | |||||||||
| 769 | "All simple FCA stores should have been pre-split")(((!SI.isSimple() || ValOp->getType()->isSingleValueType ()) && "All simple FCA stores should have been pre-split" ) ? static_cast<void> (0) : __assert_fail ("(!SI.isSimple() || ValOp->getType()->isSingleValueType()) && \"All simple FCA stores should have been pre-split\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 769, __PRETTY_FUNCTION__)); | |||||||||
| 770 | handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile()); | |||||||||
| 771 | } | |||||||||
| 772 | ||||||||||
| 773 | void visitMemSetInst(MemSetInst &II) { | |||||||||
| 774 | assert(II.getRawDest() == *U && "Pointer use is not the destination?")((II.getRawDest() == *U && "Pointer use is not the destination?" ) ? static_cast<void> (0) : __assert_fail ("II.getRawDest() == *U && \"Pointer use is not the destination?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 774, __PRETTY_FUNCTION__)); | |||||||||
| 775 | ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); | |||||||||
| 776 | if ((Length && Length->getValue() == 0) || | |||||||||
| 777 | (IsOffsetKnown && Offset.uge(AllocSize))) | |||||||||
| 778 | // Zero-length mem transfer intrinsics can be ignored entirely. | |||||||||
| 779 | return markAsDead(II); | |||||||||
| 780 | ||||||||||
| 781 | if (!IsOffsetKnown) | |||||||||
| 782 | return PI.setAborted(&II); | |||||||||
| 783 | ||||||||||
| 784 | insertUse(II, Offset, Length ? Length->getLimitedValue() | |||||||||
| 785 | : AllocSize - Offset.getLimitedValue(), | |||||||||
| 786 | (bool)Length); | |||||||||
| 787 | } | |||||||||
| 788 | ||||||||||
| 789 | void visitMemTransferInst(MemTransferInst &II) { | |||||||||
| 790 | ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); | |||||||||
| 791 | if (Length && Length->getValue() == 0) | |||||||||
| 792 | // Zero-length mem transfer intrinsics can be ignored entirely. | |||||||||
| 793 | return markAsDead(II); | |||||||||
| 794 | ||||||||||
| 795 | // Because we can visit these intrinsics twice, also check to see if the | |||||||||
| 796 | // first time marked this instruction as dead. If so, skip it. | |||||||||
| 797 | if (VisitedDeadInsts.count(&II)) | |||||||||
| 798 | return; | |||||||||
| 799 | ||||||||||
| 800 | if (!IsOffsetKnown) | |||||||||
| 801 | return PI.setAborted(&II); | |||||||||
| 802 | ||||||||||
| 803 | // This side of the transfer is completely out-of-bounds, and so we can | |||||||||
| 804 | // nuke the entire transfer. However, we also need to nuke the other side | |||||||||
| 805 | // if already added to our partitions. | |||||||||
| 806 | // FIXME: Yet another place we really should bypass this when | |||||||||
| 807 | // instrumenting for ASan. | |||||||||
| 808 | if (Offset.uge(AllocSize)) { | |||||||||
| 809 | SmallDenseMap<Instruction *, unsigned>::iterator MTPI = | |||||||||
| 810 | MemTransferSliceMap.find(&II); | |||||||||
| 811 | if (MTPI != MemTransferSliceMap.end()) | |||||||||
| 812 | AS.Slices[MTPI->second].kill(); | |||||||||
| 813 | return markAsDead(II); | |||||||||
| 814 | } | |||||||||
| 815 | ||||||||||
| 816 | uint64_t RawOffset = Offset.getLimitedValue(); | |||||||||
| 817 | uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset; | |||||||||
| 818 | ||||||||||
| 819 | // Check for the special case where the same exact value is used for both | |||||||||
| 820 | // source and dest. | |||||||||
| 821 | if (*U == II.getRawDest() && *U == II.getRawSource()) { | |||||||||
| 822 | // For non-volatile transfers this is a no-op. | |||||||||
| 823 | if (!II.isVolatile()) | |||||||||
| 824 | return markAsDead(II); | |||||||||
| 825 | ||||||||||
| 826 | return insertUse(II, Offset, Size, /*IsSplittable=*/false); | |||||||||
| 827 | } | |||||||||
| 828 | ||||||||||
| 829 | // If we have seen both source and destination for a mem transfer, then | |||||||||
| 830 | // they both point to the same alloca. | |||||||||
| 831 | bool Inserted; | |||||||||
| 832 | SmallDenseMap<Instruction *, unsigned>::iterator MTPI; | |||||||||
| 833 | std::tie(MTPI, Inserted) = | |||||||||
| 834 | MemTransferSliceMap.insert(std::make_pair(&II, AS.Slices.size())); | |||||||||
| 835 | unsigned PrevIdx = MTPI->second; | |||||||||
| 836 | if (!Inserted) { | |||||||||
| 837 | Slice &PrevP = AS.Slices[PrevIdx]; | |||||||||
| 838 | ||||||||||
| 839 | // Check if the begin offsets match and this is a non-volatile transfer. | |||||||||
| 840 | // In that case, we can completely elide the transfer. | |||||||||
| 841 | if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) { | |||||||||
| 842 | PrevP.kill(); | |||||||||
| 843 | return markAsDead(II); | |||||||||
| 844 | } | |||||||||
| 845 | ||||||||||
| 846 | // Otherwise we have an offset transfer within the same alloca. We can't | |||||||||
| 847 | // split those. | |||||||||
| 848 | PrevP.makeUnsplittable(); | |||||||||
| 849 | } | |||||||||
| 850 | ||||||||||
| 851 | // Insert the use now that we've fixed up the splittable nature. | |||||||||
| 852 | insertUse(II, Offset, Size, /*IsSplittable=*/Inserted && Length); | |||||||||
| 853 | ||||||||||
| 854 | // Check that we ended up with a valid index in the map. | |||||||||
| 855 | assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&((AS.Slices[PrevIdx].getUse()->getUser() == &II && "Map index doesn't point back to a slice with this user.") ? static_cast<void> (0) : __assert_fail ("AS.Slices[PrevIdx].getUse()->getUser() == &II && \"Map index doesn't point back to a slice with this user.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 856, __PRETTY_FUNCTION__)) | |||||||||
| 856 | "Map index doesn't point back to a slice with this user.")((AS.Slices[PrevIdx].getUse()->getUser() == &II && "Map index doesn't point back to a slice with this user.") ? static_cast<void> (0) : __assert_fail ("AS.Slices[PrevIdx].getUse()->getUser() == &II && \"Map index doesn't point back to a slice with this user.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 856, __PRETTY_FUNCTION__)); | |||||||||
| 857 | } | |||||||||
| 858 | ||||||||||
| 859 | // Disable SRoA for any intrinsics except for lifetime invariants. | |||||||||
| 860 | // FIXME: What about debug intrinsics? This matches old behavior, but | |||||||||
| 861 | // doesn't make sense. | |||||||||
| 862 | void visitIntrinsicInst(IntrinsicInst &II) { | |||||||||
| 863 | if (!IsOffsetKnown) | |||||||||
| 864 | return PI.setAborted(&II); | |||||||||
| 865 | ||||||||||
| 866 | if (II.getIntrinsicID() == Intrinsic::lifetime_start || | |||||||||
| 867 | II.getIntrinsicID() == Intrinsic::lifetime_end) { | |||||||||
| 868 | ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0)); | |||||||||
| 869 | uint64_t Size = std::min(AllocSize - Offset.getLimitedValue(), | |||||||||
| 870 | Length->getLimitedValue()); | |||||||||
| 871 | insertUse(II, Offset, Size, true); | |||||||||
| 872 | return; | |||||||||
| 873 | } | |||||||||
| 874 | ||||||||||
| 875 | Base::visitIntrinsicInst(II); | |||||||||
| 876 | } | |||||||||
| 877 | ||||||||||
| 878 | Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) { | |||||||||
| 879 | // We consider any PHI or select that results in a direct load or store of | |||||||||
| 880 | // the same offset to be a viable use for slicing purposes. These uses | |||||||||
| 881 | // are considered unsplittable and the size is the maximum loaded or stored | |||||||||
| 882 | // size. | |||||||||
| 883 | SmallPtrSet<Instruction *, 4> Visited; | |||||||||
| 884 | SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses; | |||||||||
| 885 | Visited.insert(Root); | |||||||||
| 886 | Uses.push_back(std::make_pair(cast<Instruction>(*U), Root)); | |||||||||
| 887 | const DataLayout &DL = Root->getModule()->getDataLayout(); | |||||||||
| 888 | // If there are no loads or stores, the access is dead. We mark that as | |||||||||
| 889 | // a size zero access. | |||||||||
| 890 | Size = 0; | |||||||||
| 891 | do { | |||||||||
| 892 | Instruction *I, *UsedI; | |||||||||
| 893 | std::tie(UsedI, I) = Uses.pop_back_val(); | |||||||||
| 894 | ||||||||||
| 895 | if (LoadInst *LI = dyn_cast<LoadInst>(I)) { | |||||||||
| 896 | Size = std::max(Size, DL.getTypeStoreSize(LI->getType())); | |||||||||
| 897 | continue; | |||||||||
| 898 | } | |||||||||
| 899 | if (StoreInst *SI = dyn_cast<StoreInst>(I)) { | |||||||||
| 900 | Value *Op = SI->getOperand(0); | |||||||||
| 901 | if (Op == UsedI) | |||||||||
| 902 | return SI; | |||||||||
| 903 | Size = std::max(Size, DL.getTypeStoreSize(Op->getType())); | |||||||||
| 904 | continue; | |||||||||
| 905 | } | |||||||||
| 906 | ||||||||||
| 907 | if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) { | |||||||||
| 908 | if (!GEP->hasAllZeroIndices()) | |||||||||
| 909 | return GEP; | |||||||||
| 910 | } else if (!isa<BitCastInst>(I) && !isa<PHINode>(I) && | |||||||||
| 911 | !isa<SelectInst>(I)) { | |||||||||
| 912 | return I; | |||||||||
| 913 | } | |||||||||
| 914 | ||||||||||
| 915 | for (User *U : I->users()) | |||||||||
| 916 | if (Visited.insert(cast<Instruction>(U)).second) | |||||||||
| 917 | Uses.push_back(std::make_pair(I, cast<Instruction>(U))); | |||||||||
| 918 | } while (!Uses.empty()); | |||||||||
| 919 | ||||||||||
| 920 | return nullptr; | |||||||||
| 921 | } | |||||||||
| 922 | ||||||||||
| 923 | void visitPHINodeOrSelectInst(Instruction &I) { | |||||||||
| 924 | assert(isa<PHINode>(I) || isa<SelectInst>(I))((isa<PHINode>(I) || isa<SelectInst>(I)) ? static_cast <void> (0) : __assert_fail ("isa<PHINode>(I) || isa<SelectInst>(I)" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 924, __PRETTY_FUNCTION__)); | |||||||||
| 925 | if (I.use_empty()) | |||||||||
| 926 | return markAsDead(I); | |||||||||
| 927 | ||||||||||
| 928 | // TODO: We could use SimplifyInstruction here to fold PHINodes and | |||||||||
| 929 | // SelectInsts. However, doing so requires to change the current | |||||||||
| 930 | // dead-operand-tracking mechanism. For instance, suppose neither loading | |||||||||
| 931 | // from %U nor %other traps. Then "load (select undef, %U, %other)" does not | |||||||||
| 932 | // trap either. However, if we simply replace %U with undef using the | |||||||||
| 933 | // current dead-operand-tracking mechanism, "load (select undef, undef, | |||||||||
| 934 | // %other)" may trap because the select may return the first operand | |||||||||
| 935 | // "undef". | |||||||||
| 936 | if (Value *Result = foldPHINodeOrSelectInst(I)) { | |||||||||
| 937 | if (Result == *U) | |||||||||
| 938 | // If the result of the constant fold will be the pointer, recurse | |||||||||
| 939 | // through the PHI/select as if we had RAUW'ed it. | |||||||||
| 940 | enqueueUsers(I); | |||||||||
| 941 | else | |||||||||
| 942 | // Otherwise the operand to the PHI/select is dead, and we can replace | |||||||||
| 943 | // it with undef. | |||||||||
| 944 | AS.DeadOperands.push_back(U); | |||||||||
| 945 | ||||||||||
| 946 | return; | |||||||||
| 947 | } | |||||||||
| 948 | ||||||||||
| 949 | if (!IsOffsetKnown) | |||||||||
| 950 | return PI.setAborted(&I); | |||||||||
| 951 | ||||||||||
| 952 | // See if we already have computed info on this node. | |||||||||
| 953 | uint64_t &Size = PHIOrSelectSizes[&I]; | |||||||||
| 954 | if (!Size) { | |||||||||
| 955 | // This is a new PHI/Select, check for an unsafe use of it. | |||||||||
| 956 | if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(&I, Size)) | |||||||||
| 957 | return PI.setAborted(UnsafeI); | |||||||||
| 958 | } | |||||||||
| 959 | ||||||||||
| 960 | // For PHI and select operands outside the alloca, we can't nuke the entire | |||||||||
| 961 | // phi or select -- the other side might still be relevant, so we special | |||||||||
| 962 | // case them here and use a separate structure to track the operands | |||||||||
| 963 | // themselves which should be replaced with undef. | |||||||||
| 964 | // FIXME: This should instead be escaped in the event we're instrumenting | |||||||||
| 965 | // for address sanitization. | |||||||||
| 966 | if (Offset.uge(AllocSize)) { | |||||||||
| 967 | AS.DeadOperands.push_back(U); | |||||||||
| 968 | return; | |||||||||
| 969 | } | |||||||||
| 970 | ||||||||||
| 971 | insertUse(I, Offset, Size); | |||||||||
| 972 | } | |||||||||
| 973 | ||||||||||
| 974 | void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(PN); } | |||||||||
| 975 | ||||||||||
| 976 | void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); } | |||||||||
| 977 | ||||||||||
| 978 | /// \brief Disable SROA entirely if there are unhandled users of the alloca. | |||||||||
| 979 | void visitInstruction(Instruction &I) { PI.setAborted(&I); } | |||||||||
| 980 | }; | |||||||||
| 981 | ||||||||||
| 982 | AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI) | |||||||||
| 983 | : | |||||||||
| 984 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||||||||
| 985 | AI(AI), | |||||||||
| 986 | #endif | |||||||||
| 987 | PointerEscapingInstr(nullptr) { | |||||||||
| 988 | SliceBuilder PB(DL, AI, *this); | |||||||||
| 989 | SliceBuilder::PtrInfo PtrI = PB.visitPtr(AI); | |||||||||
| 990 | if (PtrI.isEscaped() || PtrI.isAborted()) { | |||||||||
| 991 | // FIXME: We should sink the escape vs. abort info into the caller nicely, | |||||||||
| 992 | // possibly by just storing the PtrInfo in the AllocaSlices. | |||||||||
| 993 | PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst() | |||||||||
| 994 | : PtrI.getAbortingInst(); | |||||||||
| 995 | assert(PointerEscapingInstr && "Did not track a bad instruction")((PointerEscapingInstr && "Did not track a bad instruction" ) ? static_cast<void> (0) : __assert_fail ("PointerEscapingInstr && \"Did not track a bad instruction\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 995, __PRETTY_FUNCTION__)); | |||||||||
| 996 | return; | |||||||||
| 997 | } | |||||||||
| 998 | ||||||||||
| 999 | Slices.erase(remove_if(Slices, [](const Slice &S) { return S.isDead(); }), | |||||||||
| 1000 | Slices.end()); | |||||||||
| 1001 | ||||||||||
| 1002 | #ifndef NDEBUG | |||||||||
| 1003 | if (SROARandomShuffleSlices) { | |||||||||
| 1004 | std::mt19937 MT(static_cast<unsigned>( | |||||||||
| 1005 | std::chrono::system_clock::now().time_since_epoch().count())); | |||||||||
| 1006 | std::shuffle(Slices.begin(), Slices.end(), MT); | |||||||||
| 1007 | } | |||||||||
| 1008 | #endif | |||||||||
| 1009 | ||||||||||
| 1010 | // Sort the uses. This arranges for the offsets to be in ascending order, | |||||||||
| 1011 | // and the sizes to be in descending order. | |||||||||
| 1012 | std::sort(Slices.begin(), Slices.end()); | |||||||||
| 1013 | } | |||||||||
| 1014 | ||||||||||
| 1015 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||||||||
| 1016 | ||||||||||
| 1017 | void AllocaSlices::print(raw_ostream &OS, const_iterator I, | |||||||||
| 1018 | StringRef Indent) const { | |||||||||
| 1019 | printSlice(OS, I, Indent); | |||||||||
| 1020 | OS << "\n"; | |||||||||
| 1021 | printUse(OS, I, Indent); | |||||||||
| 1022 | } | |||||||||
| 1023 | ||||||||||
| 1024 | void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I, | |||||||||
| 1025 | StringRef Indent) const { | |||||||||
| 1026 | OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")" | |||||||||
| 1027 | << " slice #" << (I - begin()) | |||||||||
| 1028 | << (I->isSplittable() ? " (splittable)" : ""); | |||||||||
| 1029 | } | |||||||||
| 1030 | ||||||||||
| 1031 | void AllocaSlices::printUse(raw_ostream &OS, const_iterator I, | |||||||||
| 1032 | StringRef Indent) const { | |||||||||
| 1033 | OS << Indent << " used by: " << *I->getUse()->getUser() << "\n"; | |||||||||
| 1034 | } | |||||||||
| 1035 | ||||||||||
| 1036 | void AllocaSlices::print(raw_ostream &OS) const { | |||||||||
| 1037 | if (PointerEscapingInstr) { | |||||||||
| 1038 | OS << "Can't analyze slices for alloca: " << AI << "\n" | |||||||||
| 1039 | << " A pointer to this alloca escaped by:\n" | |||||||||
| 1040 | << " " << *PointerEscapingInstr << "\n"; | |||||||||
| 1041 | return; | |||||||||
| 1042 | } | |||||||||
| 1043 | ||||||||||
| 1044 | OS << "Slices of alloca: " << AI << "\n"; | |||||||||
| 1045 | for (const_iterator I = begin(), E = end(); I != E; ++I) | |||||||||
| 1046 | print(OS, I); | |||||||||
| 1047 | } | |||||||||
| 1048 | ||||||||||
| 1049 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void AllocaSlices::dump(const_iterator I) const { | |||||||||
| 1050 | print(dbgs(), I); | |||||||||
| 1051 | } | |||||||||
| 1052 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void AllocaSlices::dump() const { print(dbgs()); } | |||||||||
| 1053 | ||||||||||
| 1054 | #endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | |||||||||
| 1055 | ||||||||||
| 1056 | /// Walk the range of a partitioning looking for a common type to cover this | |||||||||
| 1057 | /// sequence of slices. | |||||||||
| 1058 | static Type *findCommonType(AllocaSlices::const_iterator B, | |||||||||
| 1059 | AllocaSlices::const_iterator E, | |||||||||
| 1060 | uint64_t EndOffset) { | |||||||||
| 1061 | Type *Ty = nullptr; | |||||||||
| 1062 | bool TyIsCommon = true; | |||||||||
| 1063 | IntegerType *ITy = nullptr; | |||||||||
| 1064 | ||||||||||
| 1065 | // Note that we need to look at *every* alloca slice's Use to ensure we | |||||||||
| 1066 | // always get consistent results regardless of the order of slices. | |||||||||
| 1067 | for (AllocaSlices::const_iterator I = B; I != E; ++I) { | |||||||||
| 1068 | Use *U = I->getUse(); | |||||||||
| 1069 | if (isa<IntrinsicInst>(*U->getUser())) | |||||||||
| 1070 | continue; | |||||||||
| 1071 | if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset) | |||||||||
| 1072 | continue; | |||||||||
| 1073 | ||||||||||
| 1074 | Type *UserTy = nullptr; | |||||||||
| 1075 | if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | |||||||||
| 1076 | UserTy = LI->getType(); | |||||||||
| 1077 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | |||||||||
| 1078 | UserTy = SI->getValueOperand()->getType(); | |||||||||
| 1079 | } | |||||||||
| 1080 | ||||||||||
| 1081 | if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(UserTy)) { | |||||||||
| 1082 | // If the type is larger than the partition, skip it. We only encounter | |||||||||
| 1083 | // this for split integer operations where we want to use the type of the | |||||||||
| 1084 | // entity causing the split. Also skip if the type is not a byte width | |||||||||
| 1085 | // multiple. | |||||||||
| 1086 | if (UserITy->getBitWidth() % 8 != 0 || | |||||||||
| 1087 | UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset())) | |||||||||
| 1088 | continue; | |||||||||
| 1089 | ||||||||||
| 1090 | // Track the largest bitwidth integer type used in this way in case there | |||||||||
| 1091 | // is no common type. | |||||||||
| 1092 | if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth()) | |||||||||
| 1093 | ITy = UserITy; | |||||||||
| 1094 | } | |||||||||
| 1095 | ||||||||||
| 1096 | // To avoid depending on the order of slices, Ty and TyIsCommon must not | |||||||||
| 1097 | // depend on types skipped above. | |||||||||
| 1098 | if (!UserTy || (Ty && Ty != UserTy)) | |||||||||
| 1099 | TyIsCommon = false; // Give up on anything but an iN type. | |||||||||
| 1100 | else | |||||||||
| 1101 | Ty = UserTy; | |||||||||
| 1102 | } | |||||||||
| 1103 | ||||||||||
| 1104 | return TyIsCommon ? Ty : ITy; | |||||||||
| 1105 | } | |||||||||
| 1106 | ||||||||||
| 1107 | /// PHI instructions that use an alloca and are subsequently loaded can be | |||||||||
| 1108 | /// rewritten to load both input pointers in the pred blocks and then PHI the | |||||||||
| 1109 | /// results, allowing the load of the alloca to be promoted. | |||||||||
| 1110 | /// From this: | |||||||||
| 1111 | /// %P2 = phi [i32* %Alloca, i32* %Other] | |||||||||
| 1112 | /// %V = load i32* %P2 | |||||||||
| 1113 | /// to: | |||||||||
| 1114 | /// %V1 = load i32* %Alloca -> will be mem2reg'd | |||||||||
| 1115 | /// ... | |||||||||
| 1116 | /// %V2 = load i32* %Other | |||||||||
| 1117 | /// ... | |||||||||
| 1118 | /// %V = phi [i32 %V1, i32 %V2] | |||||||||
| 1119 | /// | |||||||||
| 1120 | /// We can do this to a select if its only uses are loads and if the operands | |||||||||
| 1121 | /// to the select can be loaded unconditionally. | |||||||||
| 1122 | /// | |||||||||
| 1123 | /// FIXME: This should be hoisted into a generic utility, likely in | |||||||||
| 1124 | /// Transforms/Util/Local.h | |||||||||
| 1125 | static bool isSafePHIToSpeculate(PHINode &PN) { | |||||||||
| 1126 | // For now, we can only do this promotion if the load is in the same block | |||||||||
| 1127 | // as the PHI, and if there are no stores between the phi and load. | |||||||||
| 1128 | // TODO: Allow recursive phi users. | |||||||||
| 1129 | // TODO: Allow stores. | |||||||||
| 1130 | BasicBlock *BB = PN.getParent(); | |||||||||
| 1131 | unsigned MaxAlign = 0; | |||||||||
| 1132 | bool HaveLoad = false; | |||||||||
| 1133 | for (User *U : PN.users()) { | |||||||||
| 1134 | LoadInst *LI = dyn_cast<LoadInst>(U); | |||||||||
| 1135 | if (!LI || !LI->isSimple()) | |||||||||
| 1136 | return false; | |||||||||
| 1137 | ||||||||||
| 1138 | // For now we only allow loads in the same block as the PHI. This is | |||||||||
| 1139 | // a common case that happens when instcombine merges two loads through | |||||||||
| 1140 | // a PHI. | |||||||||
| 1141 | if (LI->getParent() != BB) | |||||||||
| 1142 | return false; | |||||||||
| 1143 | ||||||||||
| 1144 | // Ensure that there are no instructions between the PHI and the load that | |||||||||
| 1145 | // could store. | |||||||||
| 1146 | for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI) | |||||||||
| 1147 | if (BBI->mayWriteToMemory()) | |||||||||
| 1148 | return false; | |||||||||
| 1149 | ||||||||||
| 1150 | MaxAlign = std::max(MaxAlign, LI->getAlignment()); | |||||||||
| 1151 | HaveLoad = true; | |||||||||
| 1152 | } | |||||||||
| 1153 | ||||||||||
| 1154 | if (!HaveLoad) | |||||||||
| 1155 | return false; | |||||||||
| 1156 | ||||||||||
| 1157 | const DataLayout &DL = PN.getModule()->getDataLayout(); | |||||||||
| 1158 | ||||||||||
| 1159 | // We can only transform this if it is safe to push the loads into the | |||||||||
| 1160 | // predecessor blocks. The only thing to watch out for is that we can't put | |||||||||
| 1161 | // a possibly trapping load in the predecessor if it is a critical edge. | |||||||||
| 1162 | for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { | |||||||||
| 1163 | TerminatorInst *TI = PN.getIncomingBlock(Idx)->getTerminator(); | |||||||||
| 1164 | Value *InVal = PN.getIncomingValue(Idx); | |||||||||
| 1165 | ||||||||||
| 1166 | // If the value is produced by the terminator of the predecessor (an | |||||||||
| 1167 | // invoke) or it has side-effects, there is no valid place to put a load | |||||||||
| 1168 | // in the predecessor. | |||||||||
| 1169 | if (TI == InVal || TI->mayHaveSideEffects()) | |||||||||
| 1170 | return false; | |||||||||
| 1171 | ||||||||||
| 1172 | // If the predecessor has a single successor, then the edge isn't | |||||||||
| 1173 | // critical. | |||||||||
| 1174 | if (TI->getNumSuccessors() == 1) | |||||||||
| 1175 | continue; | |||||||||
| 1176 | ||||||||||
| 1177 | // If this pointer is always safe to load, or if we can prove that there | |||||||||
| 1178 | // is already a load in the block, then we can move the load to the pred | |||||||||
| 1179 | // block. | |||||||||
| 1180 | if (isSafeToLoadUnconditionally(InVal, MaxAlign, DL, TI)) | |||||||||
| 1181 | continue; | |||||||||
| 1182 | ||||||||||
| 1183 | return false; | |||||||||
| 1184 | } | |||||||||
| 1185 | ||||||||||
| 1186 | return true; | |||||||||
| 1187 | } | |||||||||
| 1188 | ||||||||||
| 1189 | static void speculatePHINodeLoads(PHINode &PN) { | |||||||||
| 1190 | DEBUG(dbgs() << " original: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << PN << "\n"; } } while (false); | |||||||||
| 1191 | ||||||||||
| 1192 | Type *LoadTy = cast<PointerType>(PN.getType())->getElementType(); | |||||||||
| 1193 | IRBuilderTy PHIBuilder(&PN); | |||||||||
| 1194 | PHINode *NewPN = PHIBuilder.CreatePHI(LoadTy, PN.getNumIncomingValues(), | |||||||||
| 1195 | PN.getName() + ".sroa.speculated"); | |||||||||
| 1196 | ||||||||||
| 1197 | // Get the AA tags and alignment to use from one of the loads. It doesn't | |||||||||
| 1198 | // matter which one we get and if any differ. | |||||||||
| 1199 | LoadInst *SomeLoad = cast<LoadInst>(PN.user_back()); | |||||||||
| 1200 | ||||||||||
| 1201 | AAMDNodes AATags; | |||||||||
| 1202 | SomeLoad->getAAMetadata(AATags); | |||||||||
| 1203 | unsigned Align = SomeLoad->getAlignment(); | |||||||||
| 1204 | ||||||||||
| 1205 | // Rewrite all loads of the PN to use the new PHI. | |||||||||
| 1206 | while (!PN.use_empty()) { | |||||||||
| 1207 | LoadInst *LI = cast<LoadInst>(PN.user_back()); | |||||||||
| 1208 | LI->replaceAllUsesWith(NewPN); | |||||||||
| 1209 | LI->eraseFromParent(); | |||||||||
| 1210 | } | |||||||||
| 1211 | ||||||||||
| 1212 | // Inject loads into all of the pred blocks. | |||||||||
| 1213 | for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { | |||||||||
| 1214 | BasicBlock *Pred = PN.getIncomingBlock(Idx); | |||||||||
| 1215 | TerminatorInst *TI = Pred->getTerminator(); | |||||||||
| 1216 | Value *InVal = PN.getIncomingValue(Idx); | |||||||||
| 1217 | IRBuilderTy PredBuilder(TI); | |||||||||
| 1218 | ||||||||||
| 1219 | LoadInst *Load = PredBuilder.CreateLoad( | |||||||||
| 1220 | InVal, (PN.getName() + ".sroa.speculate.load." + Pred->getName())); | |||||||||
| 1221 | ++NumLoadsSpeculated; | |||||||||
| 1222 | Load->setAlignment(Align); | |||||||||
| 1223 | if (AATags) | |||||||||
| 1224 | Load->setAAMetadata(AATags); | |||||||||
| 1225 | NewPN->addIncoming(Load, Pred); | |||||||||
| 1226 | } | |||||||||
| 1227 | ||||||||||
| 1228 | DEBUG(dbgs() << " speculated to: " << *NewPN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " speculated to: " << *NewPN << "\n"; } } while (false); | |||||||||
| 1229 | PN.eraseFromParent(); | |||||||||
| 1230 | } | |||||||||
| 1231 | ||||||||||
| 1232 | /// Select instructions that use an alloca and are subsequently loaded can be | |||||||||
| 1233 | /// rewritten to load both input pointers and then select between the result, | |||||||||
| 1234 | /// allowing the load of the alloca to be promoted. | |||||||||
| 1235 | /// From this: | |||||||||
| 1236 | /// %P2 = select i1 %cond, i32* %Alloca, i32* %Other | |||||||||
| 1237 | /// %V = load i32* %P2 | |||||||||
| 1238 | /// to: | |||||||||
| 1239 | /// %V1 = load i32* %Alloca -> will be mem2reg'd | |||||||||
| 1240 | /// %V2 = load i32* %Other | |||||||||
| 1241 | /// %V = select i1 %cond, i32 %V1, i32 %V2 | |||||||||
| 1242 | /// | |||||||||
| 1243 | /// We can do this to a select if its only uses are loads and if the operand | |||||||||
| 1244 | /// to the select can be loaded unconditionally. | |||||||||
| 1245 | static bool isSafeSelectToSpeculate(SelectInst &SI) { | |||||||||
| 1246 | Value *TValue = SI.getTrueValue(); | |||||||||
| 1247 | Value *FValue = SI.getFalseValue(); | |||||||||
| 1248 | const DataLayout &DL = SI.getModule()->getDataLayout(); | |||||||||
| 1249 | ||||||||||
| 1250 | for (User *U : SI.users()) { | |||||||||
| 1251 | LoadInst *LI = dyn_cast<LoadInst>(U); | |||||||||
| 1252 | if (!LI || !LI->isSimple()) | |||||||||
| 1253 | return false; | |||||||||
| 1254 | ||||||||||
| 1255 | // Both operands to the select need to be dereferencable, either | |||||||||
| 1256 | // absolutely (e.g. allocas) or at this point because we can see other | |||||||||
| 1257 | // accesses to it. | |||||||||
| 1258 | if (!isSafeToLoadUnconditionally(TValue, LI->getAlignment(), DL, LI)) | |||||||||
| 1259 | return false; | |||||||||
| 1260 | if (!isSafeToLoadUnconditionally(FValue, LI->getAlignment(), DL, LI)) | |||||||||
| 1261 | return false; | |||||||||
| 1262 | } | |||||||||
| 1263 | ||||||||||
| 1264 | return true; | |||||||||
| 1265 | } | |||||||||
| 1266 | ||||||||||
| 1267 | static void speculateSelectInstLoads(SelectInst &SI) { | |||||||||
| 1268 | DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||||||||
| 1269 | ||||||||||
| 1270 | IRBuilderTy IRB(&SI); | |||||||||
| 1271 | Value *TV = SI.getTrueValue(); | |||||||||
| 1272 | Value *FV = SI.getFalseValue(); | |||||||||
| 1273 | // Replace the loads of the select with a select of two loads. | |||||||||
| 1274 | while (!SI.use_empty()) { | |||||||||
| 1275 | LoadInst *LI = cast<LoadInst>(SI.user_back()); | |||||||||
| 1276 | assert(LI->isSimple() && "We only speculate simple loads")((LI->isSimple() && "We only speculate simple loads" ) ? static_cast<void> (0) : __assert_fail ("LI->isSimple() && \"We only speculate simple loads\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1276, __PRETTY_FUNCTION__)); | |||||||||
| 1277 | ||||||||||
| 1278 | IRB.SetInsertPoint(LI); | |||||||||
| 1279 | LoadInst *TL = | |||||||||
| 1280 | IRB.CreateLoad(TV, LI->getName() + ".sroa.speculate.load.true"); | |||||||||
| 1281 | LoadInst *FL = | |||||||||
| 1282 | IRB.CreateLoad(FV, LI->getName() + ".sroa.speculate.load.false"); | |||||||||
| 1283 | NumLoadsSpeculated += 2; | |||||||||
| 1284 | ||||||||||
| 1285 | // Transfer alignment and AA info if present. | |||||||||
| 1286 | TL->setAlignment(LI->getAlignment()); | |||||||||
| 1287 | FL->setAlignment(LI->getAlignment()); | |||||||||
| 1288 | ||||||||||
| 1289 | AAMDNodes Tags; | |||||||||
| 1290 | LI->getAAMetadata(Tags); | |||||||||
| 1291 | if (Tags) { | |||||||||
| 1292 | TL->setAAMetadata(Tags); | |||||||||
| 1293 | FL->setAAMetadata(Tags); | |||||||||
| 1294 | } | |||||||||
| 1295 | ||||||||||
| 1296 | Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, | |||||||||
| 1297 | LI->getName() + ".sroa.speculated"); | |||||||||
| 1298 | ||||||||||
| 1299 | DEBUG(dbgs() << " speculated to: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " speculated to: " << *V << "\n"; } } while (false); | |||||||||
| 1300 | LI->replaceAllUsesWith(V); | |||||||||
| 1301 | LI->eraseFromParent(); | |||||||||
| 1302 | } | |||||||||
| 1303 | SI.eraseFromParent(); | |||||||||
| 1304 | } | |||||||||
| 1305 | ||||||||||
| 1306 | /// \brief Build a GEP out of a base pointer and indices. | |||||||||
| 1307 | /// | |||||||||
| 1308 | /// This will return the BasePtr if that is valid, or build a new GEP | |||||||||
| 1309 | /// instruction using the IRBuilder if GEP-ing is needed. | |||||||||
| 1310 | static Value *buildGEP(IRBuilderTy &IRB, Value *BasePtr, | |||||||||
| 1311 | SmallVectorImpl<Value *> &Indices, Twine NamePrefix) { | |||||||||
| 1312 | if (Indices.empty()) | |||||||||
| 1313 | return BasePtr; | |||||||||
| 1314 | ||||||||||
| 1315 | // A single zero index is a no-op, so check for this and avoid building a GEP | |||||||||
| 1316 | // in that case. | |||||||||
| 1317 | if (Indices.size() == 1 && cast<ConstantInt>(Indices.back())->isZero()) | |||||||||
| 1318 | return BasePtr; | |||||||||
| 1319 | ||||||||||
| 1320 | return IRB.CreateInBoundsGEP(nullptr, BasePtr, Indices, | |||||||||
| 1321 | NamePrefix + "sroa_idx"); | |||||||||
| 1322 | } | |||||||||
| 1323 | ||||||||||
| 1324 | /// \brief Get a natural GEP off of the BasePtr walking through Ty toward | |||||||||
| 1325 | /// TargetTy without changing the offset of the pointer. | |||||||||
| 1326 | /// | |||||||||
| 1327 | /// This routine assumes we've already established a properly offset GEP with | |||||||||
| 1328 | /// Indices, and arrived at the Ty type. The goal is to continue to GEP with | |||||||||
| 1329 | /// zero-indices down through type layers until we find one the same as | |||||||||
| 1330 | /// TargetTy. If we can't find one with the same type, we at least try to use | |||||||||
| 1331 | /// one with the same size. If none of that works, we just produce the GEP as | |||||||||
| 1332 | /// indicated by Indices to have the correct offset. | |||||||||
| 1333 | static Value *getNaturalGEPWithType(IRBuilderTy &IRB, const DataLayout &DL, | |||||||||
| 1334 | Value *BasePtr, Type *Ty, Type *TargetTy, | |||||||||
| 1335 | SmallVectorImpl<Value *> &Indices, | |||||||||
| 1336 | Twine NamePrefix) { | |||||||||
| 1337 | if (Ty == TargetTy) | |||||||||
| 1338 | return buildGEP(IRB, BasePtr, Indices, NamePrefix); | |||||||||
| 1339 | ||||||||||
| 1340 | // Pointer size to use for the indices. | |||||||||
| 1341 | unsigned PtrSize = DL.getPointerTypeSizeInBits(BasePtr->getType()); | |||||||||
| 1342 | ||||||||||
| 1343 | // See if we can descend into a struct and locate a field with the correct | |||||||||
| 1344 | // type. | |||||||||
| 1345 | unsigned NumLayers = 0; | |||||||||
| 1346 | Type *ElementTy = Ty; | |||||||||
| 1347 | do { | |||||||||
| 1348 | if (ElementTy->isPointerTy()) | |||||||||
| 1349 | break; | |||||||||
| 1350 | ||||||||||
| 1351 | if (ArrayType *ArrayTy = dyn_cast<ArrayType>(ElementTy)) { | |||||||||
| 1352 | ElementTy = ArrayTy->getElementType(); | |||||||||
| 1353 | Indices.push_back(IRB.getIntN(PtrSize, 0)); | |||||||||
| 1354 | } else if (VectorType *VectorTy = dyn_cast<VectorType>(ElementTy)) { | |||||||||
| 1355 | ElementTy = VectorTy->getElementType(); | |||||||||
| 1356 | Indices.push_back(IRB.getInt32(0)); | |||||||||
| 1357 | } else if (StructType *STy = dyn_cast<StructType>(ElementTy)) { | |||||||||
| 1358 | if (STy->element_begin() == STy->element_end()) | |||||||||
| 1359 | break; // Nothing left to descend into. | |||||||||
| 1360 | ElementTy = *STy->element_begin(); | |||||||||
| 1361 | Indices.push_back(IRB.getInt32(0)); | |||||||||
| 1362 | } else { | |||||||||
| 1363 | break; | |||||||||
| 1364 | } | |||||||||
| 1365 | ++NumLayers; | |||||||||
| 1366 | } while (ElementTy != TargetTy); | |||||||||
| 1367 | if (ElementTy != TargetTy) | |||||||||
| 1368 | Indices.erase(Indices.end() - NumLayers, Indices.end()); | |||||||||
| 1369 | ||||||||||
| 1370 | return buildGEP(IRB, BasePtr, Indices, NamePrefix); | |||||||||
| 1371 | } | |||||||||
| 1372 | ||||||||||
| 1373 | /// \brief Recursively compute indices for a natural GEP. | |||||||||
| 1374 | /// | |||||||||
| 1375 | /// This is the recursive step for getNaturalGEPWithOffset that walks down the | |||||||||
| 1376 | /// element types adding appropriate indices for the GEP. | |||||||||
| 1377 | static Value *getNaturalGEPRecursively(IRBuilderTy &IRB, const DataLayout &DL, | |||||||||
| 1378 | Value *Ptr, Type *Ty, APInt &Offset, | |||||||||
| 1379 | Type *TargetTy, | |||||||||
| 1380 | SmallVectorImpl<Value *> &Indices, | |||||||||
| 1381 | Twine NamePrefix) { | |||||||||
| 1382 | if (Offset == 0) | |||||||||
| 1383 | return getNaturalGEPWithType(IRB, DL, Ptr, Ty, TargetTy, Indices, | |||||||||
| 1384 | NamePrefix); | |||||||||
| 1385 | ||||||||||
| 1386 | // We can't recurse through pointer types. | |||||||||
| 1387 | if (Ty->isPointerTy()) | |||||||||
| 1388 | return nullptr; | |||||||||
| 1389 | ||||||||||
| 1390 | // We try to analyze GEPs over vectors here, but note that these GEPs are | |||||||||
| 1391 | // extremely poorly defined currently. The long-term goal is to remove GEPing | |||||||||
| 1392 | // over a vector from the IR completely. | |||||||||
| 1393 | if (VectorType *VecTy = dyn_cast<VectorType>(Ty)) { | |||||||||
| 1394 | unsigned ElementSizeInBits = DL.getTypeSizeInBits(VecTy->getScalarType()); | |||||||||
| 1395 | if (ElementSizeInBits % 8 != 0) { | |||||||||
| 1396 | // GEPs over non-multiple of 8 size vector elements are invalid. | |||||||||
| 1397 | return nullptr; | |||||||||
| 1398 | } | |||||||||
| 1399 | APInt ElementSize(Offset.getBitWidth(), ElementSizeInBits / 8); | |||||||||
| 1400 | APInt NumSkippedElements = Offset.sdiv(ElementSize); | |||||||||
| 1401 | if (NumSkippedElements.ugt(VecTy->getNumElements())) | |||||||||
| 1402 | return nullptr; | |||||||||
| 1403 | Offset -= NumSkippedElements * ElementSize; | |||||||||
| 1404 | Indices.push_back(IRB.getInt(NumSkippedElements)); | |||||||||
| 1405 | return getNaturalGEPRecursively(IRB, DL, Ptr, VecTy->getElementType(), | |||||||||
| 1406 | Offset, TargetTy, Indices, NamePrefix); | |||||||||
| 1407 | } | |||||||||
| 1408 | ||||||||||
| 1409 | if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) { | |||||||||
| 1410 | Type *ElementTy = ArrTy->getElementType(); | |||||||||
| 1411 | APInt ElementSize(Offset.getBitWidth(), DL.getTypeAllocSize(ElementTy)); | |||||||||
| 1412 | APInt NumSkippedElements = Offset.sdiv(ElementSize); | |||||||||
| 1413 | if (NumSkippedElements.ugt(ArrTy->getNumElements())) | |||||||||
| 1414 | return nullptr; | |||||||||
| 1415 | ||||||||||
| 1416 | Offset -= NumSkippedElements * ElementSize; | |||||||||
| 1417 | Indices.push_back(IRB.getInt(NumSkippedElements)); | |||||||||
| 1418 | return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy, | |||||||||
| 1419 | Indices, NamePrefix); | |||||||||
| 1420 | } | |||||||||
| 1421 | ||||||||||
| 1422 | StructType *STy = dyn_cast<StructType>(Ty); | |||||||||
| 1423 | if (!STy) | |||||||||
| 1424 | return nullptr; | |||||||||
| 1425 | ||||||||||
| 1426 | const StructLayout *SL = DL.getStructLayout(STy); | |||||||||
| 1427 | uint64_t StructOffset = Offset.getZExtValue(); | |||||||||
| 1428 | if (StructOffset >= SL->getSizeInBytes()) | |||||||||
| 1429 | return nullptr; | |||||||||
| 1430 | unsigned Index = SL->getElementContainingOffset(StructOffset); | |||||||||
| 1431 | Offset -= APInt(Offset.getBitWidth(), SL->getElementOffset(Index)); | |||||||||
| 1432 | Type *ElementTy = STy->getElementType(Index); | |||||||||
| 1433 | if (Offset.uge(DL.getTypeAllocSize(ElementTy))) | |||||||||
| 1434 | return nullptr; // The offset points into alignment padding. | |||||||||
| 1435 | ||||||||||
| 1436 | Indices.push_back(IRB.getInt32(Index)); | |||||||||
| 1437 | return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy, | |||||||||
| 1438 | Indices, NamePrefix); | |||||||||
| 1439 | } | |||||||||
| 1440 | ||||||||||
| 1441 | /// \brief Get a natural GEP from a base pointer to a particular offset and | |||||||||
| 1442 | /// resulting in a particular type. | |||||||||
| 1443 | /// | |||||||||
| 1444 | /// The goal is to produce a "natural" looking GEP that works with the existing | |||||||||
| 1445 | /// composite types to arrive at the appropriate offset and element type for | |||||||||
| 1446 | /// a pointer. TargetTy is the element type the returned GEP should point-to if | |||||||||
| 1447 | /// possible. We recurse by decreasing Offset, adding the appropriate index to | |||||||||
| 1448 | /// Indices, and setting Ty to the result subtype. | |||||||||
| 1449 | /// | |||||||||
| 1450 | /// If no natural GEP can be constructed, this function returns null. | |||||||||
| 1451 | static Value *getNaturalGEPWithOffset(IRBuilderTy &IRB, const DataLayout &DL, | |||||||||
| 1452 | Value *Ptr, APInt Offset, Type *TargetTy, | |||||||||
| 1453 | SmallVectorImpl<Value *> &Indices, | |||||||||
| 1454 | Twine NamePrefix) { | |||||||||
| 1455 | PointerType *Ty = cast<PointerType>(Ptr->getType()); | |||||||||
| 1456 | ||||||||||
| 1457 | // Don't consider any GEPs through an i8* as natural unless the TargetTy is | |||||||||
| 1458 | // an i8. | |||||||||
| 1459 | if (Ty == IRB.getInt8PtrTy(Ty->getAddressSpace()) && TargetTy->isIntegerTy(8)) | |||||||||
| 1460 | return nullptr; | |||||||||
| 1461 | ||||||||||
| 1462 | Type *ElementTy = Ty->getElementType(); | |||||||||
| 1463 | if (!ElementTy->isSized()) | |||||||||
| 1464 | return nullptr; // We can't GEP through an unsized element. | |||||||||
| 1465 | APInt ElementSize(Offset.getBitWidth(), DL.getTypeAllocSize(ElementTy)); | |||||||||
| 1466 | if (ElementSize == 0) | |||||||||
| 1467 | return nullptr; // Zero-length arrays can't help us build a natural GEP. | |||||||||
| 1468 | APInt NumSkippedElements = Offset.sdiv(ElementSize); | |||||||||
| 1469 | ||||||||||
| 1470 | Offset -= NumSkippedElements * ElementSize; | |||||||||
| 1471 | Indices.push_back(IRB.getInt(NumSkippedElements)); | |||||||||
| 1472 | return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy, | |||||||||
| 1473 | Indices, NamePrefix); | |||||||||
| 1474 | } | |||||||||
| 1475 | ||||||||||
| 1476 | /// \brief Compute an adjusted pointer from Ptr by Offset bytes where the | |||||||||
| 1477 | /// resulting pointer has PointerTy. | |||||||||
| 1478 | /// | |||||||||
| 1479 | /// This tries very hard to compute a "natural" GEP which arrives at the offset | |||||||||
| 1480 | /// and produces the pointer type desired. Where it cannot, it will try to use | |||||||||
| 1481 | /// the natural GEP to arrive at the offset and bitcast to the type. Where that | |||||||||
| 1482 | /// fails, it will try to use an existing i8* and GEP to the byte offset and | |||||||||
| 1483 | /// bitcast to the type. | |||||||||
| 1484 | /// | |||||||||
| 1485 | /// The strategy for finding the more natural GEPs is to peel off layers of the | |||||||||
| 1486 | /// pointer, walking back through bit casts and GEPs, searching for a base | |||||||||
| 1487 | /// pointer from which we can compute a natural GEP with the desired | |||||||||
| 1488 | /// properties. The algorithm tries to fold as many constant indices into | |||||||||
| 1489 | /// a single GEP as possible, thus making each GEP more independent of the | |||||||||
| 1490 | /// surrounding code. | |||||||||
| 1491 | static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr, | |||||||||
| 1492 | APInt Offset, Type *PointerTy, Twine NamePrefix) { | |||||||||
| 1493 | // Even though we don't look through PHI nodes, we could be called on an | |||||||||
| 1494 | // instruction in an unreachable block, which may be on a cycle. | |||||||||
| 1495 | SmallPtrSet<Value *, 4> Visited; | |||||||||
| 1496 | Visited.insert(Ptr); | |||||||||
| 1497 | SmallVector<Value *, 4> Indices; | |||||||||
| 1498 | ||||||||||
| 1499 | // We may end up computing an offset pointer that has the wrong type. If we | |||||||||
| 1500 | // never are able to compute one directly that has the correct type, we'll | |||||||||
| 1501 | // fall back to it, so keep it and the base it was computed from around here. | |||||||||
| 1502 | Value *OffsetPtr = nullptr; | |||||||||
| 1503 | Value *OffsetBasePtr; | |||||||||
| 1504 | ||||||||||
| 1505 | // Remember any i8 pointer we come across to re-use if we need to do a raw | |||||||||
| 1506 | // byte offset. | |||||||||
| 1507 | Value *Int8Ptr = nullptr; | |||||||||
| 1508 | APInt Int8PtrOffset(Offset.getBitWidth(), 0); | |||||||||
| 1509 | ||||||||||
| 1510 | Type *TargetTy = PointerTy->getPointerElementType(); | |||||||||
| 1511 | ||||||||||
| 1512 | do { | |||||||||
| 1513 | // First fold any existing GEPs into the offset. | |||||||||
| 1514 | while (GEPOperator *GEP = dyn_cast<GEPOperator>(Ptr)) { | |||||||||
| 1515 | APInt GEPOffset(Offset.getBitWidth(), 0); | |||||||||
| 1516 | if (!GEP->accumulateConstantOffset(DL, GEPOffset)) | |||||||||
| 1517 | break; | |||||||||
| 1518 | Offset += GEPOffset; | |||||||||
| 1519 | Ptr = GEP->getPointerOperand(); | |||||||||
| 1520 | if (!Visited.insert(Ptr).second) | |||||||||
| 1521 | break; | |||||||||
| 1522 | } | |||||||||
| 1523 | ||||||||||
| 1524 | // See if we can perform a natural GEP here. | |||||||||
| 1525 | Indices.clear(); | |||||||||
| 1526 | if (Value *P = getNaturalGEPWithOffset(IRB, DL, Ptr, Offset, TargetTy, | |||||||||
| 1527 | Indices, NamePrefix)) { | |||||||||
| 1528 | // If we have a new natural pointer at the offset, clear out any old | |||||||||
| 1529 | // offset pointer we computed. Unless it is the base pointer or | |||||||||
| 1530 | // a non-instruction, we built a GEP we don't need. Zap it. | |||||||||
| 1531 | if (OffsetPtr && OffsetPtr != OffsetBasePtr) | |||||||||
| 1532 | if (Instruction *I = dyn_cast<Instruction>(OffsetPtr)) { | |||||||||
| 1533 | assert(I->use_empty() && "Built a GEP with uses some how!")((I->use_empty() && "Built a GEP with uses some how!" ) ? static_cast<void> (0) : __assert_fail ("I->use_empty() && \"Built a GEP with uses some how!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1533, __PRETTY_FUNCTION__)); | |||||||||
| 1534 | I->eraseFromParent(); | |||||||||
| 1535 | } | |||||||||
| 1536 | OffsetPtr = P; | |||||||||
| 1537 | OffsetBasePtr = Ptr; | |||||||||
| 1538 | // If we also found a pointer of the right type, we're done. | |||||||||
| 1539 | if (P->getType() == PointerTy) | |||||||||
| 1540 | return P; | |||||||||
| 1541 | } | |||||||||
| 1542 | ||||||||||
| 1543 | // Stash this pointer if we've found an i8*. | |||||||||
| 1544 | if (Ptr->getType()->isIntegerTy(8)) { | |||||||||
| 1545 | Int8Ptr = Ptr; | |||||||||
| 1546 | Int8PtrOffset = Offset; | |||||||||
| 1547 | } | |||||||||
| 1548 | ||||||||||
| 1549 | // Peel off a layer of the pointer and update the offset appropriately. | |||||||||
| 1550 | if (Operator::getOpcode(Ptr) == Instruction::BitCast) { | |||||||||
| 1551 | Ptr = cast<Operator>(Ptr)->getOperand(0); | |||||||||
| 1552 | } else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(Ptr)) { | |||||||||
| 1553 | if (GA->isInterposable()) | |||||||||
| 1554 | break; | |||||||||
| 1555 | Ptr = GA->getAliasee(); | |||||||||
| 1556 | } else { | |||||||||
| 1557 | break; | |||||||||
| 1558 | } | |||||||||
| 1559 | assert(Ptr->getType()->isPointerTy() && "Unexpected operand type!")((Ptr->getType()->isPointerTy() && "Unexpected operand type!" ) ? static_cast<void> (0) : __assert_fail ("Ptr->getType()->isPointerTy() && \"Unexpected operand type!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1559, __PRETTY_FUNCTION__)); | |||||||||
| 1560 | } while (Visited.insert(Ptr).second); | |||||||||
| 1561 | ||||||||||
| 1562 | if (!OffsetPtr) { | |||||||||
| 1563 | if (!Int8Ptr) { | |||||||||
| 1564 | Int8Ptr = IRB.CreateBitCast( | |||||||||
| 1565 | Ptr, IRB.getInt8PtrTy(PointerTy->getPointerAddressSpace()), | |||||||||
| 1566 | NamePrefix + "sroa_raw_cast"); | |||||||||
| 1567 | Int8PtrOffset = Offset; | |||||||||
| 1568 | } | |||||||||
| 1569 | ||||||||||
| 1570 | OffsetPtr = Int8PtrOffset == 0 | |||||||||
| 1571 | ? Int8Ptr | |||||||||
| 1572 | : IRB.CreateInBoundsGEP(IRB.getInt8Ty(), Int8Ptr, | |||||||||
| 1573 | IRB.getInt(Int8PtrOffset), | |||||||||
| 1574 | NamePrefix + "sroa_raw_idx"); | |||||||||
| 1575 | } | |||||||||
| 1576 | Ptr = OffsetPtr; | |||||||||
| 1577 | ||||||||||
| 1578 | // On the off chance we were targeting i8*, guard the bitcast here. | |||||||||
| 1579 | if (Ptr->getType() != PointerTy) | |||||||||
| 1580 | Ptr = IRB.CreateBitCast(Ptr, PointerTy, NamePrefix + "sroa_cast"); | |||||||||
| 1581 | ||||||||||
| 1582 | return Ptr; | |||||||||
| 1583 | } | |||||||||
| 1584 | ||||||||||
| 1585 | /// \brief Compute the adjusted alignment for a load or store from an offset. | |||||||||
| 1586 | static unsigned getAdjustedAlignment(Instruction *I, uint64_t Offset, | |||||||||
| 1587 | const DataLayout &DL) { | |||||||||
| 1588 | unsigned Alignment; | |||||||||
| 1589 | Type *Ty; | |||||||||
| 1590 | if (auto *LI = dyn_cast<LoadInst>(I)) { | |||||||||
| 1591 | Alignment = LI->getAlignment(); | |||||||||
| 1592 | Ty = LI->getType(); | |||||||||
| 1593 | } else if (auto *SI = dyn_cast<StoreInst>(I)) { | |||||||||
| 1594 | Alignment = SI->getAlignment(); | |||||||||
| 1595 | Ty = SI->getValueOperand()->getType(); | |||||||||
| 1596 | } else { | |||||||||
| 1597 | llvm_unreachable("Only loads and stores are allowed!")::llvm::llvm_unreachable_internal("Only loads and stores are allowed!" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1597); | |||||||||
| 1598 | } | |||||||||
| 1599 | ||||||||||
| 1600 | if (!Alignment) | |||||||||
| 1601 | Alignment = DL.getABITypeAlignment(Ty); | |||||||||
| 1602 | ||||||||||
| 1603 | return MinAlign(Alignment, Offset); | |||||||||
| 1604 | } | |||||||||
| 1605 | ||||||||||
| 1606 | /// \brief Test whether we can convert a value from the old to the new type. | |||||||||
| 1607 | /// | |||||||||
| 1608 | /// This predicate should be used to guard calls to convertValue in order to | |||||||||
| 1609 | /// ensure that we only try to convert viable values. The strategy is that we | |||||||||
| 1610 | /// will peel off single element struct and array wrappings to get to an | |||||||||
| 1611 | /// underlying value, and convert that value. | |||||||||
| 1612 | static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) { | |||||||||
| 1613 | if (OldTy == NewTy) | |||||||||
| 1614 | return true; | |||||||||
| 1615 | ||||||||||
| 1616 | // For integer types, we can't handle any bit-width differences. This would | |||||||||
| 1617 | // break both vector conversions with extension and introduce endianness | |||||||||
| 1618 | // issues when in conjunction with loads and stores. | |||||||||
| 1619 | if (isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) { | |||||||||
| 1620 | assert(cast<IntegerType>(OldTy)->getBitWidth() !=((cast<IntegerType>(OldTy)->getBitWidth() != cast< IntegerType>(NewTy)->getBitWidth() && "We can't have the same bitwidth for different int types" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1622, __PRETTY_FUNCTION__)) | |||||||||
| 1621 | cast<IntegerType>(NewTy)->getBitWidth() &&((cast<IntegerType>(OldTy)->getBitWidth() != cast< IntegerType>(NewTy)->getBitWidth() && "We can't have the same bitwidth for different int types" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1622, __PRETTY_FUNCTION__)) | |||||||||
| 1622 | "We can't have the same bitwidth for different int types")((cast<IntegerType>(OldTy)->getBitWidth() != cast< IntegerType>(NewTy)->getBitWidth() && "We can't have the same bitwidth for different int types" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1622, __PRETTY_FUNCTION__)); | |||||||||
| 1623 | return false; | |||||||||
| 1624 | } | |||||||||
| 1625 | ||||||||||
| 1626 | if (DL.getTypeSizeInBits(NewTy) != DL.getTypeSizeInBits(OldTy)) | |||||||||
| 1627 | return false; | |||||||||
| 1628 | if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType()) | |||||||||
| 1629 | return false; | |||||||||
| 1630 | ||||||||||
| 1631 | // We can convert pointers to integers and vice-versa. Same for vectors | |||||||||
| 1632 | // of pointers and integers. | |||||||||
| 1633 | OldTy = OldTy->getScalarType(); | |||||||||
| 1634 | NewTy = NewTy->getScalarType(); | |||||||||
| 1635 | if (NewTy->isPointerTy() || OldTy->isPointerTy()) { | |||||||||
| 1636 | if (NewTy->isPointerTy() && OldTy->isPointerTy()) { | |||||||||
| 1637 | return cast<PointerType>(NewTy)->getPointerAddressSpace() == | |||||||||
| 1638 | cast<PointerType>(OldTy)->getPointerAddressSpace(); | |||||||||
| 1639 | } | |||||||||
| 1640 | if (NewTy->isIntegerTy() || OldTy->isIntegerTy()) | |||||||||
| 1641 | return true; | |||||||||
| 1642 | return false; | |||||||||
| 1643 | } | |||||||||
| 1644 | ||||||||||
| 1645 | return true; | |||||||||
| 1646 | } | |||||||||
| 1647 | ||||||||||
| 1648 | /// \brief Generic routine to convert an SSA value to a value of a different | |||||||||
| 1649 | /// type. | |||||||||
| 1650 | /// | |||||||||
| 1651 | /// This will try various different casting techniques, such as bitcasts, | |||||||||
| 1652 | /// inttoptr, and ptrtoint casts. Use the \c canConvertValue predicate to test | |||||||||
| 1653 | /// two types for viability with this routine. | |||||||||
| 1654 | static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V, | |||||||||
| 1655 | Type *NewTy) { | |||||||||
| 1656 | Type *OldTy = V->getType(); | |||||||||
| 1657 | assert(canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type")((canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type" ) ? static_cast<void> (0) : __assert_fail ("canConvertValue(DL, OldTy, NewTy) && \"Value not convertable to type\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1657, __PRETTY_FUNCTION__)); | |||||||||
| 1658 | ||||||||||
| 1659 | if (OldTy == NewTy) | |||||||||
| 1660 | return V; | |||||||||
| 1661 | ||||||||||
| 1662 | assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&((!(isa<IntegerType>(OldTy) && isa<IntegerType >(NewTy)) && "Integer types must be the exact same to convert." ) ? static_cast<void> (0) : __assert_fail ("!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && \"Integer types must be the exact same to convert.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1663, __PRETTY_FUNCTION__)) | |||||||||
| 1663 | "Integer types must be the exact same to convert.")((!(isa<IntegerType>(OldTy) && isa<IntegerType >(NewTy)) && "Integer types must be the exact same to convert." ) ? static_cast<void> (0) : __assert_fail ("!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && \"Integer types must be the exact same to convert.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1663, __PRETTY_FUNCTION__)); | |||||||||
| 1664 | ||||||||||
| 1665 | // See if we need inttoptr for this type pair. A cast involving both scalars | |||||||||
| 1666 | // and vectors requires and additional bitcast. | |||||||||
| 1667 | if (OldTy->getScalarType()->isIntegerTy() && | |||||||||
| 1668 | NewTy->getScalarType()->isPointerTy()) { | |||||||||
| 1669 | // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8* | |||||||||
| 1670 | if (OldTy->isVectorTy() && !NewTy->isVectorTy()) | |||||||||
| 1671 | return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)), | |||||||||
| 1672 | NewTy); | |||||||||
| 1673 | ||||||||||
| 1674 | // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*> | |||||||||
| 1675 | if (!OldTy->isVectorTy() && NewTy->isVectorTy()) | |||||||||
| 1676 | return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)), | |||||||||
| 1677 | NewTy); | |||||||||
| 1678 | ||||||||||
| 1679 | return IRB.CreateIntToPtr(V, NewTy); | |||||||||
| 1680 | } | |||||||||
| 1681 | ||||||||||
| 1682 | // See if we need ptrtoint for this type pair. A cast involving both scalars | |||||||||
| 1683 | // and vectors requires and additional bitcast. | |||||||||
| 1684 | if (OldTy->getScalarType()->isPointerTy() && | |||||||||
| 1685 | NewTy->getScalarType()->isIntegerTy()) { | |||||||||
| 1686 | // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128 | |||||||||
| 1687 | if (OldTy->isVectorTy() && !NewTy->isVectorTy()) | |||||||||
| 1688 | return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)), | |||||||||
| 1689 | NewTy); | |||||||||
| 1690 | ||||||||||
| 1691 | // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32> | |||||||||
| 1692 | if (!OldTy->isVectorTy() && NewTy->isVectorTy()) | |||||||||
| 1693 | return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)), | |||||||||
| 1694 | NewTy); | |||||||||
| 1695 | ||||||||||
| 1696 | return IRB.CreatePtrToInt(V, NewTy); | |||||||||
| 1697 | } | |||||||||
| 1698 | ||||||||||
| 1699 | return IRB.CreateBitCast(V, NewTy); | |||||||||
| 1700 | } | |||||||||
| 1701 | ||||||||||
| 1702 | /// \brief Test whether the given slice use can be promoted to a vector. | |||||||||
| 1703 | /// | |||||||||
| 1704 | /// This function is called to test each entry in a partition which is slated | |||||||||
| 1705 | /// for a single slice. | |||||||||
| 1706 | static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S, | |||||||||
| 1707 | VectorType *Ty, | |||||||||
| 1708 | uint64_t ElementSize, | |||||||||
| 1709 | const DataLayout &DL) { | |||||||||
| 1710 | // First validate the slice offsets. | |||||||||
| 1711 | uint64_t BeginOffset = | |||||||||
| 1712 | std::max(S.beginOffset(), P.beginOffset()) - P.beginOffset(); | |||||||||
| 1713 | uint64_t BeginIndex = BeginOffset / ElementSize; | |||||||||
| 1714 | if (BeginIndex * ElementSize != BeginOffset || | |||||||||
| 1715 | BeginIndex >= Ty->getNumElements()) | |||||||||
| 1716 | return false; | |||||||||
| 1717 | uint64_t EndOffset = | |||||||||
| 1718 | std::min(S.endOffset(), P.endOffset()) - P.beginOffset(); | |||||||||
| 1719 | uint64_t EndIndex = EndOffset / ElementSize; | |||||||||
| 1720 | if (EndIndex * ElementSize != EndOffset || EndIndex > Ty->getNumElements()) | |||||||||
| 1721 | return false; | |||||||||
| 1722 | ||||||||||
| 1723 | assert(EndIndex > BeginIndex && "Empty vector!")((EndIndex > BeginIndex && "Empty vector!") ? static_cast <void> (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1723, __PRETTY_FUNCTION__)); | |||||||||
| 1724 | uint64_t NumElements = EndIndex - BeginIndex; | |||||||||
| 1725 | Type *SliceTy = (NumElements == 1) | |||||||||
| 1726 | ? Ty->getElementType() | |||||||||
| 1727 | : VectorType::get(Ty->getElementType(), NumElements); | |||||||||
| 1728 | ||||||||||
| 1729 | Type *SplitIntTy = | |||||||||
| 1730 | Type::getIntNTy(Ty->getContext(), NumElements * ElementSize * 8); | |||||||||
| 1731 | ||||||||||
| 1732 | Use *U = S.getUse(); | |||||||||
| 1733 | ||||||||||
| 1734 | if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { | |||||||||
| 1735 | if (MI->isVolatile()) | |||||||||
| 1736 | return false; | |||||||||
| 1737 | if (!S.isSplittable()) | |||||||||
| 1738 | return false; // Skip any unsplittable intrinsics. | |||||||||
| 1739 | } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) { | |||||||||
| 1740 | if (II->getIntrinsicID() != Intrinsic::lifetime_start && | |||||||||
| 1741 | II->getIntrinsicID() != Intrinsic::lifetime_end) | |||||||||
| 1742 | return false; | |||||||||
| 1743 | } else if (U->get()->getType()->getPointerElementType()->isStructTy()) { | |||||||||
| 1744 | // Disable vector promotion when there are loads or stores of an FCA. | |||||||||
| 1745 | return false; | |||||||||
| 1746 | } else if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | |||||||||
| 1747 | if (LI->isVolatile()) | |||||||||
| 1748 | return false; | |||||||||
| 1749 | Type *LTy = LI->getType(); | |||||||||
| 1750 | if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) { | |||||||||
| 1751 | assert(LTy->isIntegerTy())((LTy->isIntegerTy()) ? static_cast<void> (0) : __assert_fail ("LTy->isIntegerTy()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1751, __PRETTY_FUNCTION__)); | |||||||||
| 1752 | LTy = SplitIntTy; | |||||||||
| 1753 | } | |||||||||
| 1754 | if (!canConvertValue(DL, SliceTy, LTy)) | |||||||||
| 1755 | return false; | |||||||||
| 1756 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | |||||||||
| 1757 | if (SI->isVolatile()) | |||||||||
| 1758 | return false; | |||||||||
| 1759 | Type *STy = SI->getValueOperand()->getType(); | |||||||||
| 1760 | if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) { | |||||||||
| 1761 | assert(STy->isIntegerTy())((STy->isIntegerTy()) ? static_cast<void> (0) : __assert_fail ("STy->isIntegerTy()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1761, __PRETTY_FUNCTION__)); | |||||||||
| 1762 | STy = SplitIntTy; | |||||||||
| 1763 | } | |||||||||
| 1764 | if (!canConvertValue(DL, STy, SliceTy)) | |||||||||
| 1765 | return false; | |||||||||
| 1766 | } else { | |||||||||
| 1767 | return false; | |||||||||
| 1768 | } | |||||||||
| 1769 | ||||||||||
| 1770 | return true; | |||||||||
| 1771 | } | |||||||||
| 1772 | ||||||||||
| 1773 | /// \brief Test whether the given alloca partitioning and range of slices can be | |||||||||
| 1774 | /// promoted to a vector. | |||||||||
| 1775 | /// | |||||||||
| 1776 | /// This is a quick test to check whether we can rewrite a particular alloca | |||||||||
| 1777 | /// partition (and its newly formed alloca) into a vector alloca with only | |||||||||
| 1778 | /// whole-vector loads and stores such that it could be promoted to a vector | |||||||||
| 1779 | /// SSA value. We only can ensure this for a limited set of operations, and we | |||||||||
| 1780 | /// don't want to do the rewrites unless we are confident that the result will | |||||||||
| 1781 | /// be promotable, so we have an early test here. | |||||||||
| 1782 | static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) { | |||||||||
| 1783 | // Collect the candidate types for vector-based promotion. Also track whether | |||||||||
| 1784 | // we have different element types. | |||||||||
| 1785 | SmallVector<VectorType *, 4> CandidateTys; | |||||||||
| 1786 | Type *CommonEltTy = nullptr; | |||||||||
| 1787 | bool HaveCommonEltTy = true; | |||||||||
| 1788 | auto CheckCandidateType = [&](Type *Ty) { | |||||||||
| 1789 | if (auto *VTy = dyn_cast<VectorType>(Ty)) { | |||||||||
| 1790 | CandidateTys.push_back(VTy); | |||||||||
| 1791 | if (!CommonEltTy) | |||||||||
| 1792 | CommonEltTy = VTy->getElementType(); | |||||||||
| 1793 | else if (CommonEltTy != VTy->getElementType()) | |||||||||
| 1794 | HaveCommonEltTy = false; | |||||||||
| 1795 | } | |||||||||
| 1796 | }; | |||||||||
| 1797 | // Consider any loads or stores that are the exact size of the slice. | |||||||||
| 1798 | for (const Slice &S : P) | |||||||||
| 1799 | if (S.beginOffset() == P.beginOffset() && | |||||||||
| 1800 | S.endOffset() == P.endOffset()) { | |||||||||
| 1801 | if (auto *LI = dyn_cast<LoadInst>(S.getUse()->getUser())) | |||||||||
| 1802 | CheckCandidateType(LI->getType()); | |||||||||
| 1803 | else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser())) | |||||||||
| 1804 | CheckCandidateType(SI->getValueOperand()->getType()); | |||||||||
| 1805 | } | |||||||||
| 1806 | ||||||||||
| 1807 | // If we didn't find a vector type, nothing to do here. | |||||||||
| 1808 | if (CandidateTys.empty()) | |||||||||
| 1809 | return nullptr; | |||||||||
| 1810 | ||||||||||
| 1811 | // Remove non-integer vector types if we had multiple common element types. | |||||||||
| 1812 | // FIXME: It'd be nice to replace them with integer vector types, but we can't | |||||||||
| 1813 | // do that until all the backends are known to produce good code for all | |||||||||
| 1814 | // integer vector types. | |||||||||
| 1815 | if (!HaveCommonEltTy) { | |||||||||
| 1816 | CandidateTys.erase(remove_if(CandidateTys, | |||||||||
| 1817 | [](VectorType *VTy) { | |||||||||
| 1818 | return !VTy->getElementType()->isIntegerTy(); | |||||||||
| 1819 | }), | |||||||||
| 1820 | CandidateTys.end()); | |||||||||
| 1821 | ||||||||||
| 1822 | // If there were no integer vector types, give up. | |||||||||
| 1823 | if (CandidateTys.empty()) | |||||||||
| 1824 | return nullptr; | |||||||||
| 1825 | ||||||||||
| 1826 | // Rank the remaining candidate vector types. This is easy because we know | |||||||||
| 1827 | // they're all integer vectors. We sort by ascending number of elements. | |||||||||
| 1828 | auto RankVectorTypes = [&DL](VectorType *RHSTy, VectorType *LHSTy) { | |||||||||
| 1829 | (void)DL; | |||||||||
| 1830 | assert(DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) &&((DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) && "Cannot have vector types of different sizes!") ? static_cast <void> (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) && \"Cannot have vector types of different sizes!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1831, __PRETTY_FUNCTION__)) | |||||||||
| 1831 | "Cannot have vector types of different sizes!")((DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) && "Cannot have vector types of different sizes!") ? static_cast <void> (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) && \"Cannot have vector types of different sizes!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1831, __PRETTY_FUNCTION__)); | |||||||||
| 1832 | assert(RHSTy->getElementType()->isIntegerTy() &&((RHSTy->getElementType()->isIntegerTy() && "All non-integer types eliminated!" ) ? static_cast<void> (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1833, __PRETTY_FUNCTION__)) | |||||||||
| 1833 | "All non-integer types eliminated!")((RHSTy->getElementType()->isIntegerTy() && "All non-integer types eliminated!" ) ? static_cast<void> (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1833, __PRETTY_FUNCTION__)); | |||||||||
| 1834 | assert(LHSTy->getElementType()->isIntegerTy() &&((LHSTy->getElementType()->isIntegerTy() && "All non-integer types eliminated!" ) ? static_cast<void> (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1835, __PRETTY_FUNCTION__)) | |||||||||
| 1835 | "All non-integer types eliminated!")((LHSTy->getElementType()->isIntegerTy() && "All non-integer types eliminated!" ) ? static_cast<void> (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1835, __PRETTY_FUNCTION__)); | |||||||||
| 1836 | return RHSTy->getNumElements() < LHSTy->getNumElements(); | |||||||||
| 1837 | }; | |||||||||
| 1838 | std::sort(CandidateTys.begin(), CandidateTys.end(), RankVectorTypes); | |||||||||
| 1839 | CandidateTys.erase( | |||||||||
| 1840 | std::unique(CandidateTys.begin(), CandidateTys.end(), RankVectorTypes), | |||||||||
| 1841 | CandidateTys.end()); | |||||||||
| 1842 | } else { | |||||||||
| 1843 | // The only way to have the same element type in every vector type is to | |||||||||
| 1844 | // have the same vector type. Check that and remove all but one. | |||||||||
| 1845 | #ifndef NDEBUG | |||||||||
| 1846 | for (VectorType *VTy : CandidateTys) { | |||||||||
| 1847 | assert(VTy->getElementType() == CommonEltTy &&((VTy->getElementType() == CommonEltTy && "Unaccounted for element type!" ) ? static_cast<void> (0) : __assert_fail ("VTy->getElementType() == CommonEltTy && \"Unaccounted for element type!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1848, __PRETTY_FUNCTION__)) | |||||||||
| 1848 | "Unaccounted for element type!")((VTy->getElementType() == CommonEltTy && "Unaccounted for element type!" ) ? static_cast<void> (0) : __assert_fail ("VTy->getElementType() == CommonEltTy && \"Unaccounted for element type!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1848, __PRETTY_FUNCTION__)); | |||||||||
| 1849 | assert(VTy == CandidateTys[0] &&((VTy == CandidateTys[0] && "Different vector types with the same element type!" ) ? static_cast<void> (0) : __assert_fail ("VTy == CandidateTys[0] && \"Different vector types with the same element type!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1850, __PRETTY_FUNCTION__)) | |||||||||
| 1850 | "Different vector types with the same element type!")((VTy == CandidateTys[0] && "Different vector types with the same element type!" ) ? static_cast<void> (0) : __assert_fail ("VTy == CandidateTys[0] && \"Different vector types with the same element type!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1850, __PRETTY_FUNCTION__)); | |||||||||
| 1851 | } | |||||||||
| 1852 | #endif | |||||||||
| 1853 | CandidateTys.resize(1); | |||||||||
| 1854 | } | |||||||||
| 1855 | ||||||||||
| 1856 | // Try each vector type, and return the one which works. | |||||||||
| 1857 | auto CheckVectorTypeForPromotion = [&](VectorType *VTy) { | |||||||||
| 1858 | uint64_t ElementSize = DL.getTypeSizeInBits(VTy->getElementType()); | |||||||||
| 1859 | ||||||||||
| 1860 | // While the definition of LLVM vectors is bitpacked, we don't support sizes | |||||||||
| 1861 | // that aren't byte sized. | |||||||||
| 1862 | if (ElementSize % 8) | |||||||||
| 1863 | return false; | |||||||||
| 1864 | assert((DL.getTypeSizeInBits(VTy) % 8) == 0 &&(((DL.getTypeSizeInBits(VTy) % 8) == 0 && "vector size not a multiple of element size?" ) ? static_cast<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(VTy) % 8) == 0 && \"vector size not a multiple of element size?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1865, __PRETTY_FUNCTION__)) | |||||||||
| 1865 | "vector size not a multiple of element size?")(((DL.getTypeSizeInBits(VTy) % 8) == 0 && "vector size not a multiple of element size?" ) ? static_cast<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(VTy) % 8) == 0 && \"vector size not a multiple of element size?\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 1865, __PRETTY_FUNCTION__)); | |||||||||
| 1866 | ElementSize /= 8; | |||||||||
| 1867 | ||||||||||
| 1868 | for (const Slice &S : P) | |||||||||
| 1869 | if (!isVectorPromotionViableForSlice(P, S, VTy, ElementSize, DL)) | |||||||||
| 1870 | return false; | |||||||||
| 1871 | ||||||||||
| 1872 | for (const Slice *S : P.splitSliceTails()) | |||||||||
| 1873 | if (!isVectorPromotionViableForSlice(P, *S, VTy, ElementSize, DL)) | |||||||||
| 1874 | return false; | |||||||||
| 1875 | ||||||||||
| 1876 | return true; | |||||||||
| 1877 | }; | |||||||||
| 1878 | for (VectorType *VTy : CandidateTys) | |||||||||
| 1879 | if (CheckVectorTypeForPromotion(VTy)) | |||||||||
| 1880 | return VTy; | |||||||||
| 1881 | ||||||||||
| 1882 | return nullptr; | |||||||||
| 1883 | } | |||||||||
| 1884 | ||||||||||
| 1885 | /// \brief Test whether a slice of an alloca is valid for integer widening. | |||||||||
| 1886 | /// | |||||||||
| 1887 | /// This implements the necessary checking for the \c isIntegerWideningViable | |||||||||
| 1888 | /// test below on a single slice of the alloca. | |||||||||
| 1889 | static bool isIntegerWideningViableForSlice(const Slice &S, | |||||||||
| 1890 | uint64_t AllocBeginOffset, | |||||||||
| 1891 | Type *AllocaTy, | |||||||||
| 1892 | const DataLayout &DL, | |||||||||
| 1893 | bool &WholeAllocaOp) { | |||||||||
| 1894 | uint64_t Size = DL.getTypeStoreSize(AllocaTy); | |||||||||
| 1895 | ||||||||||
| 1896 | uint64_t RelBegin = S.beginOffset() - AllocBeginOffset; | |||||||||
| 1897 | uint64_t RelEnd = S.endOffset() - AllocBeginOffset; | |||||||||
| 1898 | ||||||||||
| 1899 | // We can't reasonably handle cases where the load or store extends past | |||||||||
| 1900 | // the end of the alloca's type and into its padding. | |||||||||
| 1901 | if (RelEnd > Size) | |||||||||
| 1902 | return false; | |||||||||
| 1903 | ||||||||||
| 1904 | Use *U = S.getUse(); | |||||||||
| 1905 | ||||||||||
| 1906 | if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | |||||||||
| 1907 | if (LI->isVolatile()) | |||||||||
| 1908 | return false; | |||||||||
| 1909 | // We can't handle loads that extend past the allocated memory. | |||||||||
| 1910 | if (DL.getTypeStoreSize(LI->getType()) > Size) | |||||||||
| 1911 | return false; | |||||||||
| 1912 | // Note that we don't count vector loads or stores as whole-alloca | |||||||||
| 1913 | // operations which enable integer widening because we would prefer to use | |||||||||
| 1914 | // vector widening instead. | |||||||||
| 1915 | if (!isa<VectorType>(LI->getType()) && RelBegin == 0 && RelEnd == Size) | |||||||||
| 1916 | WholeAllocaOp = true; | |||||||||
| 1917 | if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) { | |||||||||
| 1918 | if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy)) | |||||||||
| 1919 | return false; | |||||||||
| 1920 | } else if (RelBegin != 0 || RelEnd != Size || | |||||||||
| 1921 | !canConvertValue(DL, AllocaTy, LI->getType())) { | |||||||||
| 1922 | // Non-integer loads need to be convertible from the alloca type so that | |||||||||
| 1923 | // they are promotable. | |||||||||
| 1924 | return false; | |||||||||
| 1925 | } | |||||||||
| 1926 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | |||||||||
| 1927 | Type *ValueTy = SI->getValueOperand()->getType(); | |||||||||
| 1928 | if (SI->isVolatile()) | |||||||||
| 1929 | return false; | |||||||||
| 1930 | // We can't handle stores that extend past the allocated memory. | |||||||||
| 1931 | if (DL.getTypeStoreSize(ValueTy) > Size) | |||||||||
| 1932 | return false; | |||||||||
| 1933 | // Note that we don't count vector loads or stores as whole-alloca | |||||||||
| 1934 | // operations which enable integer widening because we would prefer to use | |||||||||
| 1935 | // vector widening instead. | |||||||||
| 1936 | if (!isa<VectorType>(ValueTy) && RelBegin == 0 && RelEnd == Size) | |||||||||
| 1937 | WholeAllocaOp = true; | |||||||||
| 1938 | if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) { | |||||||||
| 1939 | if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy)) | |||||||||
| 1940 | return false; | |||||||||
| 1941 | } else if (RelBegin != 0 || RelEnd != Size || | |||||||||
| 1942 | !canConvertValue(DL, ValueTy, AllocaTy)) { | |||||||||
| 1943 | // Non-integer stores need to be convertible to the alloca type so that | |||||||||
| 1944 | // they are promotable. | |||||||||
| 1945 | return false; | |||||||||
| 1946 | } | |||||||||
| 1947 | } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { | |||||||||
| 1948 | if (MI->isVolatile() || !isa<Constant>(MI->getLength())) | |||||||||
| 1949 | return false; | |||||||||
| 1950 | if (!S.isSplittable()) | |||||||||
| 1951 | return false; // Skip any unsplittable intrinsics. | |||||||||
| 1952 | } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) { | |||||||||
| 1953 | if (II->getIntrinsicID() != Intrinsic::lifetime_start && | |||||||||
| 1954 | II->getIntrinsicID() != Intrinsic::lifetime_end) | |||||||||
| 1955 | return false; | |||||||||
| 1956 | } else { | |||||||||
| 1957 | return false; | |||||||||
| 1958 | } | |||||||||
| 1959 | ||||||||||
| 1960 | return true; | |||||||||
| 1961 | } | |||||||||
| 1962 | ||||||||||
| 1963 | /// \brief Test whether the given alloca partition's integer operations can be | |||||||||
| 1964 | /// widened to promotable ones. | |||||||||
| 1965 | /// | |||||||||
| 1966 | /// This is a quick test to check whether we can rewrite the integer loads and | |||||||||
| 1967 | /// stores to a particular alloca into wider loads and stores and be able to | |||||||||
| 1968 | /// promote the resulting alloca. | |||||||||
| 1969 | static bool isIntegerWideningViable(Partition &P, Type *AllocaTy, | |||||||||
| 1970 | const DataLayout &DL) { | |||||||||
| 1971 | uint64_t SizeInBits = DL.getTypeSizeInBits(AllocaTy); | |||||||||
| 1972 | // Don't create integer types larger than the maximum bitwidth. | |||||||||
| 1973 | if (SizeInBits > IntegerType::MAX_INT_BITS) | |||||||||
| 1974 | return false; | |||||||||
| 1975 | ||||||||||
| 1976 | // Don't try to handle allocas with bit-padding. | |||||||||
| 1977 | if (SizeInBits != DL.getTypeStoreSizeInBits(AllocaTy)) | |||||||||
| 1978 | return false; | |||||||||
| 1979 | ||||||||||
| 1980 | // We need to ensure that an integer type with the appropriate bitwidth can | |||||||||
| 1981 | // be converted to the alloca type, whatever that is. We don't want to force | |||||||||
| 1982 | // the alloca itself to have an integer type if there is a more suitable one. | |||||||||
| 1983 | Type *IntTy = Type::getIntNTy(AllocaTy->getContext(), SizeInBits); | |||||||||
| 1984 | if (!canConvertValue(DL, AllocaTy, IntTy) || | |||||||||
| 1985 | !canConvertValue(DL, IntTy, AllocaTy)) | |||||||||
| 1986 | return false; | |||||||||
| 1987 | ||||||||||
| 1988 | // While examining uses, we ensure that the alloca has a covering load or | |||||||||
| 1989 | // store. We don't want to widen the integer operations only to fail to | |||||||||
| 1990 | // promote due to some other unsplittable entry (which we may make splittable | |||||||||
| 1991 | // later). However, if there are only splittable uses, go ahead and assume | |||||||||
| 1992 | // that we cover the alloca. | |||||||||
| 1993 | // FIXME: We shouldn't consider split slices that happen to start in the | |||||||||
| 1994 | // partition here... | |||||||||
| 1995 | bool WholeAllocaOp = | |||||||||
| 1996 | P.begin() != P.end() ? false : DL.isLegalInteger(SizeInBits); | |||||||||
| 1997 | ||||||||||
| 1998 | for (const Slice &S : P) | |||||||||
| 1999 | if (!isIntegerWideningViableForSlice(S, P.beginOffset(), AllocaTy, DL, | |||||||||
| 2000 | WholeAllocaOp)) | |||||||||
| 2001 | return false; | |||||||||
| 2002 | ||||||||||
| 2003 | for (const Slice *S : P.splitSliceTails()) | |||||||||
| 2004 | if (!isIntegerWideningViableForSlice(*S, P.beginOffset(), AllocaTy, DL, | |||||||||
| 2005 | WholeAllocaOp)) | |||||||||
| 2006 | return false; | |||||||||
| 2007 | ||||||||||
| 2008 | return WholeAllocaOp; | |||||||||
| 2009 | } | |||||||||
| 2010 | ||||||||||
| 2011 | static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V, | |||||||||
| 2012 | IntegerType *Ty, uint64_t Offset, | |||||||||
| 2013 | const Twine &Name) { | |||||||||
| 2014 | DEBUG(dbgs() << " start: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " start: " << *V << "\n"; } } while (false); | |||||||||
| 2015 | IntegerType *IntTy = cast<IntegerType>(V->getType()); | |||||||||
| 2016 | assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&((DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize( IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2017, __PRETTY_FUNCTION__)) | |||||||||
| 2017 | "Element extends past full value")((DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize( IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2017, __PRETTY_FUNCTION__)); | |||||||||
| 2018 | uint64_t ShAmt = 8 * Offset; | |||||||||
| 2019 | if (DL.isBigEndian()) | |||||||||
| 2020 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset); | |||||||||
| 2021 | if (ShAmt) { | |||||||||
| 2022 | V = IRB.CreateLShr(V, ShAmt, Name + ".shift"); | |||||||||
| 2023 | DEBUG(dbgs() << " shifted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shifted: " << *V << "\n"; } } while (false); | |||||||||
| 2024 | } | |||||||||
| 2025 | assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&((Ty->getBitWidth() <= IntTy->getBitWidth() && "Cannot extract to a larger integer!") ? static_cast<void > (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2026, __PRETTY_FUNCTION__)) | |||||||||
| 2026 | "Cannot extract to a larger integer!")((Ty->getBitWidth() <= IntTy->getBitWidth() && "Cannot extract to a larger integer!") ? static_cast<void > (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2026, __PRETTY_FUNCTION__)); | |||||||||
| 2027 | if (Ty != IntTy) { | |||||||||
| 2028 | V = IRB.CreateTrunc(V, Ty, Name + ".trunc"); | |||||||||
| 2029 | DEBUG(dbgs() << " trunced: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " trunced: " << *V << "\n"; } } while (false); | |||||||||
| 2030 | } | |||||||||
| 2031 | return V; | |||||||||
| 2032 | } | |||||||||
| 2033 | ||||||||||
| 2034 | static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old, | |||||||||
| 2035 | Value *V, uint64_t Offset, const Twine &Name) { | |||||||||
| 2036 | IntegerType *IntTy = cast<IntegerType>(Old->getType()); | |||||||||
| 2037 | IntegerType *Ty = cast<IntegerType>(V->getType()); | |||||||||
| 2038 | assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&((Ty->getBitWidth() <= IntTy->getBitWidth() && "Cannot insert a larger integer!") ? static_cast<void> (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot insert a larger integer!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2039, __PRETTY_FUNCTION__)) | |||||||||
| 2039 | "Cannot insert a larger integer!")((Ty->getBitWidth() <= IntTy->getBitWidth() && "Cannot insert a larger integer!") ? static_cast<void> (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot insert a larger integer!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2039, __PRETTY_FUNCTION__)); | |||||||||
| 2040 | DEBUG(dbgs() << " start: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " start: " << *V << "\n"; } } while (false); | |||||||||
| 2041 | if (Ty != IntTy) { | |||||||||
| 2042 | V = IRB.CreateZExt(V, IntTy, Name + ".ext"); | |||||||||
| 2043 | DEBUG(dbgs() << " extended: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " extended: " << *V << "\n"; } } while (false); | |||||||||
| 2044 | } | |||||||||
| 2045 | assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&((DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize( IntTy) && "Element store outside of alloca store") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element store outside of alloca store\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2046, __PRETTY_FUNCTION__)) | |||||||||
| 2046 | "Element store outside of alloca store")((DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize( IntTy) && "Element store outside of alloca store") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element store outside of alloca store\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2046, __PRETTY_FUNCTION__)); | |||||||||
| 2047 | uint64_t ShAmt = 8 * Offset; | |||||||||
| 2048 | if (DL.isBigEndian()) | |||||||||
| 2049 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset); | |||||||||
| 2050 | if (ShAmt) { | |||||||||
| 2051 | V = IRB.CreateShl(V, ShAmt, Name + ".shift"); | |||||||||
| 2052 | DEBUG(dbgs() << " shifted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shifted: " << *V << "\n"; } } while (false); | |||||||||
| 2053 | } | |||||||||
| 2054 | ||||||||||
| 2055 | if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) { | |||||||||
| 2056 | APInt Mask = ~Ty->getMask().zext(IntTy->getBitWidth()).shl(ShAmt); | |||||||||
| 2057 | Old = IRB.CreateAnd(Old, Mask, Name + ".mask"); | |||||||||
| 2058 | DEBUG(dbgs() << " masked: " << *Old << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " masked: " << *Old << "\n"; } } while (false); | |||||||||
| 2059 | V = IRB.CreateOr(Old, V, Name + ".insert"); | |||||||||
| 2060 | DEBUG(dbgs() << " inserted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " inserted: " << *V << "\n"; } } while (false); | |||||||||
| 2061 | } | |||||||||
| 2062 | return V; | |||||||||
| 2063 | } | |||||||||
| 2064 | ||||||||||
| 2065 | static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex, | |||||||||
| 2066 | unsigned EndIndex, const Twine &Name) { | |||||||||
| 2067 | VectorType *VecTy = cast<VectorType>(V->getType()); | |||||||||
| 2068 | unsigned NumElements = EndIndex - BeginIndex; | |||||||||
| 2069 | assert(NumElements <= VecTy->getNumElements() && "Too many elements!")((NumElements <= VecTy->getNumElements() && "Too many elements!" ) ? static_cast<void> (0) : __assert_fail ("NumElements <= VecTy->getNumElements() && \"Too many elements!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2069, __PRETTY_FUNCTION__)); | |||||||||
| 2070 | ||||||||||
| 2071 | if (NumElements == VecTy->getNumElements()) | |||||||||
| 2072 | return V; | |||||||||
| 2073 | ||||||||||
| 2074 | if (NumElements == 1) { | |||||||||
| 2075 | V = IRB.CreateExtractElement(V, IRB.getInt32(BeginIndex), | |||||||||
| 2076 | Name + ".extract"); | |||||||||
| 2077 | DEBUG(dbgs() << " extract: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " extract: " << *V << "\n"; } } while (false); | |||||||||
| 2078 | return V; | |||||||||
| 2079 | } | |||||||||
| 2080 | ||||||||||
| 2081 | SmallVector<Constant *, 8> Mask; | |||||||||
| 2082 | Mask.reserve(NumElements); | |||||||||
| 2083 | for (unsigned i = BeginIndex; i != EndIndex; ++i) | |||||||||
| 2084 | Mask.push_back(IRB.getInt32(i)); | |||||||||
| 2085 | V = IRB.CreateShuffleVector(V, UndefValue::get(V->getType()), | |||||||||
| 2086 | ConstantVector::get(Mask), Name + ".extract"); | |||||||||
| 2087 | DEBUG(dbgs() << " shuffle: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shuffle: " << *V << "\n"; } } while (false); | |||||||||
| 2088 | return V; | |||||||||
| 2089 | } | |||||||||
| 2090 | ||||||||||
| 2091 | static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V, | |||||||||
| 2092 | unsigned BeginIndex, const Twine &Name) { | |||||||||
| 2093 | VectorType *VecTy = cast<VectorType>(Old->getType()); | |||||||||
| 2094 | assert(VecTy && "Can only insert a vector into a vector")((VecTy && "Can only insert a vector into a vector") ? static_cast<void> (0) : __assert_fail ("VecTy && \"Can only insert a vector into a vector\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2094, __PRETTY_FUNCTION__)); | |||||||||
| 2095 | ||||||||||
| 2096 | VectorType *Ty = dyn_cast<VectorType>(V->getType()); | |||||||||
| 2097 | if (!Ty) { | |||||||||
| 2098 | // Single element to insert. | |||||||||
| 2099 | V = IRB.CreateInsertElement(Old, V, IRB.getInt32(BeginIndex), | |||||||||
| 2100 | Name + ".insert"); | |||||||||
| 2101 | DEBUG(dbgs() << " insert: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " insert: " << *V << "\n"; } } while (false); | |||||||||
| 2102 | return V; | |||||||||
| 2103 | } | |||||||||
| 2104 | ||||||||||
| 2105 | assert(Ty->getNumElements() <= VecTy->getNumElements() &&((Ty->getNumElements() <= VecTy->getNumElements() && "Too many elements!") ? static_cast<void> (0) : __assert_fail ("Ty->getNumElements() <= VecTy->getNumElements() && \"Too many elements!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2106, __PRETTY_FUNCTION__)) | |||||||||
| 2106 | "Too many elements!")((Ty->getNumElements() <= VecTy->getNumElements() && "Too many elements!") ? static_cast<void> (0) : __assert_fail ("Ty->getNumElements() <= VecTy->getNumElements() && \"Too many elements!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2106, __PRETTY_FUNCTION__)); | |||||||||
| 2107 | if (Ty->getNumElements() == VecTy->getNumElements()) { | |||||||||
| 2108 | assert(V->getType() == VecTy && "Vector type mismatch")((V->getType() == VecTy && "Vector type mismatch") ? static_cast<void> (0) : __assert_fail ("V->getType() == VecTy && \"Vector type mismatch\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2108, __PRETTY_FUNCTION__)); | |||||||||
| 2109 | return V; | |||||||||
| 2110 | } | |||||||||
| 2111 | unsigned EndIndex = BeginIndex + Ty->getNumElements(); | |||||||||
| 2112 | ||||||||||
| 2113 | // When inserting a smaller vector into the larger to store, we first | |||||||||
| 2114 | // use a shuffle vector to widen it with undef elements, and then | |||||||||
| 2115 | // a second shuffle vector to select between the loaded vector and the | |||||||||
| 2116 | // incoming vector. | |||||||||
| 2117 | SmallVector<Constant *, 8> Mask; | |||||||||
| 2118 | Mask.reserve(VecTy->getNumElements()); | |||||||||
| 2119 | for (unsigned i = 0; i != VecTy->getNumElements(); ++i) | |||||||||
| 2120 | if (i >= BeginIndex && i < EndIndex) | |||||||||
| 2121 | Mask.push_back(IRB.getInt32(i - BeginIndex)); | |||||||||
| 2122 | else | |||||||||
| 2123 | Mask.push_back(UndefValue::get(IRB.getInt32Ty())); | |||||||||
| 2124 | V = IRB.CreateShuffleVector(V, UndefValue::get(V->getType()), | |||||||||
| 2125 | ConstantVector::get(Mask), Name + ".expand"); | |||||||||
| 2126 | DEBUG(dbgs() << " shuffle: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shuffle: " << *V << "\n"; } } while (false); | |||||||||
| 2127 | ||||||||||
| 2128 | Mask.clear(); | |||||||||
| 2129 | for (unsigned i = 0; i != VecTy->getNumElements(); ++i) | |||||||||
| 2130 | Mask.push_back(IRB.getInt1(i >= BeginIndex && i < EndIndex)); | |||||||||
| 2131 | ||||||||||
| 2132 | V = IRB.CreateSelect(ConstantVector::get(Mask), V, Old, Name + "blend"); | |||||||||
| 2133 | ||||||||||
| 2134 | DEBUG(dbgs() << " blend: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " blend: " << *V << "\n"; } } while (false); | |||||||||
| 2135 | return V; | |||||||||
| 2136 | } | |||||||||
| 2137 | ||||||||||
| 2138 | /// \brief Visitor to rewrite instructions using p particular slice of an alloca | |||||||||
| 2139 | /// to use a new alloca. | |||||||||
| 2140 | /// | |||||||||
| 2141 | /// Also implements the rewriting to vector-based accesses when the partition | |||||||||
| 2142 | /// passes the isVectorPromotionViable predicate. Most of the rewriting logic | |||||||||
| 2143 | /// lives here. | |||||||||
| 2144 | class llvm::sroa::AllocaSliceRewriter | |||||||||
| 2145 | : public InstVisitor<AllocaSliceRewriter, bool> { | |||||||||
| 2146 | // Befriend the base class so it can delegate to private visit methods. | |||||||||
| 2147 | friend class llvm::InstVisitor<AllocaSliceRewriter, bool>; | |||||||||
| 2148 | typedef llvm::InstVisitor<AllocaSliceRewriter, bool> Base; | |||||||||
| 2149 | ||||||||||
| 2150 | const DataLayout &DL; | |||||||||
| 2151 | AllocaSlices &AS; | |||||||||
| 2152 | SROA &Pass; | |||||||||
| 2153 | AllocaInst &OldAI, &NewAI; | |||||||||
| 2154 | const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset; | |||||||||
| 2155 | Type *NewAllocaTy; | |||||||||
| 2156 | ||||||||||
| 2157 | // This is a convenience and flag variable that will be null unless the new | |||||||||
| 2158 | // alloca's integer operations should be widened to this integer type due to | |||||||||
| 2159 | // passing isIntegerWideningViable above. If it is non-null, the desired | |||||||||
| 2160 | // integer type will be stored here for easy access during rewriting. | |||||||||
| 2161 | IntegerType *IntTy; | |||||||||
| 2162 | ||||||||||
| 2163 | // If we are rewriting an alloca partition which can be written as pure | |||||||||
| 2164 | // vector operations, we stash extra information here. When VecTy is | |||||||||
| 2165 | // non-null, we have some strict guarantees about the rewritten alloca: | |||||||||
| 2166 | // - The new alloca is exactly the size of the vector type here. | |||||||||
| 2167 | // - The accesses all either map to the entire vector or to a single | |||||||||
| 2168 | // element. | |||||||||
| 2169 | // - The set of accessing instructions is only one of those handled above | |||||||||
| 2170 | // in isVectorPromotionViable. Generally these are the same access kinds | |||||||||
| 2171 | // which are promotable via mem2reg. | |||||||||
| 2172 | VectorType *VecTy; | |||||||||
| 2173 | Type *ElementTy; | |||||||||
| 2174 | uint64_t ElementSize; | |||||||||
| 2175 | ||||||||||
| 2176 | // The original offset of the slice currently being rewritten relative to | |||||||||
| 2177 | // the original alloca. | |||||||||
| 2178 | uint64_t BeginOffset, EndOffset; | |||||||||
| 2179 | // The new offsets of the slice currently being rewritten relative to the | |||||||||
| 2180 | // original alloca. | |||||||||
| 2181 | uint64_t NewBeginOffset, NewEndOffset; | |||||||||
| 2182 | ||||||||||
| 2183 | uint64_t SliceSize; | |||||||||
| 2184 | bool IsSplittable; | |||||||||
| 2185 | bool IsSplit; | |||||||||
| 2186 | Use *OldUse; | |||||||||
| 2187 | Instruction *OldPtr; | |||||||||
| 2188 | ||||||||||
| 2189 | // Track post-rewrite users which are PHI nodes and Selects. | |||||||||
| 2190 | SmallSetVector<PHINode *, 8> &PHIUsers; | |||||||||
| 2191 | SmallSetVector<SelectInst *, 8> &SelectUsers; | |||||||||
| 2192 | ||||||||||
| 2193 | // Utility IR builder, whose name prefix is setup for each visited use, and | |||||||||
| 2194 | // the insertion point is set to point to the user. | |||||||||
| 2195 | IRBuilderTy IRB; | |||||||||
| 2196 | ||||||||||
| 2197 | public: | |||||||||
| 2198 | AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass, | |||||||||
| 2199 | AllocaInst &OldAI, AllocaInst &NewAI, | |||||||||
| 2200 | uint64_t NewAllocaBeginOffset, | |||||||||
| 2201 | uint64_t NewAllocaEndOffset, bool IsIntegerPromotable, | |||||||||
| 2202 | VectorType *PromotableVecTy, | |||||||||
| 2203 | SmallSetVector<PHINode *, 8> &PHIUsers, | |||||||||
| 2204 | SmallSetVector<SelectInst *, 8> &SelectUsers) | |||||||||
| 2205 | : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI), | |||||||||
| 2206 | NewAllocaBeginOffset(NewAllocaBeginOffset), | |||||||||
| 2207 | NewAllocaEndOffset(NewAllocaEndOffset), | |||||||||
| 2208 | NewAllocaTy(NewAI.getAllocatedType()), | |||||||||
| 2209 | IntTy(IsIntegerPromotable | |||||||||
| 2210 | ? Type::getIntNTy( | |||||||||
| 2211 | NewAI.getContext(), | |||||||||
| 2212 | DL.getTypeSizeInBits(NewAI.getAllocatedType())) | |||||||||
| 2213 | : nullptr), | |||||||||
| 2214 | VecTy(PromotableVecTy), | |||||||||
| 2215 | ElementTy(VecTy ? VecTy->getElementType() : nullptr), | |||||||||
| 2216 | ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy) / 8 : 0), | |||||||||
| 2217 | BeginOffset(), EndOffset(), IsSplittable(), IsSplit(), OldUse(), | |||||||||
| 2218 | OldPtr(), PHIUsers(PHIUsers), SelectUsers(SelectUsers), | |||||||||
| 2219 | IRB(NewAI.getContext(), ConstantFolder()) { | |||||||||
| 2220 | if (VecTy) { | |||||||||
| 2221 | assert((DL.getTypeSizeInBits(ElementTy) % 8) == 0 &&(((DL.getTypeSizeInBits(ElementTy) % 8) == 0 && "Only multiple-of-8 sized vector elements are viable" ) ? static_cast<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(ElementTy) % 8) == 0 && \"Only multiple-of-8 sized vector elements are viable\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2222, __PRETTY_FUNCTION__)) | |||||||||
| 2222 | "Only multiple-of-8 sized vector elements are viable")(((DL.getTypeSizeInBits(ElementTy) % 8) == 0 && "Only multiple-of-8 sized vector elements are viable" ) ? static_cast<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(ElementTy) % 8) == 0 && \"Only multiple-of-8 sized vector elements are viable\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2222, __PRETTY_FUNCTION__)); | |||||||||
| 2223 | ++NumVectorized; | |||||||||
| 2224 | } | |||||||||
| 2225 | assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy))(((!IntTy && !VecTy) || (IntTy && !VecTy) || ( !IntTy && VecTy)) ? static_cast<void> (0) : __assert_fail ("(!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy)" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2225, __PRETTY_FUNCTION__)); | |||||||||
| 2226 | } | |||||||||
| 2227 | ||||||||||
| 2228 | bool visit(AllocaSlices::const_iterator I) { | |||||||||
| 2229 | bool CanSROA = true; | |||||||||
| 2230 | BeginOffset = I->beginOffset(); | |||||||||
| 2231 | EndOffset = I->endOffset(); | |||||||||
| 2232 | IsSplittable = I->isSplittable(); | |||||||||
| 2233 | IsSplit = | |||||||||
| 2234 | BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset; | |||||||||
| 2235 | DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " rewriting " << (IsSplit ? "split " : ""); } } while (false); | |||||||||
| 2236 | DEBUG(AS.printSlice(dbgs(), I, ""))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.printSlice(dbgs(), I, ""); } } while (false); | |||||||||
| 2237 | DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n"; } } while (false); | |||||||||
| 2238 | ||||||||||
| 2239 | // Compute the intersecting offset range. | |||||||||
| 2240 | assert(BeginOffset < NewAllocaEndOffset)((BeginOffset < NewAllocaEndOffset) ? static_cast<void> (0) : __assert_fail ("BeginOffset < NewAllocaEndOffset", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2240, __PRETTY_FUNCTION__)); | |||||||||
| 2241 | assert(EndOffset > NewAllocaBeginOffset)((EndOffset > NewAllocaBeginOffset) ? static_cast<void> (0) : __assert_fail ("EndOffset > NewAllocaBeginOffset", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2241, __PRETTY_FUNCTION__)); | |||||||||
| 2242 | NewBeginOffset = std::max(BeginOffset, NewAllocaBeginOffset); | |||||||||
| 2243 | NewEndOffset = std::min(EndOffset, NewAllocaEndOffset); | |||||||||
| 2244 | ||||||||||
| 2245 | SliceSize = NewEndOffset - NewBeginOffset; | |||||||||
| 2246 | ||||||||||
| 2247 | OldUse = I->getUse(); | |||||||||
| 2248 | OldPtr = cast<Instruction>(OldUse->get()); | |||||||||
| 2249 | ||||||||||
| 2250 | Instruction *OldUserI = cast<Instruction>(OldUse->getUser()); | |||||||||
| 2251 | IRB.SetInsertPoint(OldUserI); | |||||||||
| 2252 | IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc()); | |||||||||
| 2253 | IRB.SetNamePrefix(Twine(NewAI.getName()) + "." + Twine(BeginOffset) + "."); | |||||||||
| 2254 | ||||||||||
| 2255 | CanSROA &= visit(cast<Instruction>(OldUse->getUser())); | |||||||||
| 2256 | if (VecTy || IntTy) | |||||||||
| 2257 | assert(CanSROA)((CanSROA) ? static_cast<void> (0) : __assert_fail ("CanSROA" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2257, __PRETTY_FUNCTION__)); | |||||||||
| 2258 | return CanSROA; | |||||||||
| 2259 | } | |||||||||
| 2260 | ||||||||||
| 2261 | private: | |||||||||
| 2262 | // Make sure the other visit overloads are visible. | |||||||||
| 2263 | using Base::visit; | |||||||||
| 2264 | ||||||||||
| 2265 | // Every instruction which can end up as a user must have a rewrite rule. | |||||||||
| 2266 | bool visitInstruction(Instruction &I) { | |||||||||
| 2267 | DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " !!!! Cannot rewrite: " << I << "\n"; } } while (false); | |||||||||
| 2268 | llvm_unreachable("No rewrite rule for this instruction!")::llvm::llvm_unreachable_internal("No rewrite rule for this instruction!" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2268); | |||||||||
| 2269 | } | |||||||||
| 2270 | ||||||||||
| 2271 | Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) { | |||||||||
| 2272 | // Note that the offset computation can use BeginOffset or NewBeginOffset | |||||||||
| 2273 | // interchangeably for unsplit slices. | |||||||||
| 2274 | assert(IsSplit || BeginOffset == NewBeginOffset)((IsSplit || BeginOffset == NewBeginOffset) ? static_cast< void> (0) : __assert_fail ("IsSplit || BeginOffset == NewBeginOffset" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2274, __PRETTY_FUNCTION__)); | |||||||||
| 2275 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||||||||
| 2276 | ||||||||||
| 2277 | #ifndef NDEBUG | |||||||||
| 2278 | StringRef OldName = OldPtr->getName(); | |||||||||
| 2279 | // Skip through the last '.sroa.' component of the name. | |||||||||
| 2280 | size_t LastSROAPrefix = OldName.rfind(".sroa."); | |||||||||
| 2281 | if (LastSROAPrefix != StringRef::npos) { | |||||||||
| 2282 | OldName = OldName.substr(LastSROAPrefix + strlen(".sroa.")); | |||||||||
| 2283 | // Look for an SROA slice index. | |||||||||
| 2284 | size_t IndexEnd = OldName.find_first_not_of("0123456789"); | |||||||||
| 2285 | if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') { | |||||||||
| 2286 | // Strip the index and look for the offset. | |||||||||
| 2287 | OldName = OldName.substr(IndexEnd + 1); | |||||||||
| 2288 | size_t OffsetEnd = OldName.find_first_not_of("0123456789"); | |||||||||
| 2289 | if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.') | |||||||||
| 2290 | // Strip the offset. | |||||||||
| 2291 | OldName = OldName.substr(OffsetEnd + 1); | |||||||||
| 2292 | } | |||||||||
| 2293 | } | |||||||||
| 2294 | // Strip any SROA suffixes as well. | |||||||||
| 2295 | OldName = OldName.substr(0, OldName.find(".sroa_")); | |||||||||
| 2296 | #endif | |||||||||
| 2297 | ||||||||||
| 2298 | return getAdjustedPtr(IRB, DL, &NewAI, | |||||||||
| 2299 | APInt(DL.getPointerTypeSizeInBits(PointerTy), Offset), | |||||||||
| 2300 | PointerTy, | |||||||||
| 2301 | #ifndef NDEBUG | |||||||||
| 2302 | Twine(OldName) + "." | |||||||||
| 2303 | #else | |||||||||
| 2304 | Twine() | |||||||||
| 2305 | #endif | |||||||||
| 2306 | ); | |||||||||
| 2307 | } | |||||||||
| 2308 | ||||||||||
| 2309 | /// \brief Compute suitable alignment to access this slice of the *new* | |||||||||
| 2310 | /// alloca. | |||||||||
| 2311 | /// | |||||||||
| 2312 | /// You can optionally pass a type to this routine and if that type's ABI | |||||||||
| 2313 | /// alignment is itself suitable, this will return zero. | |||||||||
| 2314 | unsigned getSliceAlign(Type *Ty = nullptr) { | |||||||||
| 2315 | unsigned NewAIAlign = NewAI.getAlignment(); | |||||||||
| 2316 | if (!NewAIAlign) | |||||||||
| 2317 | NewAIAlign = DL.getABITypeAlignment(NewAI.getAllocatedType()); | |||||||||
| 2318 | unsigned Align = | |||||||||
| 2319 | MinAlign(NewAIAlign, NewBeginOffset - NewAllocaBeginOffset); | |||||||||
| 2320 | return (Ty && Align == DL.getABITypeAlignment(Ty)) ? 0 : Align; | |||||||||
| 2321 | } | |||||||||
| 2322 | ||||||||||
| 2323 | unsigned getIndex(uint64_t Offset) { | |||||||||
| 2324 | assert(VecTy && "Can only call getIndex when rewriting a vector")((VecTy && "Can only call getIndex when rewriting a vector" ) ? static_cast<void> (0) : __assert_fail ("VecTy && \"Can only call getIndex when rewriting a vector\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2324, __PRETTY_FUNCTION__)); | |||||||||
| 2325 | uint64_t RelOffset = Offset - NewAllocaBeginOffset; | |||||||||
| 2326 | assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds")((RelOffset / ElementSize < (4294967295U) && "Index out of bounds" ) ? static_cast<void> (0) : __assert_fail ("RelOffset / ElementSize < UINT32_MAX && \"Index out of bounds\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2326, __PRETTY_FUNCTION__)); | |||||||||
| 2327 | uint32_t Index = RelOffset / ElementSize; | |||||||||
| 2328 | assert(Index * ElementSize == RelOffset)((Index * ElementSize == RelOffset) ? static_cast<void> (0) : __assert_fail ("Index * ElementSize == RelOffset", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2328, __PRETTY_FUNCTION__)); | |||||||||
| 2329 | return Index; | |||||||||
| 2330 | } | |||||||||
| 2331 | ||||||||||
| 2332 | void deleteIfTriviallyDead(Value *V) { | |||||||||
| 2333 | Instruction *I = cast<Instruction>(V); | |||||||||
| 2334 | if (isInstructionTriviallyDead(I)) | |||||||||
| 2335 | Pass.DeadInsts.insert(I); | |||||||||
| 2336 | } | |||||||||
| 2337 | ||||||||||
| 2338 | Value *rewriteVectorizedLoadInst() { | |||||||||
| 2339 | unsigned BeginIndex = getIndex(NewBeginOffset); | |||||||||
| 2340 | unsigned EndIndex = getIndex(NewEndOffset); | |||||||||
| 2341 | assert(EndIndex > BeginIndex && "Empty vector!")((EndIndex > BeginIndex && "Empty vector!") ? static_cast <void> (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2341, __PRETTY_FUNCTION__)); | |||||||||
| 2342 | ||||||||||
| 2343 | Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load"); | |||||||||
| 2344 | return extractVector(IRB, V, BeginIndex, EndIndex, "vec"); | |||||||||
| 2345 | } | |||||||||
| 2346 | ||||||||||
| 2347 | Value *rewriteIntegerLoad(LoadInst &LI) { | |||||||||
| 2348 | assert(IntTy && "We cannot insert an integer to the alloca")((IntTy && "We cannot insert an integer to the alloca" ) ? static_cast<void> (0) : __assert_fail ("IntTy && \"We cannot insert an integer to the alloca\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2348, __PRETTY_FUNCTION__)); | |||||||||
| 2349 | assert(!LI.isVolatile())((!LI.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!LI.isVolatile()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2349, __PRETTY_FUNCTION__)); | |||||||||
| 2350 | Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load"); | |||||||||
| 2351 | V = convertValue(DL, IRB, V, IntTy); | |||||||||
| 2352 | assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset")((NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset" ) ? static_cast<void> (0) : __assert_fail ("NewBeginOffset >= NewAllocaBeginOffset && \"Out of bounds offset\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2352, __PRETTY_FUNCTION__)); | |||||||||
| 2353 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||||||||
| 2354 | if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) { | |||||||||
| 2355 | IntegerType *ExtractTy = Type::getIntNTy(LI.getContext(), SliceSize * 8); | |||||||||
| 2356 | V = extractInteger(DL, IRB, V, ExtractTy, Offset, "extract"); | |||||||||
| 2357 | } | |||||||||
| 2358 | // It is possible that the extracted type is not the load type. This | |||||||||
| 2359 | // happens if there is a load past the end of the alloca, and as | |||||||||
| 2360 | // a consequence the slice is narrower but still a candidate for integer | |||||||||
| 2361 | // lowering. To handle this case, we just zero extend the extracted | |||||||||
| 2362 | // integer. | |||||||||
| 2363 | assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&((cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && "Can only handle an extract for an overly wide load" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && \"Can only handle an extract for an overly wide load\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2364, __PRETTY_FUNCTION__)) | |||||||||
| 2364 | "Can only handle an extract for an overly wide load")((cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && "Can only handle an extract for an overly wide load" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && \"Can only handle an extract for an overly wide load\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2364, __PRETTY_FUNCTION__)); | |||||||||
| 2365 | if (cast<IntegerType>(LI.getType())->getBitWidth() > SliceSize * 8) | |||||||||
| 2366 | V = IRB.CreateZExt(V, LI.getType()); | |||||||||
| 2367 | return V; | |||||||||
| 2368 | } | |||||||||
| 2369 | ||||||||||
| 2370 | bool visitLoadInst(LoadInst &LI) { | |||||||||
| 2371 | DEBUG(dbgs() << " original: " << LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << LI << "\n"; } } while (false); | |||||||||
| 2372 | Value *OldOp = LI.getOperand(0); | |||||||||
| 2373 | assert(OldOp == OldPtr)((OldOp == OldPtr) ? static_cast<void> (0) : __assert_fail ("OldOp == OldPtr", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2373, __PRETTY_FUNCTION__)); | |||||||||
| 2374 | ||||||||||
| 2375 | unsigned AS = LI.getPointerAddressSpace(); | |||||||||
| 2376 | ||||||||||
| 2377 | Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8) | |||||||||
| ||||||||||
| 2378 | : LI.getType(); | |||||||||
| 2379 | const bool IsLoadPastEnd = DL.getTypeStoreSize(TargetTy) > SliceSize; | |||||||||
| 2380 | bool IsPtrAdjusted = false; | |||||||||
| 2381 | Value *V; | |||||||||
| 2382 | if (VecTy) { | |||||||||
| 2383 | V = rewriteVectorizedLoadInst(); | |||||||||
| 2384 | } else if (IntTy && LI.getType()->isIntegerTy()) { | |||||||||
| 2385 | V = rewriteIntegerLoad(LI); | |||||||||
| 2386 | } else if (NewBeginOffset == NewAllocaBeginOffset && | |||||||||
| 2387 | NewEndOffset == NewAllocaEndOffset && | |||||||||
| 2388 | (canConvertValue(DL, NewAllocaTy, TargetTy) || | |||||||||
| 2389 | (IsLoadPastEnd && NewAllocaTy->isIntegerTy() && | |||||||||
| 2390 | TargetTy->isIntegerTy()))) { | |||||||||
| 2391 | LoadInst *NewLI = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), | |||||||||
| 2392 | LI.isVolatile(), LI.getName()); | |||||||||
| 2393 | if (LI.isVolatile()) | |||||||||
| 2394 | NewLI->setAtomic(LI.getOrdering(), LI.getSynchScope()); | |||||||||
| 2395 | ||||||||||
| 2396 | // Try to preserve nonnull metadata | |||||||||
| 2397 | if (TargetTy->isPointerTy()) | |||||||||
| 2398 | NewLI->copyMetadata(LI, LLVMContext::MD_nonnull); | |||||||||
| 2399 | V = NewLI; | |||||||||
| 2400 | ||||||||||
| 2401 | // If this is an integer load past the end of the slice (which means the | |||||||||
| 2402 | // bytes outside the slice are undef or this load is dead) just forcibly | |||||||||
| 2403 | // fix the integer size with correct handling of endianness. | |||||||||
| 2404 | if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy)) | |||||||||
| 2405 | if (auto *TITy = dyn_cast<IntegerType>(TargetTy)) | |||||||||
| 2406 | if (AITy->getBitWidth() < TITy->getBitWidth()) { | |||||||||
| 2407 | V = IRB.CreateZExt(V, TITy, "load.ext"); | |||||||||
| 2408 | if (DL.isBigEndian()) | |||||||||
| 2409 | V = IRB.CreateShl(V, TITy->getBitWidth() - AITy->getBitWidth(), | |||||||||
| 2410 | "endian_shift"); | |||||||||
| 2411 | } | |||||||||
| 2412 | } else { | |||||||||
| 2413 | Type *LTy = TargetTy->getPointerTo(AS); | |||||||||
| 2414 | LoadInst *NewLI = IRB.CreateAlignedLoad(getNewAllocaSlicePtr(IRB, LTy), | |||||||||
| 2415 | getSliceAlign(TargetTy), | |||||||||
| 2416 | LI.isVolatile(), LI.getName()); | |||||||||
| 2417 | if (LI.isVolatile()) | |||||||||
| 2418 | NewLI->setAtomic(LI.getOrdering(), LI.getSynchScope()); | |||||||||
| 2419 | ||||||||||
| 2420 | V = NewLI; | |||||||||
| 2421 | IsPtrAdjusted = true; | |||||||||
| 2422 | } | |||||||||
| 2423 | V = convertValue(DL, IRB, V, TargetTy); | |||||||||
| 2424 | ||||||||||
| 2425 | if (IsSplit) { | |||||||||
| 2426 | assert(!LI.isVolatile())((!LI.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!LI.isVolatile()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2426, __PRETTY_FUNCTION__)); | |||||||||
| 2427 | assert(LI.getType()->isIntegerTy() &&((LI.getType()->isIntegerTy() && "Only integer type loads and stores are split" ) ? static_cast<void> (0) : __assert_fail ("LI.getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2428, __PRETTY_FUNCTION__)) | |||||||||
| 2428 | "Only integer type loads and stores are split")((LI.getType()->isIntegerTy() && "Only integer type loads and stores are split" ) ? static_cast<void> (0) : __assert_fail ("LI.getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2428, __PRETTY_FUNCTION__)); | |||||||||
| 2429 | assert(SliceSize < DL.getTypeStoreSize(LI.getType()) &&((SliceSize < DL.getTypeStoreSize(LI.getType()) && "Split load isn't smaller than original load") ? static_cast <void> (0) : __assert_fail ("SliceSize < DL.getTypeStoreSize(LI.getType()) && \"Split load isn't smaller than original load\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2430, __PRETTY_FUNCTION__)) | |||||||||
| 2430 | "Split load isn't smaller than original load")((SliceSize < DL.getTypeStoreSize(LI.getType()) && "Split load isn't smaller than original load") ? static_cast <void> (0) : __assert_fail ("SliceSize < DL.getTypeStoreSize(LI.getType()) && \"Split load isn't smaller than original load\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2430, __PRETTY_FUNCTION__)); | |||||||||
| 2431 | assert(LI.getType()->getIntegerBitWidth() ==((LI.getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits (LI.getType()) && "Non-byte-multiple bit width") ? static_cast <void> (0) : __assert_fail ("LI.getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits(LI.getType()) && \"Non-byte-multiple bit width\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2433, __PRETTY_FUNCTION__)) | |||||||||
| 2432 | DL.getTypeStoreSizeInBits(LI.getType()) &&((LI.getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits (LI.getType()) && "Non-byte-multiple bit width") ? static_cast <void> (0) : __assert_fail ("LI.getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits(LI.getType()) && \"Non-byte-multiple bit width\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2433, __PRETTY_FUNCTION__)) | |||||||||
| 2433 | "Non-byte-multiple bit width")((LI.getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits (LI.getType()) && "Non-byte-multiple bit width") ? static_cast <void> (0) : __assert_fail ("LI.getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits(LI.getType()) && \"Non-byte-multiple bit width\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2433, __PRETTY_FUNCTION__)); | |||||||||
| 2434 | // Move the insertion point just past the load so that we can refer to it. | |||||||||
| 2435 | IRB.SetInsertPoint(&*std::next(BasicBlock::iterator(&LI))); | |||||||||
| 2436 | // Create a placeholder value with the same type as LI to use as the | |||||||||
| 2437 | // basis for the new value. This allows us to replace the uses of LI with | |||||||||
| 2438 | // the computed value, and then replace the placeholder with LI, leaving | |||||||||
| 2439 | // LI only used for this computation. | |||||||||
| 2440 | Value *Placeholder = | |||||||||
| 2441 | new LoadInst(UndefValue::get(LI.getType()->getPointerTo(AS))); | |||||||||
| 2442 | V = insertInteger(DL, IRB, Placeholder, V, NewBeginOffset - BeginOffset, | |||||||||
| 2443 | "insert"); | |||||||||
| 2444 | LI.replaceAllUsesWith(V); | |||||||||
| 2445 | Placeholder->replaceAllUsesWith(&LI); | |||||||||
| 2446 | delete Placeholder; | |||||||||
| 2447 | } else { | |||||||||
| 2448 | LI.replaceAllUsesWith(V); | |||||||||
| 2449 | } | |||||||||
| 2450 | ||||||||||
| 2451 | Pass.DeadInsts.insert(&LI); | |||||||||
| 2452 | deleteIfTriviallyDead(OldOp); | |||||||||
| 2453 | DEBUG(dbgs() << " to: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *V << "\n"; } } while (false); | |||||||||
| ||||||||||
| 2454 | return !LI.isVolatile() && !IsPtrAdjusted; | |||||||||
| 2455 | } | |||||||||
| 2456 | ||||||||||
| 2457 | bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp) { | |||||||||
| 2458 | if (V->getType() != VecTy) { | |||||||||
| 2459 | unsigned BeginIndex = getIndex(NewBeginOffset); | |||||||||
| 2460 | unsigned EndIndex = getIndex(NewEndOffset); | |||||||||
| 2461 | assert(EndIndex > BeginIndex && "Empty vector!")((EndIndex > BeginIndex && "Empty vector!") ? static_cast <void> (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2461, __PRETTY_FUNCTION__)); | |||||||||
| 2462 | unsigned NumElements = EndIndex - BeginIndex; | |||||||||
| 2463 | assert(NumElements <= VecTy->getNumElements() && "Too many elements!")((NumElements <= VecTy->getNumElements() && "Too many elements!" ) ? static_cast<void> (0) : __assert_fail ("NumElements <= VecTy->getNumElements() && \"Too many elements!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2463, __PRETTY_FUNCTION__)); | |||||||||
| 2464 | Type *SliceTy = (NumElements == 1) | |||||||||
| 2465 | ? ElementTy | |||||||||
| 2466 | : VectorType::get(ElementTy, NumElements); | |||||||||
| 2467 | if (V->getType() != SliceTy) | |||||||||
| 2468 | V = convertValue(DL, IRB, V, SliceTy); | |||||||||
| 2469 | ||||||||||
| 2470 | // Mix in the existing elements. | |||||||||
| 2471 | Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load"); | |||||||||
| 2472 | V = insertVector(IRB, Old, V, BeginIndex, "vec"); | |||||||||
| 2473 | } | |||||||||
| 2474 | StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment()); | |||||||||
| 2475 | Pass.DeadInsts.insert(&SI); | |||||||||
| 2476 | ||||||||||
| 2477 | (void)Store; | |||||||||
| 2478 | DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||||||||
| 2479 | return true; | |||||||||
| 2480 | } | |||||||||
| 2481 | ||||||||||
| 2482 | bool rewriteIntegerStore(Value *V, StoreInst &SI) { | |||||||||
| 2483 | assert(IntTy && "We cannot extract an integer from the alloca")((IntTy && "We cannot extract an integer from the alloca" ) ? static_cast<void> (0) : __assert_fail ("IntTy && \"We cannot extract an integer from the alloca\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2483, __PRETTY_FUNCTION__)); | |||||||||
| 2484 | assert(!SI.isVolatile())((!SI.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!SI.isVolatile()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2484, __PRETTY_FUNCTION__)); | |||||||||
| 2485 | if (DL.getTypeSizeInBits(V->getType()) != IntTy->getBitWidth()) { | |||||||||
| 2486 | Value *Old = | |||||||||
| 2487 | IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload"); | |||||||||
| 2488 | Old = convertValue(DL, IRB, Old, IntTy); | |||||||||
| 2489 | assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset")((BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset" ) ? static_cast<void> (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"Out of bounds offset\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2489, __PRETTY_FUNCTION__)); | |||||||||
| 2490 | uint64_t Offset = BeginOffset - NewAllocaBeginOffset; | |||||||||
| 2491 | V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert"); | |||||||||
| 2492 | } | |||||||||
| 2493 | V = convertValue(DL, IRB, V, NewAllocaTy); | |||||||||
| 2494 | StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment()); | |||||||||
| 2495 | Store->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access); | |||||||||
| 2496 | Pass.DeadInsts.insert(&SI); | |||||||||
| 2497 | DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||||||||
| 2498 | return true; | |||||||||
| 2499 | } | |||||||||
| 2500 | ||||||||||
| 2501 | bool visitStoreInst(StoreInst &SI) { | |||||||||
| 2502 | DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||||||||
| 2503 | Value *OldOp = SI.getOperand(1); | |||||||||
| 2504 | assert(OldOp == OldPtr)((OldOp == OldPtr) ? static_cast<void> (0) : __assert_fail ("OldOp == OldPtr", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2504, __PRETTY_FUNCTION__)); | |||||||||
| 2505 | ||||||||||
| 2506 | Value *V = SI.getValueOperand(); | |||||||||
| 2507 | ||||||||||
| 2508 | // Strip all inbounds GEPs and pointer casts to try to dig out any root | |||||||||
| 2509 | // alloca that should be re-examined after promoting this alloca. | |||||||||
| 2510 | if (V->getType()->isPointerTy()) | |||||||||
| 2511 | if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets())) | |||||||||
| 2512 | Pass.PostPromotionWorklist.insert(AI); | |||||||||
| 2513 | ||||||||||
| 2514 | if (SliceSize < DL.getTypeStoreSize(V->getType())) { | |||||||||
| 2515 | assert(!SI.isVolatile())((!SI.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!SI.isVolatile()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2515, __PRETTY_FUNCTION__)); | |||||||||
| 2516 | assert(V->getType()->isIntegerTy() &&((V->getType()->isIntegerTy() && "Only integer type loads and stores are split" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2517, __PRETTY_FUNCTION__)) | |||||||||
| 2517 | "Only integer type loads and stores are split")((V->getType()->isIntegerTy() && "Only integer type loads and stores are split" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2517, __PRETTY_FUNCTION__)); | |||||||||
| 2518 | assert(V->getType()->getIntegerBitWidth() ==((V->getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits (V->getType()) && "Non-byte-multiple bit width") ? static_cast<void> (0) : __assert_fail ("V->getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits(V->getType()) && \"Non-byte-multiple bit width\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2520, __PRETTY_FUNCTION__)) | |||||||||
| 2519 | DL.getTypeStoreSizeInBits(V->getType()) &&((V->getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits (V->getType()) && "Non-byte-multiple bit width") ? static_cast<void> (0) : __assert_fail ("V->getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits(V->getType()) && \"Non-byte-multiple bit width\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2520, __PRETTY_FUNCTION__)) | |||||||||
| 2520 | "Non-byte-multiple bit width")((V->getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits (V->getType()) && "Non-byte-multiple bit width") ? static_cast<void> (0) : __assert_fail ("V->getType()->getIntegerBitWidth() == DL.getTypeStoreSizeInBits(V->getType()) && \"Non-byte-multiple bit width\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2520, __PRETTY_FUNCTION__)); | |||||||||
| 2521 | IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8); | |||||||||
| 2522 | V = extractInteger(DL, IRB, V, NarrowTy, NewBeginOffset - BeginOffset, | |||||||||
| 2523 | "extract"); | |||||||||
| 2524 | } | |||||||||
| 2525 | ||||||||||
| 2526 | if (VecTy) | |||||||||
| 2527 | return rewriteVectorizedStoreInst(V, SI, OldOp); | |||||||||
| 2528 | if (IntTy && V->getType()->isIntegerTy()) | |||||||||
| 2529 | return rewriteIntegerStore(V, SI); | |||||||||
| 2530 | ||||||||||
| 2531 | const bool IsStorePastEnd = DL.getTypeStoreSize(V->getType()) > SliceSize; | |||||||||
| 2532 | StoreInst *NewSI; | |||||||||
| 2533 | if (NewBeginOffset == NewAllocaBeginOffset && | |||||||||
| 2534 | NewEndOffset == NewAllocaEndOffset && | |||||||||
| 2535 | (canConvertValue(DL, V->getType(), NewAllocaTy) || | |||||||||
| 2536 | (IsStorePastEnd && NewAllocaTy->isIntegerTy() && | |||||||||
| 2537 | V->getType()->isIntegerTy()))) { | |||||||||
| 2538 | // If this is an integer store past the end of slice (and thus the bytes | |||||||||
| 2539 | // past that point are irrelevant or this is unreachable), truncate the | |||||||||
| 2540 | // value prior to storing. | |||||||||
| 2541 | if (auto *VITy = dyn_cast<IntegerType>(V->getType())) | |||||||||
| 2542 | if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy)) | |||||||||
| 2543 | if (VITy->getBitWidth() > AITy->getBitWidth()) { | |||||||||
| 2544 | if (DL.isBigEndian()) | |||||||||
| 2545 | V = IRB.CreateLShr(V, VITy->getBitWidth() - AITy->getBitWidth(), | |||||||||
| 2546 | "endian_shift"); | |||||||||
| 2547 | V = IRB.CreateTrunc(V, AITy, "load.trunc"); | |||||||||
| 2548 | } | |||||||||
| 2549 | ||||||||||
| 2550 | V = convertValue(DL, IRB, V, NewAllocaTy); | |||||||||
| 2551 | NewSI = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(), | |||||||||
| 2552 | SI.isVolatile()); | |||||||||
| 2553 | } else { | |||||||||
| 2554 | unsigned AS = SI.getPointerAddressSpace(); | |||||||||
| 2555 | Value *NewPtr = getNewAllocaSlicePtr(IRB, V->getType()->getPointerTo(AS)); | |||||||||
| 2556 | NewSI = IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(V->getType()), | |||||||||
| 2557 | SI.isVolatile()); | |||||||||
| 2558 | } | |||||||||
| 2559 | NewSI->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access); | |||||||||
| 2560 | if (SI.isVolatile()) | |||||||||
| 2561 | NewSI->setAtomic(SI.getOrdering(), SI.getSynchScope()); | |||||||||
| 2562 | Pass.DeadInsts.insert(&SI); | |||||||||
| 2563 | deleteIfTriviallyDead(OldOp); | |||||||||
| 2564 | ||||||||||
| 2565 | DEBUG(dbgs() << " to: " << *NewSI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *NewSI << "\n"; } } while (false); | |||||||||
| 2566 | return NewSI->getPointerOperand() == &NewAI && !SI.isVolatile(); | |||||||||
| 2567 | } | |||||||||
| 2568 | ||||||||||
| 2569 | /// \brief Compute an integer value from splatting an i8 across the given | |||||||||
| 2570 | /// number of bytes. | |||||||||
| 2571 | /// | |||||||||
| 2572 | /// Note that this routine assumes an i8 is a byte. If that isn't true, don't | |||||||||
| 2573 | /// call this routine. | |||||||||
| 2574 | /// FIXME: Heed the advice above. | |||||||||
| 2575 | /// | |||||||||
| 2576 | /// \param V The i8 value to splat. | |||||||||
| 2577 | /// \param Size The number of bytes in the output (assuming i8 is one byte) | |||||||||
| 2578 | Value *getIntegerSplat(Value *V, unsigned Size) { | |||||||||
| 2579 | assert(Size > 0 && "Expected a positive number of bytes.")((Size > 0 && "Expected a positive number of bytes." ) ? static_cast<void> (0) : __assert_fail ("Size > 0 && \"Expected a positive number of bytes.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2579, __PRETTY_FUNCTION__)); | |||||||||
| 2580 | IntegerType *VTy = cast<IntegerType>(V->getType()); | |||||||||
| 2581 | assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte")((VTy->getBitWidth() == 8 && "Expected an i8 value for the byte" ) ? static_cast<void> (0) : __assert_fail ("VTy->getBitWidth() == 8 && \"Expected an i8 value for the byte\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2581, __PRETTY_FUNCTION__)); | |||||||||
| 2582 | if (Size == 1) | |||||||||
| 2583 | return V; | |||||||||
| 2584 | ||||||||||
| 2585 | Type *SplatIntTy = Type::getIntNTy(VTy->getContext(), Size * 8); | |||||||||
| 2586 | V = IRB.CreateMul( | |||||||||
| 2587 | IRB.CreateZExt(V, SplatIntTy, "zext"), | |||||||||
| 2588 | ConstantExpr::getUDiv( | |||||||||
| 2589 | Constant::getAllOnesValue(SplatIntTy), | |||||||||
| 2590 | ConstantExpr::getZExt(Constant::getAllOnesValue(V->getType()), | |||||||||
| 2591 | SplatIntTy)), | |||||||||
| 2592 | "isplat"); | |||||||||
| 2593 | return V; | |||||||||
| 2594 | } | |||||||||
| 2595 | ||||||||||
| 2596 | /// \brief Compute a vector splat for a given element value. | |||||||||
| 2597 | Value *getVectorSplat(Value *V, unsigned NumElements) { | |||||||||
| 2598 | V = IRB.CreateVectorSplat(NumElements, V, "vsplat"); | |||||||||
| 2599 | DEBUG(dbgs() << " splat: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " splat: " << *V << "\n"; } } while (false); | |||||||||
| 2600 | return V; | |||||||||
| 2601 | } | |||||||||
| 2602 | ||||||||||
| 2603 | bool visitMemSetInst(MemSetInst &II) { | |||||||||
| 2604 | DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | |||||||||
| 2605 | assert(II.getRawDest() == OldPtr)((II.getRawDest() == OldPtr) ? static_cast<void> (0) : __assert_fail ("II.getRawDest() == OldPtr", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2605, __PRETTY_FUNCTION__)); | |||||||||
| 2606 | ||||||||||
| 2607 | // If the memset has a variable size, it cannot be split, just adjust the | |||||||||
| 2608 | // pointer to the new alloca. | |||||||||
| 2609 | if (!isa<Constant>(II.getLength())) { | |||||||||
| 2610 | assert(!IsSplit)((!IsSplit) ? static_cast<void> (0) : __assert_fail ("!IsSplit" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2610, __PRETTY_FUNCTION__)); | |||||||||
| 2611 | assert(NewBeginOffset == BeginOffset)((NewBeginOffset == BeginOffset) ? static_cast<void> (0 ) : __assert_fail ("NewBeginOffset == BeginOffset", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2611, __PRETTY_FUNCTION__)); | |||||||||
| 2612 | II.setDest(getNewAllocaSlicePtr(IRB, OldPtr->getType())); | |||||||||
| 2613 | Type *CstTy = II.getAlignmentCst()->getType(); | |||||||||
| 2614 | II.setAlignment(ConstantInt::get(CstTy, getSliceAlign())); | |||||||||
| 2615 | ||||||||||
| 2616 | deleteIfTriviallyDead(OldPtr); | |||||||||
| 2617 | return false; | |||||||||
| 2618 | } | |||||||||
| 2619 | ||||||||||
| 2620 | // Record this instruction for deletion. | |||||||||
| 2621 | Pass.DeadInsts.insert(&II); | |||||||||
| 2622 | ||||||||||
| 2623 | Type *AllocaTy = NewAI.getAllocatedType(); | |||||||||
| 2624 | Type *ScalarTy = AllocaTy->getScalarType(); | |||||||||
| 2625 | ||||||||||
| 2626 | // If this doesn't map cleanly onto the alloca type, and that type isn't | |||||||||
| 2627 | // a single value type, just emit a memset. | |||||||||
| 2628 | if (!VecTy && !IntTy && | |||||||||
| 2629 | (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset || | |||||||||
| 2630 | SliceSize != DL.getTypeStoreSize(AllocaTy) || | |||||||||
| 2631 | !AllocaTy->isSingleValueType() || | |||||||||
| 2632 | !DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy)) || | |||||||||
| 2633 | DL.getTypeSizeInBits(ScalarTy) % 8 != 0)) { | |||||||||
| 2634 | Type *SizeTy = II.getLength()->getType(); | |||||||||
| 2635 | Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset); | |||||||||
| 2636 | CallInst *New = IRB.CreateMemSet( | |||||||||
| 2637 | getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size, | |||||||||
| 2638 | getSliceAlign(), II.isVolatile()); | |||||||||
| 2639 | (void)New; | |||||||||
| 2640 | DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||||||||
| 2641 | return false; | |||||||||
| 2642 | } | |||||||||
| 2643 | ||||||||||
| 2644 | // If we can represent this as a simple value, we have to build the actual | |||||||||
| 2645 | // value to store, which requires expanding the byte present in memset to | |||||||||
| 2646 | // a sensible representation for the alloca type. This is essentially | |||||||||
| 2647 | // splatting the byte to a sufficiently wide integer, splatting it across | |||||||||
| 2648 | // any desired vector width, and bitcasting to the final type. | |||||||||
| 2649 | Value *V; | |||||||||
| 2650 | ||||||||||
| 2651 | if (VecTy) { | |||||||||
| 2652 | // If this is a memset of a vectorized alloca, insert it. | |||||||||
| 2653 | assert(ElementTy == ScalarTy)((ElementTy == ScalarTy) ? static_cast<void> (0) : __assert_fail ("ElementTy == ScalarTy", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2653, __PRETTY_FUNCTION__)); | |||||||||
| 2654 | ||||||||||
| 2655 | unsigned BeginIndex = getIndex(NewBeginOffset); | |||||||||
| 2656 | unsigned EndIndex = getIndex(NewEndOffset); | |||||||||
| 2657 | assert(EndIndex > BeginIndex && "Empty vector!")((EndIndex > BeginIndex && "Empty vector!") ? static_cast <void> (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2657, __PRETTY_FUNCTION__)); | |||||||||
| 2658 | unsigned NumElements = EndIndex - BeginIndex; | |||||||||
| 2659 | assert(NumElements <= VecTy->getNumElements() && "Too many elements!")((NumElements <= VecTy->getNumElements() && "Too many elements!" ) ? static_cast<void> (0) : __assert_fail ("NumElements <= VecTy->getNumElements() && \"Too many elements!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2659, __PRETTY_FUNCTION__)); | |||||||||
| 2660 | ||||||||||
| 2661 | Value *Splat = | |||||||||
| 2662 | getIntegerSplat(II.getValue(), DL.getTypeSizeInBits(ElementTy) / 8); | |||||||||
| 2663 | Splat = convertValue(DL, IRB, Splat, ElementTy); | |||||||||
| 2664 | if (NumElements > 1) | |||||||||
| 2665 | Splat = getVectorSplat(Splat, NumElements); | |||||||||
| 2666 | ||||||||||
| 2667 | Value *Old = | |||||||||
| 2668 | IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload"); | |||||||||
| 2669 | V = insertVector(IRB, Old, Splat, BeginIndex, "vec"); | |||||||||
| 2670 | } else if (IntTy) { | |||||||||
| 2671 | // If this is a memset on an alloca where we can widen stores, insert the | |||||||||
| 2672 | // set integer. | |||||||||
| 2673 | assert(!II.isVolatile())((!II.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!II.isVolatile()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2673, __PRETTY_FUNCTION__)); | |||||||||
| 2674 | ||||||||||
| 2675 | uint64_t Size = NewEndOffset - NewBeginOffset; | |||||||||
| 2676 | V = getIntegerSplat(II.getValue(), Size); | |||||||||
| 2677 | ||||||||||
| 2678 | if (IntTy && (BeginOffset != NewAllocaBeginOffset || | |||||||||
| 2679 | EndOffset != NewAllocaBeginOffset)) { | |||||||||
| 2680 | Value *Old = | |||||||||
| 2681 | IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload"); | |||||||||
| 2682 | Old = convertValue(DL, IRB, Old, IntTy); | |||||||||
| 2683 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||||||||
| 2684 | V = insertInteger(DL, IRB, Old, V, Offset, "insert"); | |||||||||
| 2685 | } else { | |||||||||
| 2686 | assert(V->getType() == IntTy &&((V->getType() == IntTy && "Wrong type for an alloca wide integer!" ) ? static_cast<void> (0) : __assert_fail ("V->getType() == IntTy && \"Wrong type for an alloca wide integer!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2687, __PRETTY_FUNCTION__)) | |||||||||
| 2687 | "Wrong type for an alloca wide integer!")((V->getType() == IntTy && "Wrong type for an alloca wide integer!" ) ? static_cast<void> (0) : __assert_fail ("V->getType() == IntTy && \"Wrong type for an alloca wide integer!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2687, __PRETTY_FUNCTION__)); | |||||||||
| 2688 | } | |||||||||
| 2689 | V = convertValue(DL, IRB, V, AllocaTy); | |||||||||
| 2690 | } else { | |||||||||
| 2691 | // Established these invariants above. | |||||||||
| 2692 | assert(NewBeginOffset == NewAllocaBeginOffset)((NewBeginOffset == NewAllocaBeginOffset) ? static_cast<void > (0) : __assert_fail ("NewBeginOffset == NewAllocaBeginOffset" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2692, __PRETTY_FUNCTION__)); | |||||||||
| 2693 | assert(NewEndOffset == NewAllocaEndOffset)((NewEndOffset == NewAllocaEndOffset) ? static_cast<void> (0) : __assert_fail ("NewEndOffset == NewAllocaEndOffset", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2693, __PRETTY_FUNCTION__)); | |||||||||
| 2694 | ||||||||||
| 2695 | V = getIntegerSplat(II.getValue(), DL.getTypeSizeInBits(ScalarTy) / 8); | |||||||||
| 2696 | if (VectorType *AllocaVecTy = dyn_cast<VectorType>(AllocaTy)) | |||||||||
| 2697 | V = getVectorSplat(V, AllocaVecTy->getNumElements()); | |||||||||
| 2698 | ||||||||||
| 2699 | V = convertValue(DL, IRB, V, AllocaTy); | |||||||||
| 2700 | } | |||||||||
| 2701 | ||||||||||
| 2702 | Value *New = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(), | |||||||||
| 2703 | II.isVolatile()); | |||||||||
| 2704 | (void)New; | |||||||||
| 2705 | DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||||||||
| 2706 | return !II.isVolatile(); | |||||||||
| 2707 | } | |||||||||
| 2708 | ||||||||||
| 2709 | bool visitMemTransferInst(MemTransferInst &II) { | |||||||||
| 2710 | // Rewriting of memory transfer instructions can be a bit tricky. We break | |||||||||
| 2711 | // them into two categories: split intrinsics and unsplit intrinsics. | |||||||||
| 2712 | ||||||||||
| 2713 | DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | |||||||||
| 2714 | ||||||||||
| 2715 | bool IsDest = &II.getRawDestUse() == OldUse; | |||||||||
| 2716 | assert((IsDest && II.getRawDest() == OldPtr) ||(((IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)) ? static_cast<void> (0) : __assert_fail ("(IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2717, __PRETTY_FUNCTION__)) | |||||||||
| 2717 | (!IsDest && II.getRawSource() == OldPtr))(((IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)) ? static_cast<void> (0) : __assert_fail ("(IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2717, __PRETTY_FUNCTION__)); | |||||||||
| 2718 | ||||||||||
| 2719 | unsigned SliceAlign = getSliceAlign(); | |||||||||
| 2720 | ||||||||||
| 2721 | // For unsplit intrinsics, we simply modify the source and destination | |||||||||
| 2722 | // pointers in place. This isn't just an optimization, it is a matter of | |||||||||
| 2723 | // correctness. With unsplit intrinsics we may be dealing with transfers | |||||||||
| 2724 | // within a single alloca before SROA ran, or with transfers that have | |||||||||
| 2725 | // a variable length. We may also be dealing with memmove instead of | |||||||||
| 2726 | // memcpy, and so simply updating the pointers is the necessary for us to | |||||||||
| 2727 | // update both source and dest of a single call. | |||||||||
| 2728 | if (!IsSplittable) { | |||||||||
| 2729 | Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||||||||
| 2730 | if (IsDest) | |||||||||
| 2731 | II.setDest(AdjustedPtr); | |||||||||
| 2732 | else | |||||||||
| 2733 | II.setSource(AdjustedPtr); | |||||||||
| 2734 | ||||||||||
| 2735 | if (II.getAlignment() > SliceAlign) { | |||||||||
| 2736 | Type *CstTy = II.getAlignmentCst()->getType(); | |||||||||
| 2737 | II.setAlignment( | |||||||||
| 2738 | ConstantInt::get(CstTy, MinAlign(II.getAlignment(), SliceAlign))); | |||||||||
| 2739 | } | |||||||||
| 2740 | ||||||||||
| 2741 | DEBUG(dbgs() << " to: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << II << "\n"; } } while (false); | |||||||||
| 2742 | deleteIfTriviallyDead(OldPtr); | |||||||||
| 2743 | return false; | |||||||||
| 2744 | } | |||||||||
| 2745 | // For split transfer intrinsics we have an incredibly useful assurance: | |||||||||
| 2746 | // the source and destination do not reside within the same alloca, and at | |||||||||
| 2747 | // least one of them does not escape. This means that we can replace | |||||||||
| 2748 | // memmove with memcpy, and we don't need to worry about all manner of | |||||||||
| 2749 | // downsides to splitting and transforming the operations. | |||||||||
| 2750 | ||||||||||
| 2751 | // If this doesn't map cleanly onto the alloca type, and that type isn't | |||||||||
| 2752 | // a single value type, just emit a memcpy. | |||||||||
| 2753 | bool EmitMemCpy = | |||||||||
| 2754 | !VecTy && !IntTy && | |||||||||
| 2755 | (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset || | |||||||||
| 2756 | SliceSize != DL.getTypeStoreSize(NewAI.getAllocatedType()) || | |||||||||
| 2757 | !NewAI.getAllocatedType()->isSingleValueType()); | |||||||||
| 2758 | ||||||||||
| 2759 | // If we're just going to emit a memcpy, the alloca hasn't changed, and the | |||||||||
| 2760 | // size hasn't been shrunk based on analysis of the viable range, this is | |||||||||
| 2761 | // a no-op. | |||||||||
| 2762 | if (EmitMemCpy && &OldAI == &NewAI) { | |||||||||
| 2763 | // Ensure the start lines up. | |||||||||
| 2764 | assert(NewBeginOffset == BeginOffset)((NewBeginOffset == BeginOffset) ? static_cast<void> (0 ) : __assert_fail ("NewBeginOffset == BeginOffset", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2764, __PRETTY_FUNCTION__)); | |||||||||
| 2765 | ||||||||||
| 2766 | // Rewrite the size as needed. | |||||||||
| 2767 | if (NewEndOffset != EndOffset) | |||||||||
| 2768 | II.setLength(ConstantInt::get(II.getLength()->getType(), | |||||||||
| 2769 | NewEndOffset - NewBeginOffset)); | |||||||||
| 2770 | return false; | |||||||||
| 2771 | } | |||||||||
| 2772 | // Record this instruction for deletion. | |||||||||
| 2773 | Pass.DeadInsts.insert(&II); | |||||||||
| 2774 | ||||||||||
| 2775 | // Strip all inbounds GEPs and pointer casts to try to dig out any root | |||||||||
| 2776 | // alloca that should be re-examined after rewriting this instruction. | |||||||||
| 2777 | Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest(); | |||||||||
| 2778 | if (AllocaInst *AI = | |||||||||
| 2779 | dyn_cast<AllocaInst>(OtherPtr->stripInBoundsOffsets())) { | |||||||||
| 2780 | assert(AI != &OldAI && AI != &NewAI &&((AI != &OldAI && AI != &NewAI && "Splittable transfers cannot reach the same alloca on both ends." ) ? static_cast<void> (0) : __assert_fail ("AI != &OldAI && AI != &NewAI && \"Splittable transfers cannot reach the same alloca on both ends.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2781, __PRETTY_FUNCTION__)) | |||||||||
| 2781 | "Splittable transfers cannot reach the same alloca on both ends.")((AI != &OldAI && AI != &NewAI && "Splittable transfers cannot reach the same alloca on both ends." ) ? static_cast<void> (0) : __assert_fail ("AI != &OldAI && AI != &NewAI && \"Splittable transfers cannot reach the same alloca on both ends.\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2781, __PRETTY_FUNCTION__)); | |||||||||
| 2782 | Pass.Worklist.insert(AI); | |||||||||
| 2783 | } | |||||||||
| 2784 | ||||||||||
| 2785 | Type *OtherPtrTy = OtherPtr->getType(); | |||||||||
| 2786 | unsigned OtherAS = OtherPtrTy->getPointerAddressSpace(); | |||||||||
| 2787 | ||||||||||
| 2788 | // Compute the relative offset for the other pointer within the transfer. | |||||||||
| 2789 | unsigned IntPtrWidth = DL.getPointerSizeInBits(OtherAS); | |||||||||
| 2790 | APInt OtherOffset(IntPtrWidth, NewBeginOffset - BeginOffset); | |||||||||
| 2791 | unsigned OtherAlign = MinAlign(II.getAlignment() ? II.getAlignment() : 1, | |||||||||
| 2792 | OtherOffset.zextOrTrunc(64).getZExtValue()); | |||||||||
| 2793 | ||||||||||
| 2794 | if (EmitMemCpy) { | |||||||||
| 2795 | // Compute the other pointer, folding as much as possible to produce | |||||||||
| 2796 | // a single, simple GEP in most cases. | |||||||||
| 2797 | OtherPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy, | |||||||||
| 2798 | OtherPtr->getName() + "."); | |||||||||
| 2799 | ||||||||||
| 2800 | Value *OurPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||||||||
| 2801 | Type *SizeTy = II.getLength()->getType(); | |||||||||
| 2802 | Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset); | |||||||||
| 2803 | ||||||||||
| 2804 | CallInst *New = IRB.CreateMemCpy( | |||||||||
| 2805 | IsDest ? OurPtr : OtherPtr, IsDest ? OtherPtr : OurPtr, Size, | |||||||||
| 2806 | MinAlign(SliceAlign, OtherAlign), II.isVolatile()); | |||||||||
| 2807 | (void)New; | |||||||||
| 2808 | DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||||||||
| 2809 | return false; | |||||||||
| 2810 | } | |||||||||
| 2811 | ||||||||||
| 2812 | bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset && | |||||||||
| 2813 | NewEndOffset == NewAllocaEndOffset; | |||||||||
| 2814 | uint64_t Size = NewEndOffset - NewBeginOffset; | |||||||||
| 2815 | unsigned BeginIndex = VecTy ? getIndex(NewBeginOffset) : 0; | |||||||||
| 2816 | unsigned EndIndex = VecTy ? getIndex(NewEndOffset) : 0; | |||||||||
| 2817 | unsigned NumElements = EndIndex - BeginIndex; | |||||||||
| 2818 | IntegerType *SubIntTy = | |||||||||
| 2819 | IntTy ? Type::getIntNTy(IntTy->getContext(), Size * 8) : nullptr; | |||||||||
| 2820 | ||||||||||
| 2821 | // Reset the other pointer type to match the register type we're going to | |||||||||
| 2822 | // use, but using the address space of the original other pointer. | |||||||||
| 2823 | if (VecTy && !IsWholeAlloca) { | |||||||||
| 2824 | if (NumElements == 1) | |||||||||
| 2825 | OtherPtrTy = VecTy->getElementType(); | |||||||||
| 2826 | else | |||||||||
| 2827 | OtherPtrTy = VectorType::get(VecTy->getElementType(), NumElements); | |||||||||
| 2828 | ||||||||||
| 2829 | OtherPtrTy = OtherPtrTy->getPointerTo(OtherAS); | |||||||||
| 2830 | } else if (IntTy && !IsWholeAlloca) { | |||||||||
| 2831 | OtherPtrTy = SubIntTy->getPointerTo(OtherAS); | |||||||||
| 2832 | } else { | |||||||||
| 2833 | OtherPtrTy = NewAllocaTy->getPointerTo(OtherAS); | |||||||||
| 2834 | } | |||||||||
| 2835 | ||||||||||
| 2836 | Value *SrcPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy, | |||||||||
| 2837 | OtherPtr->getName() + "."); | |||||||||
| 2838 | unsigned SrcAlign = OtherAlign; | |||||||||
| 2839 | Value *DstPtr = &NewAI; | |||||||||
| 2840 | unsigned DstAlign = SliceAlign; | |||||||||
| 2841 | if (!IsDest) { | |||||||||
| 2842 | std::swap(SrcPtr, DstPtr); | |||||||||
| 2843 | std::swap(SrcAlign, DstAlign); | |||||||||
| 2844 | } | |||||||||
| 2845 | ||||||||||
| 2846 | Value *Src; | |||||||||
| 2847 | if (VecTy && !IsWholeAlloca && !IsDest) { | |||||||||
| 2848 | Src = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load"); | |||||||||
| 2849 | Src = extractVector(IRB, Src, BeginIndex, EndIndex, "vec"); | |||||||||
| 2850 | } else if (IntTy && !IsWholeAlloca && !IsDest) { | |||||||||
| 2851 | Src = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load"); | |||||||||
| 2852 | Src = convertValue(DL, IRB, Src, IntTy); | |||||||||
| 2853 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||||||||
| 2854 | Src = extractInteger(DL, IRB, Src, SubIntTy, Offset, "extract"); | |||||||||
| 2855 | } else { | |||||||||
| 2856 | Src = | |||||||||
| 2857 | IRB.CreateAlignedLoad(SrcPtr, SrcAlign, II.isVolatile(), "copyload"); | |||||||||
| 2858 | } | |||||||||
| 2859 | ||||||||||
| 2860 | if (VecTy && !IsWholeAlloca && IsDest) { | |||||||||
| 2861 | Value *Old = | |||||||||
| 2862 | IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload"); | |||||||||
| 2863 | Src = insertVector(IRB, Old, Src, BeginIndex, "vec"); | |||||||||
| 2864 | } else if (IntTy && !IsWholeAlloca && IsDest) { | |||||||||
| 2865 | Value *Old = | |||||||||
| 2866 | IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload"); | |||||||||
| 2867 | Old = convertValue(DL, IRB, Old, IntTy); | |||||||||
| 2868 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | |||||||||
| 2869 | Src = insertInteger(DL, IRB, Old, Src, Offset, "insert"); | |||||||||
| 2870 | Src = convertValue(DL, IRB, Src, NewAllocaTy); | |||||||||
| 2871 | } | |||||||||
| 2872 | ||||||||||
| 2873 | StoreInst *Store = cast<StoreInst>( | |||||||||
| 2874 | IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile())); | |||||||||
| 2875 | (void)Store; | |||||||||
| 2876 | DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||||||||
| 2877 | return !II.isVolatile(); | |||||||||
| 2878 | } | |||||||||
| 2879 | ||||||||||
| 2880 | bool visitIntrinsicInst(IntrinsicInst &II) { | |||||||||
| 2881 | assert(II.getIntrinsicID() == Intrinsic::lifetime_start ||((II.getIntrinsicID() == Intrinsic::lifetime_start || II.getIntrinsicID () == Intrinsic::lifetime_end) ? static_cast<void> (0) : __assert_fail ("II.getIntrinsicID() == Intrinsic::lifetime_start || II.getIntrinsicID() == Intrinsic::lifetime_end" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2882, __PRETTY_FUNCTION__)) | |||||||||
| 2882 | II.getIntrinsicID() == Intrinsic::lifetime_end)((II.getIntrinsicID() == Intrinsic::lifetime_start || II.getIntrinsicID () == Intrinsic::lifetime_end) ? static_cast<void> (0) : __assert_fail ("II.getIntrinsicID() == Intrinsic::lifetime_start || II.getIntrinsicID() == Intrinsic::lifetime_end" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2882, __PRETTY_FUNCTION__)); | |||||||||
| 2883 | DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | |||||||||
| 2884 | assert(II.getArgOperand(1) == OldPtr)((II.getArgOperand(1) == OldPtr) ? static_cast<void> (0 ) : __assert_fail ("II.getArgOperand(1) == OldPtr", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2884, __PRETTY_FUNCTION__)); | |||||||||
| 2885 | ||||||||||
| 2886 | // Record this instruction for deletion. | |||||||||
| 2887 | Pass.DeadInsts.insert(&II); | |||||||||
| 2888 | ||||||||||
| 2889 | // Lifetime intrinsics are only promotable if they cover the whole alloca. | |||||||||
| 2890 | // Therefore, we drop lifetime intrinsics which don't cover the whole | |||||||||
| 2891 | // alloca. | |||||||||
| 2892 | // (In theory, intrinsics which partially cover an alloca could be | |||||||||
| 2893 | // promoted, but PromoteMemToReg doesn't handle that case.) | |||||||||
| 2894 | // FIXME: Check whether the alloca is promotable before dropping the | |||||||||
| 2895 | // lifetime intrinsics? | |||||||||
| 2896 | if (NewBeginOffset != NewAllocaBeginOffset || | |||||||||
| 2897 | NewEndOffset != NewAllocaEndOffset) | |||||||||
| 2898 | return true; | |||||||||
| 2899 | ||||||||||
| 2900 | ConstantInt *Size = | |||||||||
| 2901 | ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()), | |||||||||
| 2902 | NewEndOffset - NewBeginOffset); | |||||||||
| 2903 | Value *Ptr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||||||||
| 2904 | Value *New; | |||||||||
| 2905 | if (II.getIntrinsicID() == Intrinsic::lifetime_start) | |||||||||
| 2906 | New = IRB.CreateLifetimeStart(Ptr, Size); | |||||||||
| 2907 | else | |||||||||
| 2908 | New = IRB.CreateLifetimeEnd(Ptr, Size); | |||||||||
| 2909 | ||||||||||
| 2910 | (void)New; | |||||||||
| 2911 | DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | |||||||||
| 2912 | ||||||||||
| 2913 | return true; | |||||||||
| 2914 | } | |||||||||
| 2915 | ||||||||||
| 2916 | bool visitPHINode(PHINode &PN) { | |||||||||
| 2917 | DEBUG(dbgs() << " original: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << PN << "\n"; } } while (false); | |||||||||
| 2918 | assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable")((BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable" ) ? static_cast<void> (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"PHIs are unsplittable\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2918, __PRETTY_FUNCTION__)); | |||||||||
| 2919 | assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable")((EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable" ) ? static_cast<void> (0) : __assert_fail ("EndOffset <= NewAllocaEndOffset && \"PHIs are unsplittable\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2919, __PRETTY_FUNCTION__)); | |||||||||
| 2920 | ||||||||||
| 2921 | // We would like to compute a new pointer in only one place, but have it be | |||||||||
| 2922 | // as local as possible to the PHI. To do that, we re-use the location of | |||||||||
| 2923 | // the old pointer, which necessarily must be in the right position to | |||||||||
| 2924 | // dominate the PHI. | |||||||||
| 2925 | IRBuilderTy PtrBuilder(IRB); | |||||||||
| 2926 | if (isa<PHINode>(OldPtr)) | |||||||||
| 2927 | PtrBuilder.SetInsertPoint(&*OldPtr->getParent()->getFirstInsertionPt()); | |||||||||
| 2928 | else | |||||||||
| 2929 | PtrBuilder.SetInsertPoint(OldPtr); | |||||||||
| 2930 | PtrBuilder.SetCurrentDebugLocation(OldPtr->getDebugLoc()); | |||||||||
| 2931 | ||||||||||
| 2932 | Value *NewPtr = getNewAllocaSlicePtr(PtrBuilder, OldPtr->getType()); | |||||||||
| 2933 | // Replace the operands which were using the old pointer. | |||||||||
| 2934 | std::replace(PN.op_begin(), PN.op_end(), cast<Value>(OldPtr), NewPtr); | |||||||||
| 2935 | ||||||||||
| 2936 | DEBUG(dbgs() << " to: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << PN << "\n"; } } while (false); | |||||||||
| 2937 | deleteIfTriviallyDead(OldPtr); | |||||||||
| 2938 | ||||||||||
| 2939 | // PHIs can't be promoted on their own, but often can be speculated. We | |||||||||
| 2940 | // check the speculation outside of the rewriter so that we see the | |||||||||
| 2941 | // fully-rewritten alloca. | |||||||||
| 2942 | PHIUsers.insert(&PN); | |||||||||
| 2943 | return true; | |||||||||
| 2944 | } | |||||||||
| 2945 | ||||||||||
| 2946 | bool visitSelectInst(SelectInst &SI) { | |||||||||
| 2947 | DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||||||||
| 2948 | assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&(((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr ) && "Pointer isn't an operand!") ? static_cast<void > (0) : __assert_fail ("(SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && \"Pointer isn't an operand!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2949, __PRETTY_FUNCTION__)) | |||||||||
| 2949 | "Pointer isn't an operand!")(((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr ) && "Pointer isn't an operand!") ? static_cast<void > (0) : __assert_fail ("(SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && \"Pointer isn't an operand!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2949, __PRETTY_FUNCTION__)); | |||||||||
| 2950 | assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable")((BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable" ) ? static_cast<void> (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"Selects are unsplittable\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2950, __PRETTY_FUNCTION__)); | |||||||||
| 2951 | assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable")((EndOffset <= NewAllocaEndOffset && "Selects are unsplittable" ) ? static_cast<void> (0) : __assert_fail ("EndOffset <= NewAllocaEndOffset && \"Selects are unsplittable\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 2951, __PRETTY_FUNCTION__)); | |||||||||
| 2952 | ||||||||||
| 2953 | Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | |||||||||
| 2954 | // Replace the operands which were using the old pointer. | |||||||||
| 2955 | if (SI.getOperand(1) == OldPtr) | |||||||||
| 2956 | SI.setOperand(1, NewPtr); | |||||||||
| 2957 | if (SI.getOperand(2) == OldPtr) | |||||||||
| 2958 | SI.setOperand(2, NewPtr); | |||||||||
| 2959 | ||||||||||
| 2960 | DEBUG(dbgs() << " to: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << SI << "\n"; } } while (false); | |||||||||
| 2961 | deleteIfTriviallyDead(OldPtr); | |||||||||
| 2962 | ||||||||||
| 2963 | // Selects can't be promoted on their own, but often can be speculated. We | |||||||||
| 2964 | // check the speculation outside of the rewriter so that we see the | |||||||||
| 2965 | // fully-rewritten alloca. | |||||||||
| 2966 | SelectUsers.insert(&SI); | |||||||||
| 2967 | return true; | |||||||||
| 2968 | } | |||||||||
| 2969 | }; | |||||||||
| 2970 | ||||||||||
| 2971 | namespace { | |||||||||
| 2972 | /// \brief Visitor to rewrite aggregate loads and stores as scalar. | |||||||||
| 2973 | /// | |||||||||
| 2974 | /// This pass aggressively rewrites all aggregate loads and stores on | |||||||||
| 2975 | /// a particular pointer (or any pointer derived from it which we can identify) | |||||||||
| 2976 | /// with scalar loads and stores. | |||||||||
| 2977 | class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> { | |||||||||
| 2978 | // Befriend the base class so it can delegate to private visit methods. | |||||||||
| 2979 | friend class llvm::InstVisitor<AggLoadStoreRewriter, bool>; | |||||||||
| 2980 | ||||||||||
| 2981 | /// Queue of pointer uses to analyze and potentially rewrite. | |||||||||
| 2982 | SmallVector<Use *, 8> Queue; | |||||||||
| 2983 | ||||||||||
| 2984 | /// Set to prevent us from cycling with phi nodes and loops. | |||||||||
| 2985 | SmallPtrSet<User *, 8> Visited; | |||||||||
| 2986 | ||||||||||
| 2987 | /// The current pointer use being rewritten. This is used to dig up the used | |||||||||
| 2988 | /// value (as opposed to the user). | |||||||||
| 2989 | Use *U; | |||||||||
| 2990 | ||||||||||
| 2991 | public: | |||||||||
| 2992 | /// Rewrite loads and stores through a pointer and all pointers derived from | |||||||||
| 2993 | /// it. | |||||||||
| 2994 | bool rewrite(Instruction &I) { | |||||||||
| 2995 | DEBUG(dbgs() << " Rewriting FCA loads and stores...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting FCA loads and stores...\n" ; } } while (false); | |||||||||
| 2996 | enqueueUsers(I); | |||||||||
| 2997 | bool Changed = false; | |||||||||
| 2998 | while (!Queue.empty()) { | |||||||||
| 2999 | U = Queue.pop_back_val(); | |||||||||
| 3000 | Changed |= visit(cast<Instruction>(U->getUser())); | |||||||||
| 3001 | } | |||||||||
| 3002 | return Changed; | |||||||||
| 3003 | } | |||||||||
| 3004 | ||||||||||
| 3005 | private: | |||||||||
| 3006 | /// Enqueue all the users of the given instruction for further processing. | |||||||||
| 3007 | /// This uses a set to de-duplicate users. | |||||||||
| 3008 | void enqueueUsers(Instruction &I) { | |||||||||
| 3009 | for (Use &U : I.uses()) | |||||||||
| 3010 | if (Visited.insert(U.getUser()).second) | |||||||||
| 3011 | Queue.push_back(&U); | |||||||||
| 3012 | } | |||||||||
| 3013 | ||||||||||
| 3014 | // Conservative default is to not rewrite anything. | |||||||||
| 3015 | bool visitInstruction(Instruction &I) { return false; } | |||||||||
| 3016 | ||||||||||
| 3017 | /// \brief Generic recursive split emission class. | |||||||||
| 3018 | template <typename Derived> class OpSplitter { | |||||||||
| 3019 | protected: | |||||||||
| 3020 | /// The builder used to form new instructions. | |||||||||
| 3021 | IRBuilderTy IRB; | |||||||||
| 3022 | /// The indices which to be used with insert- or extractvalue to select the | |||||||||
| 3023 | /// appropriate value within the aggregate. | |||||||||
| 3024 | SmallVector<unsigned, 4> Indices; | |||||||||
| 3025 | /// The indices to a GEP instruction which will move Ptr to the correct slot | |||||||||
| 3026 | /// within the aggregate. | |||||||||
| 3027 | SmallVector<Value *, 4> GEPIndices; | |||||||||
| 3028 | /// The base pointer of the original op, used as a base for GEPing the | |||||||||
| 3029 | /// split operations. | |||||||||
| 3030 | Value *Ptr; | |||||||||
| 3031 | ||||||||||
| 3032 | /// Initialize the splitter with an insertion point, Ptr and start with a | |||||||||
| 3033 | /// single zero GEP index. | |||||||||
| 3034 | OpSplitter(Instruction *InsertionPoint, Value *Ptr) | |||||||||
| 3035 | : IRB(InsertionPoint), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr) {} | |||||||||
| 3036 | ||||||||||
| 3037 | public: | |||||||||
| 3038 | /// \brief Generic recursive split emission routine. | |||||||||
| 3039 | /// | |||||||||
| 3040 | /// This method recursively splits an aggregate op (load or store) into | |||||||||
| 3041 | /// scalar or vector ops. It splits recursively until it hits a single value | |||||||||
| 3042 | /// and emits that single value operation via the template argument. | |||||||||
| 3043 | /// | |||||||||
| 3044 | /// The logic of this routine relies on GEPs and insertvalue and | |||||||||
| 3045 | /// extractvalue all operating with the same fundamental index list, merely | |||||||||
| 3046 | /// formatted differently (GEPs need actual values). | |||||||||
| 3047 | /// | |||||||||
| 3048 | /// \param Ty The type being split recursively into smaller ops. | |||||||||
| 3049 | /// \param Agg The aggregate value being built up or stored, depending on | |||||||||
| 3050 | /// whether this is splitting a load or a store respectively. | |||||||||
| 3051 | void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) { | |||||||||
| 3052 | if (Ty->isSingleValueType()) | |||||||||
| 3053 | return static_cast<Derived *>(this)->emitFunc(Ty, Agg, Name); | |||||||||
| 3054 | ||||||||||
| 3055 | if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { | |||||||||
| 3056 | unsigned OldSize = Indices.size(); | |||||||||
| 3057 | (void)OldSize; | |||||||||
| 3058 | for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size; | |||||||||
| 3059 | ++Idx) { | |||||||||
| 3060 | assert(Indices.size() == OldSize && "Did not return to the old size")((Indices.size() == OldSize && "Did not return to the old size" ) ? static_cast<void> (0) : __assert_fail ("Indices.size() == OldSize && \"Did not return to the old size\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3060, __PRETTY_FUNCTION__)); | |||||||||
| 3061 | Indices.push_back(Idx); | |||||||||
| 3062 | GEPIndices.push_back(IRB.getInt32(Idx)); | |||||||||
| 3063 | emitSplitOps(ATy->getElementType(), Agg, Name + "." + Twine(Idx)); | |||||||||
| 3064 | GEPIndices.pop_back(); | |||||||||
| 3065 | Indices.pop_back(); | |||||||||
| 3066 | } | |||||||||
| 3067 | return; | |||||||||
| 3068 | } | |||||||||
| 3069 | ||||||||||
| 3070 | if (StructType *STy = dyn_cast<StructType>(Ty)) { | |||||||||
| 3071 | unsigned OldSize = Indices.size(); | |||||||||
| 3072 | (void)OldSize; | |||||||||
| 3073 | for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size; | |||||||||
| 3074 | ++Idx) { | |||||||||
| 3075 | assert(Indices.size() == OldSize && "Did not return to the old size")((Indices.size() == OldSize && "Did not return to the old size" ) ? static_cast<void> (0) : __assert_fail ("Indices.size() == OldSize && \"Did not return to the old size\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3075, __PRETTY_FUNCTION__)); | |||||||||
| 3076 | Indices.push_back(Idx); | |||||||||
| 3077 | GEPIndices.push_back(IRB.getInt32(Idx)); | |||||||||
| 3078 | emitSplitOps(STy->getElementType(Idx), Agg, Name + "." + Twine(Idx)); | |||||||||
| 3079 | GEPIndices.pop_back(); | |||||||||
| 3080 | Indices.pop_back(); | |||||||||
| 3081 | } | |||||||||
| 3082 | return; | |||||||||
| 3083 | } | |||||||||
| 3084 | ||||||||||
| 3085 | llvm_unreachable("Only arrays and structs are aggregate loadable types")::llvm::llvm_unreachable_internal("Only arrays and structs are aggregate loadable types" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3085); | |||||||||
| 3086 | } | |||||||||
| 3087 | }; | |||||||||
| 3088 | ||||||||||
| 3089 | struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> { | |||||||||
| 3090 | LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr) | |||||||||
| 3091 | : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr) {} | |||||||||
| 3092 | ||||||||||
| 3093 | /// Emit a leaf load of a single value. This is called at the leaves of the | |||||||||
| 3094 | /// recursive emission to actually load values. | |||||||||
| 3095 | void emitFunc(Type *Ty, Value *&Agg, const Twine &Name) { | |||||||||
| 3096 | assert(Ty->isSingleValueType())((Ty->isSingleValueType()) ? static_cast<void> (0) : __assert_fail ("Ty->isSingleValueType()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3096, __PRETTY_FUNCTION__)); | |||||||||
| 3097 | // Load the single value and insert it using the indices. | |||||||||
| 3098 | Value *GEP = | |||||||||
| 3099 | IRB.CreateInBoundsGEP(nullptr, Ptr, GEPIndices, Name + ".gep"); | |||||||||
| 3100 | Value *Load = IRB.CreateLoad(GEP, Name + ".load"); | |||||||||
| 3101 | Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert"); | |||||||||
| 3102 | DEBUG(dbgs() << " to: " << *Load << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Load << "\n"; } } while (false); | |||||||||
| 3103 | } | |||||||||
| 3104 | }; | |||||||||
| 3105 | ||||||||||
| 3106 | bool visitLoadInst(LoadInst &LI) { | |||||||||
| 3107 | assert(LI.getPointerOperand() == *U)((LI.getPointerOperand() == *U) ? static_cast<void> (0) : __assert_fail ("LI.getPointerOperand() == *U", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3107, __PRETTY_FUNCTION__)); | |||||||||
| 3108 | if (!LI.isSimple() || LI.getType()->isSingleValueType()) | |||||||||
| 3109 | return false; | |||||||||
| 3110 | ||||||||||
| 3111 | // We have an aggregate being loaded, split it apart. | |||||||||
| 3112 | DEBUG(dbgs() << " original: " << LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << LI << "\n"; } } while (false); | |||||||||
| 3113 | LoadOpSplitter Splitter(&LI, *U); | |||||||||
| 3114 | Value *V = UndefValue::get(LI.getType()); | |||||||||
| 3115 | Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca"); | |||||||||
| 3116 | LI.replaceAllUsesWith(V); | |||||||||
| 3117 | LI.eraseFromParent(); | |||||||||
| 3118 | return true; | |||||||||
| 3119 | } | |||||||||
| 3120 | ||||||||||
| 3121 | struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> { | |||||||||
| 3122 | StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr) | |||||||||
| 3123 | : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr) {} | |||||||||
| 3124 | ||||||||||
| 3125 | /// Emit a leaf store of a single value. This is called at the leaves of the | |||||||||
| 3126 | /// recursive emission to actually produce stores. | |||||||||
| 3127 | void emitFunc(Type *Ty, Value *&Agg, const Twine &Name) { | |||||||||
| 3128 | assert(Ty->isSingleValueType())((Ty->isSingleValueType()) ? static_cast<void> (0) : __assert_fail ("Ty->isSingleValueType()", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3128, __PRETTY_FUNCTION__)); | |||||||||
| 3129 | // Extract the single value and store it using the indices. | |||||||||
| 3130 | // | |||||||||
| 3131 | // The gep and extractvalue values are factored out of the CreateStore | |||||||||
| 3132 | // call to make the output independent of the argument evaluation order. | |||||||||
| 3133 | Value *ExtractValue = | |||||||||
| 3134 | IRB.CreateExtractValue(Agg, Indices, Name + ".extract"); | |||||||||
| 3135 | Value *InBoundsGEP = | |||||||||
| 3136 | IRB.CreateInBoundsGEP(nullptr, Ptr, GEPIndices, Name + ".gep"); | |||||||||
| 3137 | Value *Store = IRB.CreateStore(ExtractValue, InBoundsGEP); | |||||||||
| 3138 | (void)Store; | |||||||||
| 3139 | DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | |||||||||
| 3140 | } | |||||||||
| 3141 | }; | |||||||||
| 3142 | ||||||||||
| 3143 | bool visitStoreInst(StoreInst &SI) { | |||||||||
| 3144 | if (!SI.isSimple() || SI.getPointerOperand() != *U) | |||||||||
| 3145 | return false; | |||||||||
| 3146 | Value *V = SI.getValueOperand(); | |||||||||
| 3147 | if (V->getType()->isSingleValueType()) | |||||||||
| 3148 | return false; | |||||||||
| 3149 | ||||||||||
| 3150 | // We have an aggregate being stored, split it apart. | |||||||||
| 3151 | DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | |||||||||
| 3152 | StoreOpSplitter Splitter(&SI, *U); | |||||||||
| 3153 | Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca"); | |||||||||
| 3154 | SI.eraseFromParent(); | |||||||||
| 3155 | return true; | |||||||||
| 3156 | } | |||||||||
| 3157 | ||||||||||
| 3158 | bool visitBitCastInst(BitCastInst &BC) { | |||||||||
| 3159 | enqueueUsers(BC); | |||||||||
| 3160 | return false; | |||||||||
| 3161 | } | |||||||||
| 3162 | ||||||||||
| 3163 | bool visitGetElementPtrInst(GetElementPtrInst &GEPI) { | |||||||||
| 3164 | enqueueUsers(GEPI); | |||||||||
| 3165 | return false; | |||||||||
| 3166 | } | |||||||||
| 3167 | ||||||||||
| 3168 | bool visitPHINode(PHINode &PN) { | |||||||||
| 3169 | enqueueUsers(PN); | |||||||||
| 3170 | return false; | |||||||||
| 3171 | } | |||||||||
| 3172 | ||||||||||
| 3173 | bool visitSelectInst(SelectInst &SI) { | |||||||||
| 3174 | enqueueUsers(SI); | |||||||||
| 3175 | return false; | |||||||||
| 3176 | } | |||||||||
| 3177 | }; | |||||||||
| 3178 | } | |||||||||
| 3179 | ||||||||||
| 3180 | /// \brief Strip aggregate type wrapping. | |||||||||
| 3181 | /// | |||||||||
| 3182 | /// This removes no-op aggregate types wrapping an underlying type. It will | |||||||||
| 3183 | /// strip as many layers of types as it can without changing either the type | |||||||||
| 3184 | /// size or the allocated size. | |||||||||
| 3185 | static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) { | |||||||||
| 3186 | if (Ty->isSingleValueType()) | |||||||||
| 3187 | return Ty; | |||||||||
| 3188 | ||||||||||
| 3189 | uint64_t AllocSize = DL.getTypeAllocSize(Ty); | |||||||||
| 3190 | uint64_t TypeSize = DL.getTypeSizeInBits(Ty); | |||||||||
| 3191 | ||||||||||
| 3192 | Type *InnerTy; | |||||||||
| 3193 | if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) { | |||||||||
| 3194 | InnerTy = ArrTy->getElementType(); | |||||||||
| 3195 | } else if (StructType *STy = dyn_cast<StructType>(Ty)) { | |||||||||
| 3196 | const StructLayout *SL = DL.getStructLayout(STy); | |||||||||
| 3197 | unsigned Index = SL->getElementContainingOffset(0); | |||||||||
| 3198 | InnerTy = STy->getElementType(Index); | |||||||||
| 3199 | } else { | |||||||||
| 3200 | return Ty; | |||||||||
| 3201 | } | |||||||||
| 3202 | ||||||||||
| 3203 | if (AllocSize > DL.getTypeAllocSize(InnerTy) || | |||||||||
| 3204 | TypeSize > DL.getTypeSizeInBits(InnerTy)) | |||||||||
| 3205 | return Ty; | |||||||||
| 3206 | ||||||||||
| 3207 | return stripAggregateTypeWrapping(DL, InnerTy); | |||||||||
| 3208 | } | |||||||||
| 3209 | ||||||||||
| 3210 | /// \brief Try to find a partition of the aggregate type passed in for a given | |||||||||
| 3211 | /// offset and size. | |||||||||
| 3212 | /// | |||||||||
| 3213 | /// This recurses through the aggregate type and tries to compute a subtype | |||||||||
| 3214 | /// based on the offset and size. When the offset and size span a sub-section | |||||||||
| 3215 | /// of an array, it will even compute a new array type for that sub-section, | |||||||||
| 3216 | /// and the same for structs. | |||||||||
| 3217 | /// | |||||||||
| 3218 | /// Note that this routine is very strict and tries to find a partition of the | |||||||||
| 3219 | /// type which produces the *exact* right offset and size. It is not forgiving | |||||||||
| 3220 | /// when the size or offset cause either end of type-based partition to be off. | |||||||||
| 3221 | /// Also, this is a best-effort routine. It is reasonable to give up and not | |||||||||
| 3222 | /// return a type if necessary. | |||||||||
| 3223 | static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset, | |||||||||
| 3224 | uint64_t Size) { | |||||||||
| 3225 | if (Offset == 0 && DL.getTypeAllocSize(Ty) == Size) | |||||||||
| 3226 | return stripAggregateTypeWrapping(DL, Ty); | |||||||||
| 3227 | if (Offset > DL.getTypeAllocSize(Ty) || | |||||||||
| 3228 | (DL.getTypeAllocSize(Ty) - Offset) < Size) | |||||||||
| 3229 | return nullptr; | |||||||||
| 3230 | ||||||||||
| 3231 | if (SequentialType *SeqTy = dyn_cast<SequentialType>(Ty)) { | |||||||||
| 3232 | Type *ElementTy = SeqTy->getElementType(); | |||||||||
| 3233 | uint64_t ElementSize = DL.getTypeAllocSize(ElementTy); | |||||||||
| 3234 | uint64_t NumSkippedElements = Offset / ElementSize; | |||||||||
| 3235 | if (NumSkippedElements >= SeqTy->getNumElements()) | |||||||||
| 3236 | return nullptr; | |||||||||
| 3237 | Offset -= NumSkippedElements * ElementSize; | |||||||||
| 3238 | ||||||||||
| 3239 | // First check if we need to recurse. | |||||||||
| 3240 | if (Offset > 0 || Size < ElementSize) { | |||||||||
| 3241 | // Bail if the partition ends in a different array element. | |||||||||
| 3242 | if ((Offset + Size) > ElementSize) | |||||||||
| 3243 | return nullptr; | |||||||||
| 3244 | // Recurse through the element type trying to peel off offset bytes. | |||||||||
| 3245 | return getTypePartition(DL, ElementTy, Offset, Size); | |||||||||
| 3246 | } | |||||||||
| 3247 | assert(Offset == 0)((Offset == 0) ? static_cast<void> (0) : __assert_fail ( "Offset == 0", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3247, __PRETTY_FUNCTION__)); | |||||||||
| 3248 | ||||||||||
| 3249 | if (Size == ElementSize) | |||||||||
| 3250 | return stripAggregateTypeWrapping(DL, ElementTy); | |||||||||
| 3251 | assert(Size > ElementSize)((Size > ElementSize) ? static_cast<void> (0) : __assert_fail ("Size > ElementSize", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3251, __PRETTY_FUNCTION__)); | |||||||||
| 3252 | uint64_t NumElements = Size / ElementSize; | |||||||||
| 3253 | if (NumElements * ElementSize != Size) | |||||||||
| 3254 | return nullptr; | |||||||||
| 3255 | return ArrayType::get(ElementTy, NumElements); | |||||||||
| 3256 | } | |||||||||
| 3257 | ||||||||||
| 3258 | StructType *STy = dyn_cast<StructType>(Ty); | |||||||||
| 3259 | if (!STy) | |||||||||
| 3260 | return nullptr; | |||||||||
| 3261 | ||||||||||
| 3262 | const StructLayout *SL = DL.getStructLayout(STy); | |||||||||
| 3263 | if (Offset >= SL->getSizeInBytes()) | |||||||||
| 3264 | return nullptr; | |||||||||
| 3265 | uint64_t EndOffset = Offset + Size; | |||||||||
| 3266 | if (EndOffset > SL->getSizeInBytes()) | |||||||||
| 3267 | return nullptr; | |||||||||
| 3268 | ||||||||||
| 3269 | unsigned Index = SL->getElementContainingOffset(Offset); | |||||||||
| 3270 | Offset -= SL->getElementOffset(Index); | |||||||||
| 3271 | ||||||||||
| 3272 | Type *ElementTy = STy->getElementType(Index); | |||||||||
| 3273 | uint64_t ElementSize = DL.getTypeAllocSize(ElementTy); | |||||||||
| 3274 | if (Offset >= ElementSize) | |||||||||
| 3275 | return nullptr; // The offset points into alignment padding. | |||||||||
| 3276 | ||||||||||
| 3277 | // See if any partition must be contained by the element. | |||||||||
| 3278 | if (Offset > 0 || Size < ElementSize) { | |||||||||
| 3279 | if ((Offset + Size) > ElementSize) | |||||||||
| 3280 | return nullptr; | |||||||||
| 3281 | return getTypePartition(DL, ElementTy, Offset, Size); | |||||||||
| 3282 | } | |||||||||
| 3283 | assert(Offset == 0)((Offset == 0) ? static_cast<void> (0) : __assert_fail ( "Offset == 0", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3283, __PRETTY_FUNCTION__)); | |||||||||
| 3284 | ||||||||||
| 3285 | if (Size == ElementSize) | |||||||||
| 3286 | return stripAggregateTypeWrapping(DL, ElementTy); | |||||||||
| 3287 | ||||||||||
| 3288 | StructType::element_iterator EI = STy->element_begin() + Index, | |||||||||
| 3289 | EE = STy->element_end(); | |||||||||
| 3290 | if (EndOffset < SL->getSizeInBytes()) { | |||||||||
| 3291 | unsigned EndIndex = SL->getElementContainingOffset(EndOffset); | |||||||||
| 3292 | if (Index == EndIndex) | |||||||||
| 3293 | return nullptr; // Within a single element and its padding. | |||||||||
| 3294 | ||||||||||
| 3295 | // Don't try to form "natural" types if the elements don't line up with the | |||||||||
| 3296 | // expected size. | |||||||||
| 3297 | // FIXME: We could potentially recurse down through the last element in the | |||||||||
| 3298 | // sub-struct to find a natural end point. | |||||||||
| 3299 | if (SL->getElementOffset(EndIndex) != EndOffset) | |||||||||
| 3300 | return nullptr; | |||||||||
| 3301 | ||||||||||
| 3302 | assert(Index < EndIndex)((Index < EndIndex) ? static_cast<void> (0) : __assert_fail ("Index < EndIndex", "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3302, __PRETTY_FUNCTION__)); | |||||||||
| 3303 | EE = STy->element_begin() + EndIndex; | |||||||||
| 3304 | } | |||||||||
| 3305 | ||||||||||
| 3306 | // Try to build up a sub-structure. | |||||||||
| 3307 | StructType *SubTy = | |||||||||
| 3308 | StructType::get(STy->getContext(), makeArrayRef(EI, EE), STy->isPacked()); | |||||||||
| 3309 | const StructLayout *SubSL = DL.getStructLayout(SubTy); | |||||||||
| 3310 | if (Size != SubSL->getSizeInBytes()) | |||||||||
| 3311 | return nullptr; // The sub-struct doesn't have quite the size needed. | |||||||||
| 3312 | ||||||||||
| 3313 | return SubTy; | |||||||||
| 3314 | } | |||||||||
| 3315 | ||||||||||
| 3316 | /// \brief Pre-split loads and stores to simplify rewriting. | |||||||||
| 3317 | /// | |||||||||
| 3318 | /// We want to break up the splittable load+store pairs as much as | |||||||||
| 3319 | /// possible. This is important to do as a preprocessing step, as once we | |||||||||
| 3320 | /// start rewriting the accesses to partitions of the alloca we lose the | |||||||||
| 3321 | /// necessary information to correctly split apart paired loads and stores | |||||||||
| 3322 | /// which both point into this alloca. The case to consider is something like | |||||||||
| 3323 | /// the following: | |||||||||
| 3324 | /// | |||||||||
| 3325 | /// %a = alloca [12 x i8] | |||||||||
| 3326 | /// %gep1 = getelementptr [12 x i8]* %a, i32 0, i32 0 | |||||||||
| 3327 | /// %gep2 = getelementptr [12 x i8]* %a, i32 0, i32 4 | |||||||||
| 3328 | /// %gep3 = getelementptr [12 x i8]* %a, i32 0, i32 8 | |||||||||
| 3329 | /// %iptr1 = bitcast i8* %gep1 to i64* | |||||||||
| 3330 | /// %iptr2 = bitcast i8* %gep2 to i64* | |||||||||
| 3331 | /// %fptr1 = bitcast i8* %gep1 to float* | |||||||||
| 3332 | /// %fptr2 = bitcast i8* %gep2 to float* | |||||||||
| 3333 | /// %fptr3 = bitcast i8* %gep3 to float* | |||||||||
| 3334 | /// store float 0.0, float* %fptr1 | |||||||||
| 3335 | /// store float 1.0, float* %fptr2 | |||||||||
| 3336 | /// %v = load i64* %iptr1 | |||||||||
| 3337 | /// store i64 %v, i64* %iptr2 | |||||||||
| 3338 | /// %f1 = load float* %fptr2 | |||||||||
| 3339 | /// %f2 = load float* %fptr3 | |||||||||
| 3340 | /// | |||||||||
| 3341 | /// Here we want to form 3 partitions of the alloca, each 4 bytes large, and | |||||||||
| 3342 | /// promote everything so we recover the 2 SSA values that should have been | |||||||||
| 3343 | /// there all along. | |||||||||
| 3344 | /// | |||||||||
| 3345 | /// \returns true if any changes are made. | |||||||||
| 3346 | bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { | |||||||||
| 3347 | DEBUG(dbgs() << "Pre-splitting loads and stores\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Pre-splitting loads and stores\n" ; } } while (false); | |||||||||
| 3348 | ||||||||||
| 3349 | // Track the loads and stores which are candidates for pre-splitting here, in | |||||||||
| 3350 | // the order they first appear during the partition scan. These give stable | |||||||||
| 3351 | // iteration order and a basis for tracking which loads and stores we | |||||||||
| 3352 | // actually split. | |||||||||
| 3353 | SmallVector<LoadInst *, 4> Loads; | |||||||||
| 3354 | SmallVector<StoreInst *, 4> Stores; | |||||||||
| 3355 | ||||||||||
| 3356 | // We need to accumulate the splits required of each load or store where we | |||||||||
| 3357 | // can find them via a direct lookup. This is important to cross-check loads | |||||||||
| 3358 | // and stores against each other. We also track the slice so that we can kill | |||||||||
| 3359 | // all the slices that end up split. | |||||||||
| 3360 | struct SplitOffsets { | |||||||||
| 3361 | Slice *S; | |||||||||
| 3362 | std::vector<uint64_t> Splits; | |||||||||
| 3363 | }; | |||||||||
| 3364 | SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap; | |||||||||
| 3365 | ||||||||||
| 3366 | // Track loads out of this alloca which cannot, for any reason, be pre-split. | |||||||||
| 3367 | // This is important as we also cannot pre-split stores of those loads! | |||||||||
| 3368 | // FIXME: This is all pretty gross. It means that we can be more aggressive | |||||||||
| 3369 | // in pre-splitting when the load feeding the store happens to come from | |||||||||
| 3370 | // a separate alloca. Put another way, the effectiveness of SROA would be | |||||||||
| 3371 | // decreased by a frontend which just concatenated all of its local allocas | |||||||||
| 3372 | // into one big flat alloca. But defeating such patterns is exactly the job | |||||||||
| 3373 | // SROA is tasked with! Sadly, to not have this discrepancy we would have | |||||||||
| 3374 | // change store pre-splitting to actually force pre-splitting of the load | |||||||||
| 3375 | // that feeds it *and all stores*. That makes pre-splitting much harder, but | |||||||||
| 3376 | // maybe it would make it more principled? | |||||||||
| 3377 | SmallPtrSet<LoadInst *, 8> UnsplittableLoads; | |||||||||
| 3378 | ||||||||||
| 3379 | DEBUG(dbgs() << " Searching for candidate loads and stores\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Searching for candidate loads and stores\n" ; } } while (false); | |||||||||
| 3380 | for (auto &P : AS.partitions()) { | |||||||||
| 3381 | for (Slice &S : P) { | |||||||||
| 3382 | Instruction *I = cast<Instruction>(S.getUse()->getUser()); | |||||||||
| 3383 | if (!S.isSplittable() || S.endOffset() <= P.endOffset()) { | |||||||||
| 3384 | // If this is a load we have to track that it can't participate in any | |||||||||
| 3385 | // pre-splitting. If this is a store of a load we have to track that | |||||||||
| 3386 | // that load also can't participate in any pre-splitting. | |||||||||
| 3387 | if (auto *LI = dyn_cast<LoadInst>(I)) | |||||||||
| 3388 | UnsplittableLoads.insert(LI); | |||||||||
| 3389 | else if (auto *SI = dyn_cast<StoreInst>(I)) | |||||||||
| 3390 | if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand())) | |||||||||
| 3391 | UnsplittableLoads.insert(LI); | |||||||||
| 3392 | continue; | |||||||||
| 3393 | } | |||||||||
| 3394 | assert(P.endOffset() > S.beginOffset() &&((P.endOffset() > S.beginOffset() && "Empty or backwards partition!" ) ? static_cast<void> (0) : __assert_fail ("P.endOffset() > S.beginOffset() && \"Empty or backwards partition!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3395, __PRETTY_FUNCTION__)) | |||||||||
| 3395 | "Empty or backwards partition!")((P.endOffset() > S.beginOffset() && "Empty or backwards partition!" ) ? static_cast<void> (0) : __assert_fail ("P.endOffset() > S.beginOffset() && \"Empty or backwards partition!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3395, __PRETTY_FUNCTION__)); | |||||||||
| 3396 | ||||||||||
| 3397 | // Determine if this is a pre-splittable slice. | |||||||||
| 3398 | if (auto *LI = dyn_cast<LoadInst>(I)) { | |||||||||
| 3399 | assert(!LI->isVolatile() && "Cannot split volatile loads!")((!LI->isVolatile() && "Cannot split volatile loads!" ) ? static_cast<void> (0) : __assert_fail ("!LI->isVolatile() && \"Cannot split volatile loads!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3399, __PRETTY_FUNCTION__)); | |||||||||
| 3400 | ||||||||||
| 3401 | // The load must be used exclusively to store into other pointers for | |||||||||
| 3402 | // us to be able to arbitrarily pre-split it. The stores must also be | |||||||||
| 3403 | // simple to avoid changing semantics. | |||||||||
| 3404 | auto IsLoadSimplyStored = [](LoadInst *LI) { | |||||||||
| 3405 | for (User *LU : LI->users()) { | |||||||||
| 3406 | auto *SI = dyn_cast<StoreInst>(LU); | |||||||||
| 3407 | if (!SI || !SI->isSimple()) | |||||||||
| 3408 | return false; | |||||||||
| 3409 | } | |||||||||
| 3410 | return true; | |||||||||
| 3411 | }; | |||||||||
| 3412 | if (!IsLoadSimplyStored(LI)) { | |||||||||
| 3413 | UnsplittableLoads.insert(LI); | |||||||||
| 3414 | continue; | |||||||||
| 3415 | } | |||||||||
| 3416 | ||||||||||
| 3417 | Loads.push_back(LI); | |||||||||
| 3418 | } else if (auto *SI = dyn_cast<StoreInst>(I)) { | |||||||||
| 3419 | if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex())) | |||||||||
| 3420 | // Skip stores *of* pointers. FIXME: This shouldn't even be possible! | |||||||||
| 3421 | continue; | |||||||||
| 3422 | auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand()); | |||||||||
| 3423 | if (!StoredLoad || !StoredLoad->isSimple()) | |||||||||
| 3424 | continue; | |||||||||
| 3425 | assert(!SI->isVolatile() && "Cannot split volatile stores!")((!SI->isVolatile() && "Cannot split volatile stores!" ) ? static_cast<void> (0) : __assert_fail ("!SI->isVolatile() && \"Cannot split volatile stores!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3425, __PRETTY_FUNCTION__)); | |||||||||
| 3426 | ||||||||||
| 3427 | Stores.push_back(SI); | |||||||||
| 3428 | } else { | |||||||||
| 3429 | // Other uses cannot be pre-split. | |||||||||
| 3430 | continue; | |||||||||
| 3431 | } | |||||||||
| 3432 | ||||||||||
| 3433 | // Record the initial split. | |||||||||
| 3434 | DEBUG(dbgs() << " Candidate: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Candidate: " << *I << "\n"; } } while (false); | |||||||||
| 3435 | auto &Offsets = SplitOffsetsMap[I]; | |||||||||
| 3436 | assert(Offsets.Splits.empty() &&((Offsets.Splits.empty() && "Should not have splits the first time we see an instruction!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.empty() && \"Should not have splits the first time we see an instruction!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3437, __PRETTY_FUNCTION__)) | |||||||||
| 3437 | "Should not have splits the first time we see an instruction!")((Offsets.Splits.empty() && "Should not have splits the first time we see an instruction!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.empty() && \"Should not have splits the first time we see an instruction!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3437, __PRETTY_FUNCTION__)); | |||||||||
| 3438 | Offsets.S = &S; | |||||||||
| 3439 | Offsets.Splits.push_back(P.endOffset() - S.beginOffset()); | |||||||||
| 3440 | } | |||||||||
| 3441 | ||||||||||
| 3442 | // Now scan the already split slices, and add a split for any of them which | |||||||||
| 3443 | // we're going to pre-split. | |||||||||
| 3444 | for (Slice *S : P.splitSliceTails()) { | |||||||||
| 3445 | auto SplitOffsetsMapI = | |||||||||
| 3446 | SplitOffsetsMap.find(cast<Instruction>(S->getUse()->getUser())); | |||||||||
| 3447 | if (SplitOffsetsMapI == SplitOffsetsMap.end()) | |||||||||
| 3448 | continue; | |||||||||
| 3449 | auto &Offsets = SplitOffsetsMapI->second; | |||||||||
| 3450 | ||||||||||
| 3451 | assert(Offsets.S == S && "Found a mismatched slice!")((Offsets.S == S && "Found a mismatched slice!") ? static_cast <void> (0) : __assert_fail ("Offsets.S == S && \"Found a mismatched slice!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3451, __PRETTY_FUNCTION__)); | |||||||||
| 3452 | assert(!Offsets.Splits.empty() &&((!Offsets.Splits.empty() && "Cannot have an empty set of splits on the second partition!" ) ? static_cast<void> (0) : __assert_fail ("!Offsets.Splits.empty() && \"Cannot have an empty set of splits on the second partition!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3453, __PRETTY_FUNCTION__)) | |||||||||
| 3453 | "Cannot have an empty set of splits on the second partition!")((!Offsets.Splits.empty() && "Cannot have an empty set of splits on the second partition!" ) ? static_cast<void> (0) : __assert_fail ("!Offsets.Splits.empty() && \"Cannot have an empty set of splits on the second partition!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3453, __PRETTY_FUNCTION__)); | |||||||||
| 3454 | assert(Offsets.Splits.back() ==((Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset () && "Previous split does not end where this one begins!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3456, __PRETTY_FUNCTION__)) | |||||||||
| 3455 | P.beginOffset() - Offsets.S->beginOffset() &&((Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset () && "Previous split does not end where this one begins!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3456, __PRETTY_FUNCTION__)) | |||||||||
| 3456 | "Previous split does not end where this one begins!")((Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset () && "Previous split does not end where this one begins!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3456, __PRETTY_FUNCTION__)); | |||||||||
| 3457 | ||||||||||
| 3458 | // Record each split. The last partition's end isn't needed as the size | |||||||||
| 3459 | // of the slice dictates that. | |||||||||
| 3460 | if (S->endOffset() > P.endOffset()) | |||||||||
| 3461 | Offsets.Splits.push_back(P.endOffset() - Offsets.S->beginOffset()); | |||||||||
| 3462 | } | |||||||||
| 3463 | } | |||||||||
| 3464 | ||||||||||
| 3465 | // We may have split loads where some of their stores are split stores. For | |||||||||
| 3466 | // such loads and stores, we can only pre-split them if their splits exactly | |||||||||
| 3467 | // match relative to their starting offset. We have to verify this prior to | |||||||||
| 3468 | // any rewriting. | |||||||||
| 3469 | Stores.erase( | |||||||||
| 3470 | remove_if(Stores, | |||||||||
| 3471 | [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) { | |||||||||
| 3472 | // Lookup the load we are storing in our map of split | |||||||||
| 3473 | // offsets. | |||||||||
| 3474 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | |||||||||
| 3475 | // If it was completely unsplittable, then we're done, | |||||||||
| 3476 | // and this store can't be pre-split. | |||||||||
| 3477 | if (UnsplittableLoads.count(LI)) | |||||||||
| 3478 | return true; | |||||||||
| 3479 | ||||||||||
| 3480 | auto LoadOffsetsI = SplitOffsetsMap.find(LI); | |||||||||
| 3481 | if (LoadOffsetsI == SplitOffsetsMap.end()) | |||||||||
| 3482 | return false; // Unrelated loads are definitely safe. | |||||||||
| 3483 | auto &LoadOffsets = LoadOffsetsI->second; | |||||||||
| 3484 | ||||||||||
| 3485 | // Now lookup the store's offsets. | |||||||||
| 3486 | auto &StoreOffsets = SplitOffsetsMap[SI]; | |||||||||
| 3487 | ||||||||||
| 3488 | // If the relative offsets of each split in the load and | |||||||||
| 3489 | // store match exactly, then we can split them and we | |||||||||
| 3490 | // don't need to remove them here. | |||||||||
| 3491 | if (LoadOffsets.Splits == StoreOffsets.Splits) | |||||||||
| 3492 | return false; | |||||||||
| 3493 | ||||||||||
| 3494 | DEBUG(dbgs() << " Mismatched splits for load and store:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | |||||||||
| 3495 | << " " << *LI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | |||||||||
| 3496 | << " " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false); | |||||||||
| 3497 | ||||||||||
| 3498 | // We've found a store and load that we need to split | |||||||||
| 3499 | // with mismatched relative splits. Just give up on them | |||||||||
| 3500 | // and remove both instructions from our list of | |||||||||
| 3501 | // candidates. | |||||||||
| 3502 | UnsplittableLoads.insert(LI); | |||||||||
| 3503 | return true; | |||||||||
| 3504 | }), | |||||||||
| 3505 | Stores.end()); | |||||||||
| 3506 | // Now we have to go *back* through all the stores, because a later store may | |||||||||
| 3507 | // have caused an earlier store's load to become unsplittable and if it is | |||||||||
| 3508 | // unsplittable for the later store, then we can't rely on it being split in | |||||||||
| 3509 | // the earlier store either. | |||||||||
| 3510 | Stores.erase(remove_if(Stores, | |||||||||
| 3511 | [&UnsplittableLoads](StoreInst *SI) { | |||||||||
| 3512 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | |||||||||
| 3513 | return UnsplittableLoads.count(LI); | |||||||||
| 3514 | }), | |||||||||
| 3515 | Stores.end()); | |||||||||
| 3516 | // Once we've established all the loads that can't be split for some reason, | |||||||||
| 3517 | // filter any that made it into our list out. | |||||||||
| 3518 | Loads.erase(remove_if(Loads, | |||||||||
| 3519 | [&UnsplittableLoads](LoadInst *LI) { | |||||||||
| 3520 | return UnsplittableLoads.count(LI); | |||||||||
| 3521 | }), | |||||||||
| 3522 | Loads.end()); | |||||||||
| 3523 | ||||||||||
| 3524 | // If no loads or stores are left, there is no pre-splitting to be done for | |||||||||
| 3525 | // this alloca. | |||||||||
| 3526 | if (Loads.empty() && Stores.empty()) | |||||||||
| 3527 | return false; | |||||||||
| 3528 | ||||||||||
| 3529 | // From here on, we can't fail and will be building new accesses, so rig up | |||||||||
| 3530 | // an IR builder. | |||||||||
| 3531 | IRBuilderTy IRB(&AI); | |||||||||
| 3532 | ||||||||||
| 3533 | // Collect the new slices which we will merge into the alloca slices. | |||||||||
| 3534 | SmallVector<Slice, 4> NewSlices; | |||||||||
| 3535 | ||||||||||
| 3536 | // Track any allocas we end up splitting loads and stores for so we iterate | |||||||||
| 3537 | // on them. | |||||||||
| 3538 | SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas; | |||||||||
| 3539 | ||||||||||
| 3540 | // At this point, we have collected all of the loads and stores we can | |||||||||
| 3541 | // pre-split, and the specific splits needed for them. We actually do the | |||||||||
| 3542 | // splitting in a specific order in order to handle when one of the loads in | |||||||||
| 3543 | // the value operand to one of the stores. | |||||||||
| 3544 | // | |||||||||
| 3545 | // First, we rewrite all of the split loads, and just accumulate each split | |||||||||
| 3546 | // load in a parallel structure. We also build the slices for them and append | |||||||||
| 3547 | // them to the alloca slices. | |||||||||
| 3548 | SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap; | |||||||||
| 3549 | std::vector<LoadInst *> SplitLoads; | |||||||||
| 3550 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||||||||
| 3551 | for (LoadInst *LI : Loads) { | |||||||||
| 3552 | SplitLoads.clear(); | |||||||||
| 3553 | ||||||||||
| 3554 | IntegerType *Ty = cast<IntegerType>(LI->getType()); | |||||||||
| 3555 | uint64_t LoadSize = Ty->getBitWidth() / 8; | |||||||||
| 3556 | assert(LoadSize > 0 && "Cannot have a zero-sized integer load!")((LoadSize > 0 && "Cannot have a zero-sized integer load!" ) ? static_cast<void> (0) : __assert_fail ("LoadSize > 0 && \"Cannot have a zero-sized integer load!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3556, __PRETTY_FUNCTION__)); | |||||||||
| 3557 | ||||||||||
| 3558 | auto &Offsets = SplitOffsetsMap[LI]; | |||||||||
| 3559 | assert(LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&((LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset () && "Slice size should always match load size exactly!" ) ? static_cast<void> (0) : __assert_fail ("LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3560, __PRETTY_FUNCTION__)) | |||||||||
| 3560 | "Slice size should always match load size exactly!")((LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset () && "Slice size should always match load size exactly!" ) ? static_cast<void> (0) : __assert_fail ("LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3560, __PRETTY_FUNCTION__)); | |||||||||
| 3561 | uint64_t BaseOffset = Offsets.S->beginOffset(); | |||||||||
| 3562 | assert(BaseOffset + LoadSize > BaseOffset &&((BaseOffset + LoadSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? static_cast<void> (0) : __assert_fail ("BaseOffset + LoadSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3563, __PRETTY_FUNCTION__)) | |||||||||
| 3563 | "Cannot represent alloca access size using 64-bit integers!")((BaseOffset + LoadSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? static_cast<void> (0) : __assert_fail ("BaseOffset + LoadSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3563, __PRETTY_FUNCTION__)); | |||||||||
| 3564 | ||||||||||
| 3565 | Instruction *BasePtr = cast<Instruction>(LI->getPointerOperand()); | |||||||||
| 3566 | IRB.SetInsertPoint(LI); | |||||||||
| 3567 | ||||||||||
| 3568 | DEBUG(dbgs() << " Splitting load: " << *LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting load: " << *LI << "\n"; } } while (false); | |||||||||
| 3569 | ||||||||||
| 3570 | uint64_t PartOffset = 0, PartSize = Offsets.Splits.front(); | |||||||||
| 3571 | int Idx = 0, Size = Offsets.Splits.size(); | |||||||||
| 3572 | for (;;) { | |||||||||
| 3573 | auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8); | |||||||||
| 3574 | auto *PartPtrTy = PartTy->getPointerTo(LI->getPointerAddressSpace()); | |||||||||
| 3575 | LoadInst *PLoad = IRB.CreateAlignedLoad( | |||||||||
| 3576 | getAdjustedPtr(IRB, DL, BasePtr, | |||||||||
| 3577 | APInt(DL.getPointerSizeInBits(), PartOffset), | |||||||||
| 3578 | PartPtrTy, BasePtr->getName() + "."), | |||||||||
| 3579 | getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false, | |||||||||
| 3580 | LI->getName()); | |||||||||
| 3581 | PLoad->copyMetadata(*LI, LLVMContext::MD_mem_parallel_loop_access); | |||||||||
| 3582 | ||||||||||
| 3583 | // Append this load onto the list of split loads so we can find it later | |||||||||
| 3584 | // to rewrite the stores. | |||||||||
| 3585 | SplitLoads.push_back(PLoad); | |||||||||
| 3586 | ||||||||||
| 3587 | // Now build a new slice for the alloca. | |||||||||
| 3588 | NewSlices.push_back( | |||||||||
| 3589 | Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, | |||||||||
| 3590 | &PLoad->getOperandUse(PLoad->getPointerOperandIndex()), | |||||||||
| 3591 | /*IsSplittable*/ false)); | |||||||||
| 3592 | DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false) | |||||||||
| 3593 | << ", " << NewSlices.back().endOffset() << "): " << *PLoaddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false) | |||||||||
| 3594 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false); | |||||||||
| 3595 | ||||||||||
| 3596 | // See if we've handled all the splits. | |||||||||
| 3597 | if (Idx >= Size) | |||||||||
| 3598 | break; | |||||||||
| 3599 | ||||||||||
| 3600 | // Setup the next partition. | |||||||||
| 3601 | PartOffset = Offsets.Splits[Idx]; | |||||||||
| 3602 | ++Idx; | |||||||||
| 3603 | PartSize = (Idx < Size ? Offsets.Splits[Idx] : LoadSize) - PartOffset; | |||||||||
| 3604 | } | |||||||||
| 3605 | ||||||||||
| 3606 | // Now that we have the split loads, do the slow walk over all uses of the | |||||||||
| 3607 | // load and rewrite them as split stores, or save the split loads to use | |||||||||
| 3608 | // below if the store is going to be split there anyways. | |||||||||
| 3609 | bool DeferredStores = false; | |||||||||
| 3610 | for (User *LU : LI->users()) { | |||||||||
| 3611 | StoreInst *SI = cast<StoreInst>(LU); | |||||||||
| 3612 | if (!Stores.empty() && SplitOffsetsMap.count(SI)) { | |||||||||
| 3613 | DeferredStores = true; | |||||||||
| 3614 | DEBUG(dbgs() << " Deferred splitting of store: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Deferred splitting of store: " << *SI << "\n"; } } while (false); | |||||||||
| 3615 | continue; | |||||||||
| 3616 | } | |||||||||
| 3617 | ||||||||||
| 3618 | Value *StoreBasePtr = SI->getPointerOperand(); | |||||||||
| 3619 | IRB.SetInsertPoint(SI); | |||||||||
| 3620 | ||||||||||
| 3621 | DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting store of load: " << *SI << "\n"; } } while (false); | |||||||||
| 3622 | ||||||||||
| 3623 | for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) { | |||||||||
| 3624 | LoadInst *PLoad = SplitLoads[Idx]; | |||||||||
| 3625 | uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1]; | |||||||||
| 3626 | auto *PartPtrTy = | |||||||||
| 3627 | PLoad->getType()->getPointerTo(SI->getPointerAddressSpace()); | |||||||||
| 3628 | ||||||||||
| 3629 | StoreInst *PStore = IRB.CreateAlignedStore( | |||||||||
| 3630 | PLoad, getAdjustedPtr(IRB, DL, StoreBasePtr, | |||||||||
| 3631 | APInt(DL.getPointerSizeInBits(), PartOffset), | |||||||||
| 3632 | PartPtrTy, StoreBasePtr->getName() + "."), | |||||||||
| 3633 | getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false); | |||||||||
| 3634 | PStore->copyMetadata(*LI, LLVMContext::MD_mem_parallel_loop_access); | |||||||||
| 3635 | DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " +" << PartOffset << ":" << *PStore << "\n"; } } while (false); | |||||||||
| 3636 | } | |||||||||
| 3637 | ||||||||||
| 3638 | // We want to immediately iterate on any allocas impacted by splitting | |||||||||
| 3639 | // this store, and we have to track any promotable alloca (indicated by | |||||||||
| 3640 | // a direct store) as needing to be resplit because it is no longer | |||||||||
| 3641 | // promotable. | |||||||||
| 3642 | if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(StoreBasePtr)) { | |||||||||
| 3643 | ResplitPromotableAllocas.insert(OtherAI); | |||||||||
| 3644 | Worklist.insert(OtherAI); | |||||||||
| 3645 | } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>( | |||||||||
| 3646 | StoreBasePtr->stripInBoundsOffsets())) { | |||||||||
| 3647 | Worklist.insert(OtherAI); | |||||||||
| 3648 | } | |||||||||
| 3649 | ||||||||||
| 3650 | // Mark the original store as dead. | |||||||||
| 3651 | DeadInsts.insert(SI); | |||||||||
| 3652 | } | |||||||||
| 3653 | ||||||||||
| 3654 | // Save the split loads if there are deferred stores among the users. | |||||||||
| 3655 | if (DeferredStores) | |||||||||
| 3656 | SplitLoadsMap.insert(std::make_pair(LI, std::move(SplitLoads))); | |||||||||
| 3657 | ||||||||||
| 3658 | // Mark the original load as dead and kill the original slice. | |||||||||
| 3659 | DeadInsts.insert(LI); | |||||||||
| 3660 | Offsets.S->kill(); | |||||||||
| 3661 | } | |||||||||
| 3662 | ||||||||||
| 3663 | // Second, we rewrite all of the split stores. At this point, we know that | |||||||||
| 3664 | // all loads from this alloca have been split already. For stores of such | |||||||||
| 3665 | // loads, we can simply look up the pre-existing split loads. For stores of | |||||||||
| 3666 | // other loads, we split those loads first and then write split stores of | |||||||||
| 3667 | // them. | |||||||||
| 3668 | for (StoreInst *SI : Stores) { | |||||||||
| 3669 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | |||||||||
| 3670 | IntegerType *Ty = cast<IntegerType>(LI->getType()); | |||||||||
| 3671 | uint64_t StoreSize = Ty->getBitWidth() / 8; | |||||||||
| 3672 | assert(StoreSize > 0 && "Cannot have a zero-sized integer store!")((StoreSize > 0 && "Cannot have a zero-sized integer store!" ) ? static_cast<void> (0) : __assert_fail ("StoreSize > 0 && \"Cannot have a zero-sized integer store!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3672, __PRETTY_FUNCTION__)); | |||||||||
| 3673 | ||||||||||
| 3674 | auto &Offsets = SplitOffsetsMap[SI]; | |||||||||
| 3675 | assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&((StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset () && "Slice size should always match load size exactly!" ) ? static_cast<void> (0) : __assert_fail ("StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3676, __PRETTY_FUNCTION__)) | |||||||||
| 3676 | "Slice size should always match load size exactly!")((StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset () && "Slice size should always match load size exactly!" ) ? static_cast<void> (0) : __assert_fail ("StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3676, __PRETTY_FUNCTION__)); | |||||||||
| 3677 | uint64_t BaseOffset = Offsets.S->beginOffset(); | |||||||||
| 3678 | assert(BaseOffset + StoreSize > BaseOffset &&((BaseOffset + StoreSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? static_cast<void> (0) : __assert_fail ("BaseOffset + StoreSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3679, __PRETTY_FUNCTION__)) | |||||||||
| 3679 | "Cannot represent alloca access size using 64-bit integers!")((BaseOffset + StoreSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? static_cast<void> (0) : __assert_fail ("BaseOffset + StoreSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3679, __PRETTY_FUNCTION__)); | |||||||||
| 3680 | ||||||||||
| 3681 | Value *LoadBasePtr = LI->getPointerOperand(); | |||||||||
| 3682 | Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand()); | |||||||||
| 3683 | ||||||||||
| 3684 | DEBUG(dbgs() << " Splitting store: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting store: " << *SI << "\n"; } } while (false); | |||||||||
| 3685 | ||||||||||
| 3686 | // Check whether we have an already split load. | |||||||||
| 3687 | auto SplitLoadsMapI = SplitLoadsMap.find(LI); | |||||||||
| 3688 | std::vector<LoadInst *> *SplitLoads = nullptr; | |||||||||
| 3689 | if (SplitLoadsMapI != SplitLoadsMap.end()) { | |||||||||
| 3690 | SplitLoads = &SplitLoadsMapI->second; | |||||||||
| 3691 | assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&((SplitLoads->size() == Offsets.Splits.size() + 1 && "Too few split loads for the number of splits in the store!" ) ? static_cast<void> (0) : __assert_fail ("SplitLoads->size() == Offsets.Splits.size() + 1 && \"Too few split loads for the number of splits in the store!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3692, __PRETTY_FUNCTION__)) | |||||||||
| 3692 | "Too few split loads for the number of splits in the store!")((SplitLoads->size() == Offsets.Splits.size() + 1 && "Too few split loads for the number of splits in the store!" ) ? static_cast<void> (0) : __assert_fail ("SplitLoads->size() == Offsets.Splits.size() + 1 && \"Too few split loads for the number of splits in the store!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3692, __PRETTY_FUNCTION__)); | |||||||||
| 3693 | } else { | |||||||||
| 3694 | DEBUG(dbgs() << " of load: " << *LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " of load: " << *LI << "\n"; } } while (false); | |||||||||
| 3695 | } | |||||||||
| 3696 | ||||||||||
| 3697 | uint64_t PartOffset = 0, PartSize = Offsets.Splits.front(); | |||||||||
| 3698 | int Idx = 0, Size = Offsets.Splits.size(); | |||||||||
| 3699 | for (;;) { | |||||||||
| 3700 | auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8); | |||||||||
| 3701 | auto *PartPtrTy = PartTy->getPointerTo(SI->getPointerAddressSpace()); | |||||||||
| 3702 | ||||||||||
| 3703 | // Either lookup a split load or create one. | |||||||||
| 3704 | LoadInst *PLoad; | |||||||||
| 3705 | if (SplitLoads) { | |||||||||
| 3706 | PLoad = (*SplitLoads)[Idx]; | |||||||||
| 3707 | } else { | |||||||||
| 3708 | IRB.SetInsertPoint(LI); | |||||||||
| 3709 | PLoad = IRB.CreateAlignedLoad( | |||||||||
| 3710 | getAdjustedPtr(IRB, DL, LoadBasePtr, | |||||||||
| 3711 | APInt(DL.getPointerSizeInBits(), PartOffset), | |||||||||
| 3712 | PartPtrTy, LoadBasePtr->getName() + "."), | |||||||||
| 3713 | getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false, | |||||||||
| 3714 | LI->getName()); | |||||||||
| 3715 | } | |||||||||
| 3716 | ||||||||||
| 3717 | // And store this partition. | |||||||||
| 3718 | IRB.SetInsertPoint(SI); | |||||||||
| 3719 | StoreInst *PStore = IRB.CreateAlignedStore( | |||||||||
| 3720 | PLoad, getAdjustedPtr(IRB, DL, StoreBasePtr, | |||||||||
| 3721 | APInt(DL.getPointerSizeInBits(), PartOffset), | |||||||||
| 3722 | PartPtrTy, StoreBasePtr->getName() + "."), | |||||||||
| 3723 | getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false); | |||||||||
| 3724 | ||||||||||
| 3725 | // Now build a new slice for the alloca. | |||||||||
| 3726 | NewSlices.push_back( | |||||||||
| 3727 | Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, | |||||||||
| 3728 | &PStore->getOperandUse(PStore->getPointerOperandIndex()), | |||||||||
| 3729 | /*IsSplittable*/ false)); | |||||||||
| 3730 | DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false) | |||||||||
| 3731 | << ", " << NewSlices.back().endOffset() << "): " << *PStoredo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false) | |||||||||
| 3732 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false); | |||||||||
| 3733 | if (!SplitLoads) { | |||||||||
| 3734 | DEBUG(dbgs() << " of split load: " << *PLoad << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " of split load: " << * PLoad << "\n"; } } while (false); | |||||||||
| 3735 | } | |||||||||
| 3736 | ||||||||||
| 3737 | // See if we've finished all the splits. | |||||||||
| 3738 | if (Idx >= Size) | |||||||||
| 3739 | break; | |||||||||
| 3740 | ||||||||||
| 3741 | // Setup the next partition. | |||||||||
| 3742 | PartOffset = Offsets.Splits[Idx]; | |||||||||
| 3743 | ++Idx; | |||||||||
| 3744 | PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset; | |||||||||
| 3745 | } | |||||||||
| 3746 | ||||||||||
| 3747 | // We want to immediately iterate on any allocas impacted by splitting | |||||||||
| 3748 | // this load, which is only relevant if it isn't a load of this alloca and | |||||||||
| 3749 | // thus we didn't already split the loads above. We also have to keep track | |||||||||
| 3750 | // of any promotable allocas we split loads on as they can no longer be | |||||||||
| 3751 | // promoted. | |||||||||
| 3752 | if (!SplitLoads) { | |||||||||
| 3753 | if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(LoadBasePtr)) { | |||||||||
| 3754 | assert(OtherAI != &AI && "We can't re-split our own alloca!")((OtherAI != &AI && "We can't re-split our own alloca!" ) ? static_cast<void> (0) : __assert_fail ("OtherAI != &AI && \"We can't re-split our own alloca!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3754, __PRETTY_FUNCTION__)); | |||||||||
| 3755 | ResplitPromotableAllocas.insert(OtherAI); | |||||||||
| 3756 | Worklist.insert(OtherAI); | |||||||||
| 3757 | } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>( | |||||||||
| 3758 | LoadBasePtr->stripInBoundsOffsets())) { | |||||||||
| 3759 | assert(OtherAI != &AI && "We can't re-split our own alloca!")((OtherAI != &AI && "We can't re-split our own alloca!" ) ? static_cast<void> (0) : __assert_fail ("OtherAI != &AI && \"We can't re-split our own alloca!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3759, __PRETTY_FUNCTION__)); | |||||||||
| 3760 | Worklist.insert(OtherAI); | |||||||||
| 3761 | } | |||||||||
| 3762 | } | |||||||||
| 3763 | ||||||||||
| 3764 | // Mark the original store as dead now that we've split it up and kill its | |||||||||
| 3765 | // slice. Note that we leave the original load in place unless this store | |||||||||
| 3766 | // was its only use. It may in turn be split up if it is an alloca load | |||||||||
| 3767 | // for some other alloca, but it may be a normal load. This may introduce | |||||||||
| 3768 | // redundant loads, but where those can be merged the rest of the optimizer | |||||||||
| 3769 | // should handle the merging, and this uncovers SSA splits which is more | |||||||||
| 3770 | // important. In practice, the original loads will almost always be fully | |||||||||
| 3771 | // split and removed eventually, and the splits will be merged by any | |||||||||
| 3772 | // trivial CSE, including instcombine. | |||||||||
| 3773 | if (LI->hasOneUse()) { | |||||||||
| 3774 | assert(*LI->user_begin() == SI && "Single use isn't this store!")((*LI->user_begin() == SI && "Single use isn't this store!" ) ? static_cast<void> (0) : __assert_fail ("*LI->user_begin() == SI && \"Single use isn't this store!\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3774, __PRETTY_FUNCTION__)); | |||||||||
| 3775 | DeadInsts.insert(LI); | |||||||||
| 3776 | } | |||||||||
| 3777 | DeadInsts.insert(SI); | |||||||||
| 3778 | Offsets.S->kill(); | |||||||||
| 3779 | } | |||||||||
| 3780 | ||||||||||
| 3781 | // Remove the killed slices that have ben pre-split. | |||||||||
| 3782 | AS.erase(remove_if(AS, [](const Slice &S) { return S.isDead(); }), AS.end()); | |||||||||
| 3783 | ||||||||||
| 3784 | // Insert our new slices. This will sort and merge them into the sorted | |||||||||
| 3785 | // sequence. | |||||||||
| 3786 | AS.insert(NewSlices); | |||||||||
| 3787 | ||||||||||
| 3788 | DEBUG(dbgs() << " Pre-split slices:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Pre-split slices:\n"; } } while (false); | |||||||||
| 3789 | #ifndef NDEBUG | |||||||||
| 3790 | for (auto I = AS.begin(), E = AS.end(); I != E; ++I) | |||||||||
| 3791 | DEBUG(AS.print(dbgs(), I, " "))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.print(dbgs(), I, " "); } } while (false); | |||||||||
| 3792 | #endif | |||||||||
| 3793 | ||||||||||
| 3794 | // Finally, don't try to promote any allocas that new require re-splitting. | |||||||||
| 3795 | // They have already been added to the worklist above. | |||||||||
| 3796 | PromotableAllocas.erase( | |||||||||
| 3797 | remove_if( | |||||||||
| 3798 | PromotableAllocas, | |||||||||
| 3799 | [&](AllocaInst *AI) { return ResplitPromotableAllocas.count(AI); }), | |||||||||
| 3800 | PromotableAllocas.end()); | |||||||||
| 3801 | ||||||||||
| 3802 | return true; | |||||||||
| 3803 | } | |||||||||
| 3804 | ||||||||||
| 3805 | /// \brief Rewrite an alloca partition's users. | |||||||||
| 3806 | /// | |||||||||
| 3807 | /// This routine drives both of the rewriting goals of the SROA pass. It tries | |||||||||
| 3808 | /// to rewrite uses of an alloca partition to be conducive for SSA value | |||||||||
| 3809 | /// promotion. If the partition needs a new, more refined alloca, this will | |||||||||
| 3810 | /// build that new alloca, preserving as much type information as possible, and | |||||||||
| 3811 | /// rewrite the uses of the old alloca to point at the new one and have the | |||||||||
| 3812 | /// appropriate new offsets. It also evaluates how successful the rewrite was | |||||||||
| 3813 | /// at enabling promotion and if it was successful queues the alloca to be | |||||||||
| 3814 | /// promoted. | |||||||||
| 3815 | AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, | |||||||||
| 3816 | Partition &P) { | |||||||||
| 3817 | // Try to compute a friendly type for this partition of the alloca. This | |||||||||
| 3818 | // won't always succeed, in which case we fall back to a legal integer type | |||||||||
| 3819 | // or an i8 array of an appropriate size. | |||||||||
| 3820 | Type *SliceTy = nullptr; | |||||||||
| 3821 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||||||||
| 3822 | if (Type *CommonUseTy = findCommonType(P.begin(), P.end(), P.endOffset())) | |||||||||
| 3823 | if (DL.getTypeAllocSize(CommonUseTy) >= P.size()) | |||||||||
| 3824 | SliceTy = CommonUseTy; | |||||||||
| 3825 | if (!SliceTy) | |||||||||
| 3826 | if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(), | |||||||||
| 3827 | P.beginOffset(), P.size())) | |||||||||
| 3828 | SliceTy = TypePartitionTy; | |||||||||
| 3829 | if ((!SliceTy || (SliceTy->isArrayTy() && | |||||||||
| 3830 | SliceTy->getArrayElementType()->isIntegerTy())) && | |||||||||
| 3831 | DL.isLegalInteger(P.size() * 8)) | |||||||||
| 3832 | SliceTy = Type::getIntNTy(*C, P.size() * 8); | |||||||||
| 3833 | if (!SliceTy) | |||||||||
| 3834 | SliceTy = ArrayType::get(Type::getInt8Ty(*C), P.size()); | |||||||||
| 3835 | assert(DL.getTypeAllocSize(SliceTy) >= P.size())((DL.getTypeAllocSize(SliceTy) >= P.size()) ? static_cast< void> (0) : __assert_fail ("DL.getTypeAllocSize(SliceTy) >= P.size()" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3835, __PRETTY_FUNCTION__)); | |||||||||
| 3836 | ||||||||||
| 3837 | bool IsIntegerPromotable = isIntegerWideningViable(P, SliceTy, DL); | |||||||||
| 3838 | ||||||||||
| 3839 | VectorType *VecTy = | |||||||||
| 3840 | IsIntegerPromotable ? nullptr : isVectorPromotionViable(P, DL); | |||||||||
| 3841 | if (VecTy) | |||||||||
| 3842 | SliceTy = VecTy; | |||||||||
| 3843 | ||||||||||
| 3844 | // Check for the case where we're going to rewrite to a new alloca of the | |||||||||
| 3845 | // exact same type as the original, and with the same access offsets. In that | |||||||||
| 3846 | // case, re-use the existing alloca, but still run through the rewriter to | |||||||||
| 3847 | // perform phi and select speculation. | |||||||||
| 3848 | AllocaInst *NewAI; | |||||||||
| 3849 | if (SliceTy == AI.getAllocatedType()) { | |||||||||
| 3850 | assert(P.beginOffset() == 0 &&((P.beginOffset() == 0 && "Non-zero begin offset but same alloca type" ) ? static_cast<void> (0) : __assert_fail ("P.beginOffset() == 0 && \"Non-zero begin offset but same alloca type\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3851, __PRETTY_FUNCTION__)) | |||||||||
| 3851 | "Non-zero begin offset but same alloca type")((P.beginOffset() == 0 && "Non-zero begin offset but same alloca type" ) ? static_cast<void> (0) : __assert_fail ("P.beginOffset() == 0 && \"Non-zero begin offset but same alloca type\"" , "/tmp/buildd/llvm-toolchain-snapshot-5.0~svn303289/lib/Transforms/Scalar/SROA.cpp" , 3851, __PRETTY_FUNCTION__)); | |||||||||
| 3852 | NewAI = &AI; | |||||||||
| 3853 | // FIXME: We should be able to bail at this point with "nothing changed". | |||||||||
| 3854 | // FIXME: We might want to defer PHI speculation until after here. | |||||||||
| 3855 | // FIXME: return nullptr; | |||||||||
| 3856 | } else { | |||||||||
| 3857 | unsigned Alignment = AI.getAlignment(); | |||||||||
| 3858 | if (!Alignment) { | |||||||||
| 3859 | // The minimum alignment which users can rely on when the explicit | |||||||||
| 3860 | // alignment is omitted or zero is that required by the ABI for this | |||||||||
| 3861 | // type. | |||||||||
| 3862 | Alignment = DL.getABITypeAlignment(AI.getAllocatedType()); | |||||||||
| 3863 | } | |||||||||
| 3864 | Alignment = MinAlign(Alignment, P.beginOffset()); | |||||||||
| 3865 | // If we will get at least this much alignment from the type alone, leave | |||||||||
| 3866 | // the alloca's alignment unconstrained. | |||||||||
| 3867 | if (Alignment <= DL.getABITypeAlignment(SliceTy)) | |||||||||
| 3868 | Alignment = 0; | |||||||||
| 3869 | NewAI = new AllocaInst( | |||||||||
| 3870 | SliceTy, AI.getType()->getAddressSpace(), nullptr, Alignment, | |||||||||
| 3871 | AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()), &AI); | |||||||||
| 3872 | ++NumNewAllocas; | |||||||||
| 3873 | } | |||||||||
| 3874 | ||||||||||
| 3875 | DEBUG(dbgs() << "Rewriting alloca partition "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false) | |||||||||
| 3876 | << "[" << P.beginOffset() << "," << P.endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false) | |||||||||
| 3877 | << ") to: " << *NewAI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false); | |||||||||
| 3878 | ||||||||||
| 3879 | // Track the high watermark on the worklist as it is only relevant for | |||||||||
| 3880 | // promoted allocas. We will reset it to this point if the alloca is not in | |||||||||
| 3881 | // fact scheduled for promotion. | |||||||||
| 3882 | unsigned PPWOldSize = PostPromotionWorklist.size(); | |||||||||
| 3883 | unsigned NumUses = 0; | |||||||||
| 3884 | SmallSetVector<PHINode *, 8> PHIUsers; | |||||||||
| 3885 | SmallSetVector<SelectInst *, 8> SelectUsers; | |||||||||
| 3886 | ||||||||||
| 3887 | AllocaSliceRewriter Rewriter(DL, AS, *this, AI, *NewAI, P.beginOffset(), | |||||||||
| 3888 | P.endOffset(), IsIntegerPromotable, VecTy, | |||||||||
| 3889 | PHIUsers, SelectUsers); | |||||||||
| 3890 | bool Promotable = true; | |||||||||
| 3891 | for (Slice *S : P.splitSliceTails()) { | |||||||||
| 3892 | Promotable &= Rewriter.visit(S); | |||||||||
| 3893 | ++NumUses; | |||||||||
| 3894 | } | |||||||||
| 3895 | for (Slice &S : P) { | |||||||||
| 3896 | Promotable &= Rewriter.visit(&S); | |||||||||
| 3897 | ++NumUses; | |||||||||
| 3898 | } | |||||||||
| 3899 | ||||||||||
| 3900 | NumAllocaPartitionUses += NumUses; | |||||||||
| 3901 | MaxUsesPerAllocaPartition = | |||||||||
| 3902 | std::max<unsigned>(NumUses, MaxUsesPerAllocaPartition); | |||||||||
| 3903 | ||||||||||
| 3904 | // Now that we've processed all the slices in the new partition, check if any | |||||||||
| 3905 | // PHIs or Selects would block promotion. | |||||||||
| 3906 | for (PHINode *PHI : PHIUsers) | |||||||||
| 3907 | if (!isSafePHIToSpeculate(*PHI)) { | |||||||||
| 3908 | Promotable = false; | |||||||||
| 3909 | PHIUsers.clear(); | |||||||||
| 3910 | SelectUsers.clear(); | |||||||||
| 3911 | break; | |||||||||
| 3912 | } | |||||||||
| 3913 | ||||||||||
| 3914 | for (SelectInst *Sel : SelectUsers) | |||||||||
| 3915 | if (!isSafeSelectToSpeculate(*Sel)) { | |||||||||
| 3916 | Promotable = false; | |||||||||
| 3917 | PHIUsers.clear(); | |||||||||
| 3918 | SelectUsers.clear(); | |||||||||
| 3919 | break; | |||||||||
| 3920 | } | |||||||||
| 3921 | ||||||||||
| 3922 | if (Promotable) { | |||||||||
| 3923 | if (PHIUsers.empty() && SelectUsers.empty()) { | |||||||||
| 3924 | // Promote the alloca. | |||||||||
| 3925 | PromotableAllocas.push_back(NewAI); | |||||||||
| 3926 | } else { | |||||||||
| 3927 | // If we have either PHIs or Selects to speculate, add them to those | |||||||||
| 3928 | // worklists and re-queue the new alloca so that we promote in on the | |||||||||
| 3929 | // next iteration. | |||||||||
| 3930 | for (PHINode *PHIUser : PHIUsers) | |||||||||
| 3931 | SpeculatablePHIs.insert(PHIUser); | |||||||||
| 3932 | for (SelectInst *SelectUser : SelectUsers) | |||||||||
| 3933 | SpeculatableSelects.insert(SelectUser); | |||||||||
| 3934 | Worklist.insert(NewAI); | |||||||||
| 3935 | } | |||||||||
| 3936 | } else { | |||||||||
| 3937 | // Drop any post-promotion work items if promotion didn't happen. | |||||||||
| 3938 | while (PostPromotionWorklist.size() > PPWOldSize) | |||||||||
| 3939 | PostPromotionWorklist.pop_back(); | |||||||||
| 3940 | ||||||||||
| 3941 | // We couldn't promote and we didn't create a new partition, nothing | |||||||||
| 3942 | // happened. | |||||||||
| 3943 | if (NewAI == &AI) | |||||||||
| 3944 | return nullptr; | |||||||||
| 3945 | ||||||||||
| 3946 | // If we can't promote the alloca, iterate on it to check for new | |||||||||
| 3947 | // refinements exposed by splitting the current alloca. Don't iterate on an | |||||||||
| 3948 | // alloca which didn't actually change and didn't get promoted. | |||||||||
| 3949 | Worklist.insert(NewAI); | |||||||||
| 3950 | } | |||||||||
| 3951 | ||||||||||
| 3952 | return NewAI; | |||||||||
| 3953 | } | |||||||||
| 3954 | ||||||||||
| 3955 | /// \brief Walks the slices of an alloca and form partitions based on them, | |||||||||
| 3956 | /// rewriting each of their uses. | |||||||||
| 3957 | bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { | |||||||||
| 3958 | if (AS.begin() == AS.end()) | |||||||||
| 3959 | return false; | |||||||||
| 3960 | ||||||||||
| 3961 | unsigned NumPartitions = 0; | |||||||||
| 3962 | bool Changed = false; | |||||||||
| 3963 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||||||||
| 3964 | ||||||||||
| 3965 | // First try to pre-split loads and stores. | |||||||||
| 3966 | Changed |= presplitLoadsAndStores(AI, AS); | |||||||||
| 3967 | ||||||||||
| 3968 | // Now that we have identified any pre-splitting opportunities, mark any | |||||||||
| 3969 | // splittable (non-whole-alloca) loads and stores as unsplittable. If we fail | |||||||||
| 3970 | // to split these during pre-splitting, we want to force them to be | |||||||||
| 3971 | // rewritten into a partition. | |||||||||
| 3972 | bool IsSorted = true; | |||||||||
| 3973 | for (Slice &S : AS) { | |||||||||
| 3974 | if (!S.isSplittable()) | |||||||||
| 3975 | continue; | |||||||||
| 3976 | // FIXME: We currently leave whole-alloca splittable loads and stores. This | |||||||||
| 3977 | // used to be the only splittable loads and stores and we need to be | |||||||||
| 3978 | // confident that the above handling of splittable loads and stores is | |||||||||
| 3979 | // completely sufficient before we forcibly disable the remaining handling. | |||||||||
| 3980 | if (S.beginOffset() == 0 && | |||||||||
| 3981 | S.endOffset() >= DL.getTypeAllocSize(AI.getAllocatedType())) | |||||||||
| 3982 | continue; | |||||||||
| 3983 | if (isa<LoadInst>(S.getUse()->getUser()) || | |||||||||
| 3984 | isa<StoreInst>(S.getUse()->getUser())) { | |||||||||
| 3985 | S.makeUnsplittable(); | |||||||||
| 3986 | IsSorted = false; | |||||||||
| 3987 | } | |||||||||
| 3988 | } | |||||||||
| 3989 | if (!IsSorted) | |||||||||
| 3990 | std::sort(AS.begin(), AS.end()); | |||||||||
| 3991 | ||||||||||
| 3992 | /// Describes the allocas introduced by rewritePartition in order to migrate | |||||||||
| 3993 | /// the debug info. | |||||||||
| 3994 | struct Fragment { | |||||||||
| 3995 | AllocaInst *Alloca; | |||||||||
| 3996 | uint64_t Offset; | |||||||||
| 3997 | uint64_t Size; | |||||||||
| 3998 | Fragment(AllocaInst *AI, uint64_t O, uint64_t S) | |||||||||
| 3999 | : Alloca(AI), Offset(O), Size(S) {} | |||||||||
| 4000 | }; | |||||||||
| 4001 | SmallVector<Fragment, 4> Fragments; | |||||||||
| 4002 | ||||||||||
| 4003 | // Rewrite each partition. | |||||||||
| 4004 | for (auto &P : AS.partitions()) { | |||||||||
| 4005 | if (AllocaInst *NewAI = rewritePartition(AI, AS, P)) { | |||||||||
| 4006 | Changed = true; | |||||||||
| 4007 | if (NewAI != &AI) { | |||||||||
| 4008 | uint64_t SizeOfByte = 8; | |||||||||
| 4009 | uint64_t AllocaSize = DL.getTypeSizeInBits(NewAI->getAllocatedType()); | |||||||||
| 4010 | // Don't include any padding. | |||||||||
| 4011 | uint64_t Size = std::min(AllocaSize, P.size() * SizeOfByte); | |||||||||
| 4012 | Fragments.push_back(Fragment(NewAI, P.beginOffset() * SizeOfByte, Size)); | |||||||||
| 4013 | } | |||||||||
| 4014 | } | |||||||||
| 4015 | ++NumPartitions; | |||||||||
| 4016 | } | |||||||||
| 4017 | ||||||||||
| 4018 | NumAllocaPartitions += NumPartitions; | |||||||||
| 4019 | MaxPartitionsPerAlloca = | |||||||||
| 4020 | std::max<unsigned>(NumPartitions, MaxPartitionsPerAlloca); | |||||||||
| 4021 | ||||||||||
| 4022 | // Migrate debug information from the old alloca to the new alloca(s) | |||||||||
| 4023 | // and the individual partitions. | |||||||||
| 4024 | if (DbgDeclareInst *DbgDecl = FindAllocaDbgDeclare(&AI)) { | |||||||||
| 4025 | auto *Var = DbgDecl->getVariable(); | |||||||||
| 4026 | auto *Expr = DbgDecl->getExpression(); | |||||||||
| 4027 | DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false); | |||||||||
| 4028 | uint64_t AllocaSize = DL.getTypeSizeInBits(AI.getAllocatedType()); | |||||||||
| 4029 | for (auto Fragment : Fragments) { | |||||||||
| 4030 | // Create a fragment expression describing the new partition or reuse AI's | |||||||||
| 4031 | // expression if there is only one partition. | |||||||||
| 4032 | auto *FragmentExpr = Expr; | |||||||||
| 4033 | if (Fragment.Size < AllocaSize || Expr->isFragment()) { | |||||||||
| 4034 | // If this alloca is already a scalar replacement of a larger aggregate, | |||||||||
| 4035 | // Fragment.Offset describes the offset inside the scalar. | |||||||||
| 4036 | auto ExprFragment = Expr->getFragmentInfo(); | |||||||||
| 4037 | uint64_t Offset = ExprFragment ? ExprFragment->OffsetInBits : 0; | |||||||||
| 4038 | uint64_t Start = Offset + Fragment.Offset; | |||||||||
| 4039 | uint64_t Size = Fragment.Size; | |||||||||
| 4040 | if (ExprFragment) { | |||||||||
| 4041 | uint64_t AbsEnd = | |||||||||
| 4042 | ExprFragment->OffsetInBits + ExprFragment->SizeInBits; | |||||||||
| 4043 | if (Start >= AbsEnd) | |||||||||
| 4044 | // No need to describe a SROAed padding. | |||||||||
| 4045 | continue; | |||||||||
| 4046 | Size = std::min(Size, AbsEnd - Start); | |||||||||
| 4047 | } | |||||||||
| 4048 | FragmentExpr = DIB.createFragmentExpression(Start, Size); | |||||||||
| 4049 | } | |||||||||
| 4050 | ||||||||||
| 4051 | // Remove any existing dbg.declare intrinsic describing the same alloca. | |||||||||
| 4052 | if (DbgDeclareInst *OldDDI = FindAllocaDbgDeclare(Fragment.Alloca)) | |||||||||
| 4053 | OldDDI->eraseFromParent(); | |||||||||
| 4054 | ||||||||||
| 4055 | DIB.insertDeclare(Fragment.Alloca, Var, FragmentExpr, | |||||||||
| 4056 | DbgDecl->getDebugLoc(), &AI); | |||||||||
| 4057 | } | |||||||||
| 4058 | } | |||||||||
| 4059 | return Changed; | |||||||||
| 4060 | } | |||||||||
| 4061 | ||||||||||
| 4062 | /// \brief Clobber a use with undef, deleting the used value if it becomes dead. | |||||||||
| 4063 | void SROA::clobberUse(Use &U) { | |||||||||
| 4064 | Value *OldV = U; | |||||||||
| 4065 | // Replace the use with an undef value. | |||||||||
| 4066 | U = UndefValue::get(OldV->getType()); | |||||||||
| 4067 | ||||||||||
| 4068 | // Check for this making an instruction dead. We have to garbage collect | |||||||||
| 4069 | // all the dead instructions to ensure the uses of any alloca end up being | |||||||||
| 4070 | // minimal. | |||||||||
| 4071 | if (Instruction *OldI = dyn_cast<Instruction>(OldV)) | |||||||||
| 4072 | if (isInstructionTriviallyDead(OldI)) { | |||||||||
| 4073 | DeadInsts.insert(OldI); | |||||||||
| 4074 | } | |||||||||
| 4075 | } | |||||||||
| 4076 | ||||||||||
| 4077 | /// \brief Analyze an alloca for SROA. | |||||||||
| 4078 | /// | |||||||||
| 4079 | /// This analyzes the alloca to ensure we can reason about it, builds | |||||||||
| 4080 | /// the slices of the alloca, and then hands it off to be split and | |||||||||
| 4081 | /// rewritten as needed. | |||||||||
| 4082 | bool SROA::runOnAlloca(AllocaInst &AI) { | |||||||||
| 4083 | DEBUG(dbgs() << "SROA alloca: " << AI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "SROA alloca: " << AI << "\n"; } } while (false); | |||||||||
| 4084 | ++NumAllocasAnalyzed; | |||||||||
| 4085 | ||||||||||
| 4086 | // Special case dead allocas, as they're trivial. | |||||||||
| 4087 | if (AI.use_empty()) { | |||||||||
| 4088 | AI.eraseFromParent(); | |||||||||
| 4089 | return true; | |||||||||
| 4090 | } | |||||||||
| 4091 | const DataLayout &DL = AI.getModule()->getDataLayout(); | |||||||||
| 4092 | ||||||||||
| 4093 | // Skip alloca forms that this analysis can't handle. | |||||||||
| 4094 | if (AI.isArrayAllocation() || !AI.getAllocatedType()->isSized() || | |||||||||
| 4095 | DL.getTypeAllocSize(AI.getAllocatedType()) == 0) | |||||||||
| 4096 | return false; | |||||||||
| 4097 | ||||||||||
| 4098 | bool Changed = false; | |||||||||
| 4099 | ||||||||||
| 4100 | // First, split any FCA loads and stores touching this alloca to promote | |||||||||
| 4101 | // better splitting and promotion opportunities. | |||||||||
| 4102 | AggLoadStoreRewriter AggRewriter; | |||||||||
| 4103 | Changed |= AggRewriter.rewrite(AI); | |||||||||
| 4104 | ||||||||||
| 4105 | // Build the slices using a recursive instruction-visiting builder. | |||||||||
| 4106 | AllocaSlices AS(DL, AI); | |||||||||
| 4107 | DEBUG(AS.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.print(dbgs()); } } while (false); | |||||||||
| 4108 | if (AS.isEscaped()) | |||||||||
| 4109 | return Changed; | |||||||||
| 4110 | ||||||||||
| 4111 | // Delete all the dead users of this alloca before splitting and rewriting it. | |||||||||
| 4112 | for (Instruction *DeadUser : AS.getDeadUsers()) { | |||||||||
| 4113 | // Free up everything used by this instruction. | |||||||||
| 4114 | for (Use &DeadOp : DeadUser->operands()) | |||||||||
| 4115 | clobberUse(DeadOp); | |||||||||
| 4116 | ||||||||||
| 4117 | // Now replace the uses of this instruction. | |||||||||
| 4118 | DeadUser->replaceAllUsesWith(UndefValue::get(DeadUser->getType())); | |||||||||
| 4119 | ||||||||||
| 4120 | // And mark it for deletion. | |||||||||
| 4121 | DeadInsts.insert(DeadUser); | |||||||||
| 4122 | Changed = true; | |||||||||
| 4123 | } | |||||||||
| 4124 | for (Use *DeadOp : AS.getDeadOperands()) { | |||||||||
| 4125 | clobberUse(*DeadOp); | |||||||||
| 4126 | Changed = true; | |||||||||
| 4127 | } | |||||||||
| 4128 | ||||||||||
| 4129 | // No slices to split. Leave the dead alloca for a later pass to clean up. | |||||||||
| 4130 | if (AS.begin() == AS.end()) | |||||||||
| 4131 | return Changed; | |||||||||
| 4132 | ||||||||||
| 4133 | Changed |= splitAlloca(AI, AS); | |||||||||
| 4134 | ||||||||||
| 4135 | DEBUG(dbgs() << " Speculating PHIs\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Speculating PHIs\n"; } } while (false); | |||||||||
| 4136 | while (!SpeculatablePHIs.empty()) | |||||||||
| 4137 | speculatePHINodeLoads(*SpeculatablePHIs.pop_back_val()); | |||||||||
| 4138 | ||||||||||
| 4139 | DEBUG(dbgs() << " Speculating Selects\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Speculating Selects\n"; } } while (false); | |||||||||
| 4140 | while (!SpeculatableSelects.empty()) | |||||||||
| 4141 | speculateSelectInstLoads(*SpeculatableSelects.pop_back_val()); | |||||||||
| 4142 | ||||||||||
| 4143 | return Changed; | |||||||||
| 4144 | } | |||||||||
| 4145 | ||||||||||
| 4146 | /// \brief Delete the dead instructions accumulated in this run. | |||||||||
| 4147 | /// | |||||||||
| 4148 | /// Recursively deletes the dead instructions we've accumulated. This is done | |||||||||
| 4149 | /// at the very end to maximize locality of the recursive delete and to | |||||||||
| 4150 | /// minimize the problems of invalidated instruction pointers as such pointers | |||||||||
| 4151 | /// are used heavily in the intermediate stages of the algorithm. | |||||||||
| 4152 | /// | |||||||||
| 4153 | /// We also record the alloca instructions deleted here so that they aren't | |||||||||
| 4154 | /// subsequently handed to mem2reg to promote. | |||||||||
| 4155 | void SROA::deleteDeadInstructions( | |||||||||
| 4156 | SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) { | |||||||||
| 4157 | while (!DeadInsts.empty()) { | |||||||||
| 4158 | Instruction *I = DeadInsts.pop_back_val(); | |||||||||
| 4159 | DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Deleting dead instruction: " << *I << "\n"; } } while (false); | |||||||||
| 4160 | ||||||||||
| 4161 | I->replaceAllUsesWith(UndefValue::get(I->getType())); | |||||||||
| 4162 | ||||||||||
| 4163 | for (Use &Operand : I->operands()) | |||||||||
| 4164 | if (Instruction *U = dyn_cast<Instruction>(Operand)) { | |||||||||
| 4165 | // Zero out the operand and see if it becomes trivially dead. | |||||||||
| 4166 | Operand = nullptr; | |||||||||
| 4167 | if (isInstructionTriviallyDead(U)) | |||||||||
| 4168 | DeadInsts.insert(U); | |||||||||
| 4169 | } | |||||||||
| 4170 | ||||||||||
| 4171 | if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) { | |||||||||
| 4172 | DeletedAllocas.insert(AI); | |||||||||
| 4173 | if (DbgDeclareInst *DbgDecl = FindAllocaDbgDeclare(AI)) | |||||||||
| 4174 | DbgDecl->eraseFromParent(); | |||||||||
| 4175 | } | |||||||||
| 4176 | ||||||||||
| 4177 | ++NumDeleted; | |||||||||
| 4178 | I->eraseFromParent(); | |||||||||
| 4179 | } | |||||||||
| 4180 | } | |||||||||
| 4181 | ||||||||||
| 4182 | /// \brief Promote the allocas, using the best available technique. | |||||||||
| 4183 | /// | |||||||||
| 4184 | /// This attempts to promote whatever allocas have been identified as viable in | |||||||||
| 4185 | /// the PromotableAllocas list. If that list is empty, there is nothing to do. | |||||||||
| 4186 | /// This function returns whether any promotion occurred. | |||||||||
| 4187 | bool SROA::promoteAllocas(Function &F) { | |||||||||
| 4188 | if (PromotableAllocas.empty()) | |||||||||
| 4189 | return false; | |||||||||
| 4190 | ||||||||||
| 4191 | NumPromoted += PromotableAllocas.size(); | |||||||||
| 4192 | ||||||||||
| 4193 | DEBUG(dbgs() << "Promoting allocas with mem2reg...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Promoting allocas with mem2reg...\n" ; } } while (false); | |||||||||
| 4194 | PromoteMemToReg(PromotableAllocas, *DT, AC); | |||||||||
| 4195 | PromotableAllocas.clear(); | |||||||||
| 4196 | return true; | |||||||||
| 4197 | } | |||||||||
| 4198 | ||||||||||
| 4199 | PreservedAnalyses SROA::runImpl(Function &F, DominatorTree &RunDT, | |||||||||
| 4200 | AssumptionCache &RunAC) { | |||||||||
| 4201 | DEBUG(dbgs() << "SROA function: " << F.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "SROA function: " << F.getName () << "\n"; } } while (false); | |||||||||
| 4202 | C = &F.getContext(); | |||||||||
| 4203 | DT = &RunDT; | |||||||||
| 4204 | AC = &RunAC; | |||||||||
| 4205 | ||||||||||
| 4206 | BasicBlock &EntryBB = F.getEntryBlock(); | |||||||||
| 4207 | for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(EntryBB.end()); | |||||||||
| 4208 | I != E; ++I) { | |||||||||
| 4209 | if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) | |||||||||
| 4210 | Worklist.insert(AI); | |||||||||
| 4211 | } | |||||||||
| 4212 | ||||||||||
| 4213 | bool Changed = false; | |||||||||
| 4214 | // A set of deleted alloca instruction pointers which should be removed from | |||||||||
| 4215 | // the list of promotable allocas. | |||||||||
| 4216 | SmallPtrSet<AllocaInst *, 4> DeletedAllocas; | |||||||||
| 4217 | ||||||||||
| 4218 | do { | |||||||||
| 4219 | while (!Worklist.empty()) { | |||||||||
| 4220 | Changed |= runOnAlloca(*Worklist.pop_back_val()); | |||||||||
| 4221 | deleteDeadInstructions(DeletedAllocas); | |||||||||
| 4222 | ||||||||||
| 4223 | // Remove the deleted allocas from various lists so that we don't try to | |||||||||
| 4224 | // continue processing them. | |||||||||
| 4225 | if (!DeletedAllocas.empty()) { | |||||||||
| 4226 | auto IsInSet = [&](AllocaInst *AI) { return DeletedAllocas.count(AI); }; | |||||||||
| 4227 | Worklist.remove_if(IsInSet); | |||||||||
| 4228 | PostPromotionWorklist.remove_if(IsInSet); | |||||||||
| 4229 | PromotableAllocas.erase(remove_if(PromotableAllocas, IsInSet), | |||||||||
| 4230 | PromotableAllocas.end()); | |||||||||
| 4231 | DeletedAllocas.clear(); | |||||||||
| 4232 | } | |||||||||
| 4233 | } | |||||||||
| 4234 | ||||||||||
| 4235 | Changed |= promoteAllocas(F); | |||||||||
| 4236 | ||||||||||
| 4237 | Worklist = PostPromotionWorklist; | |||||||||
| 4238 | PostPromotionWorklist.clear(); | |||||||||
| 4239 | } while (!Worklist.empty()); | |||||||||
| 4240 | ||||||||||
| 4241 | if (!Changed) | |||||||||
| 4242 | return PreservedAnalyses::all(); | |||||||||
| 4243 | ||||||||||
| 4244 | PreservedAnalyses PA; | |||||||||
| 4245 | PA.preserveSet<CFGAnalyses>(); | |||||||||
| 4246 | PA.preserve<GlobalsAA>(); | |||||||||
| 4247 | return PA; | |||||||||
| 4248 | } | |||||||||
| 4249 | ||||||||||
| 4250 | PreservedAnalyses SROA::run(Function &F, FunctionAnalysisManager &AM) { | |||||||||
| 4251 | return runImpl(F, AM.getResult<DominatorTreeAnalysis>(F), | |||||||||
| 4252 | AM.getResult<AssumptionAnalysis>(F)); | |||||||||
| 4253 | } | |||||||||
| 4254 | ||||||||||
| 4255 | /// A legacy pass for the legacy pass manager that wraps the \c SROA pass. | |||||||||
| 4256 | /// | |||||||||
| 4257 | /// This is in the llvm namespace purely to allow it to be a friend of the \c | |||||||||
| 4258 | /// SROA pass. | |||||||||
| 4259 | class llvm::sroa::SROALegacyPass : public FunctionPass { | |||||||||
| 4260 | /// The SROA implementation. | |||||||||
| 4261 | SROA Impl; | |||||||||
| 4262 | ||||||||||
| 4263 | public: | |||||||||
| 4264 | SROALegacyPass() : FunctionPass(ID) { | |||||||||
| 4265 | initializeSROALegacyPassPass(*PassRegistry::getPassRegistry()); | |||||||||
| 4266 | } | |||||||||
| 4267 | bool runOnFunction(Function &F) override { | |||||||||
| 4268 | if (skipFunction(F)) | |||||||||
| 4269 | return false; | |||||||||
| 4270 | ||||||||||
| 4271 | auto PA = Impl.runImpl( | |||||||||
| 4272 | F, getAnalysis<DominatorTreeWrapperPass>().getDomTree(), | |||||||||
| 4273 | getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F)); | |||||||||
| 4274 | return !PA.areAllPreserved(); | |||||||||
| 4275 | } | |||||||||
| 4276 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||||||||
| 4277 | AU.addRequired<AssumptionCacheTracker>(); | |||||||||
| 4278 | AU.addRequired<DominatorTreeWrapperPass>(); | |||||||||
| 4279 | AU.addPreserved<GlobalsAAWrapperPass>(); | |||||||||
| 4280 | AU.setPreservesCFG(); | |||||||||
| 4281 | } | |||||||||
| 4282 | ||||||||||
| 4283 | StringRef getPassName() const override { return "SROA"; } | |||||||||
| 4284 | static char ID; | |||||||||
| 4285 | }; | |||||||||
| 4286 | ||||||||||
| 4287 | char SROALegacyPass::ID = 0; | |||||||||
| 4288 | ||||||||||
| 4289 | FunctionPass *llvm::createSROAPass() { return new SROALegacyPass(); } | |||||||||
| 4290 | ||||||||||
| 4291 | INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",static void *initializeSROALegacyPassPassOnce(PassRegistry & Registry) { | |||||||||
| 4292 | "Scalar Replacement Of Aggregates", false, false)static void *initializeSROALegacyPassPassOnce(PassRegistry & Registry) { | |||||||||
| 4293 | INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry); | |||||||||
| 4294 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); | |||||||||
| 4295 | INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",PassInfo *PI = new PassInfo( "Scalar Replacement Of Aggregates" , "sroa", &SROALegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <SROALegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeSROALegacyPassPassFlag ; void llvm::initializeSROALegacyPassPass(PassRegistry &Registry ) { llvm::call_once(InitializeSROALegacyPassPassFlag, initializeSROALegacyPassPassOnce , std::ref(Registry)); } | |||||||||
| 4296 | false, false)PassInfo *PI = new PassInfo( "Scalar Replacement Of Aggregates" , "sroa", &SROALegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <SROALegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeSROALegacyPassPassFlag ; void llvm::initializeSROALegacyPassPass(PassRegistry &Registry ) { llvm::call_once(InitializeSROALegacyPassPassFlag, initializeSROALegacyPassPassOnce , std::ref(Registry)); } |