LLVM 24.0.0git
SROA.cpp
Go to the documentation of this file.
1//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This transformation implements the well known scalar replacement of
10/// aggregates transformation. It tries to identify promotable elements of an
11/// aggregate alloca, and promote them to registers. It will also try to
12/// convert uses of an element (or set of elements) of an alloca into a vector
13/// or bitfield-style integer scalar if appropriate.
14///
15/// It works to do this with minimal slicing of the alloca so that regions
16/// which are merely transferred in and out of external memory remain unchanged
17/// and are not decomposed to scalar code.
18///
19/// Because this also performs alloca promotion, it can be thought of as also
20/// serving the purpose of SSA formation. The algorithm iterates on the
21/// function until all opportunities for promotion have been realized.
22///
23//===----------------------------------------------------------------------===//
24
26#include "llvm/ADT/APInt.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/MapVector.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/SetVector.h"
36#include "llvm/ADT/Statistic.h"
37#include "llvm/ADT/StringRef.h"
38#include "llvm/ADT/Twine.h"
39#include "llvm/ADT/iterator.h"
44#include "llvm/Analysis/Loads.h"
48#include "llvm/IR/BasicBlock.h"
49#include "llvm/IR/Constant.h"
51#include "llvm/IR/Constants.h"
52#include "llvm/IR/DIBuilder.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfo.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GlobalAlias.h"
60#include "llvm/IR/IRBuilder.h"
61#include "llvm/IR/InstVisitor.h"
62#include "llvm/IR/Instruction.h"
65#include "llvm/IR/LLVMContext.h"
66#include "llvm/IR/Metadata.h"
67#include "llvm/IR/Module.h"
68#include "llvm/IR/Operator.h"
69#include "llvm/IR/PassManager.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
76#include "llvm/Pass.h"
80#include "llvm/Support/Debug.h"
88#include <algorithm>
89#include <cassert>
90#include <cstddef>
91#include <cstdint>
92#include <cstring>
93#include <iterator>
94#include <string>
95#include <tuple>
96#include <utility>
97#include <variant>
98#include <vector>
99
100using namespace llvm;
101
102#define DEBUG_TYPE "sroa"
103
104STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
105STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed");
106STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca");
107STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten");
108STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition");
109STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced");
110STATISTIC(NumPromoted, "Number of allocas promoted to SSA values");
111STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
112STATISTIC(NumLoadsPredicated,
113 "Number of loads rewritten into predicated loads to allow promotion");
115 NumStoresPredicated,
116 "Number of stores rewritten into predicated stores to allow promotion");
117STATISTIC(NumDeleted, "Number of instructions deleted");
118STATISTIC(NumVectorized, "Number of vectorized aggregates");
119
120namespace llvm {
121/// Disable running mem2reg during SROA in order to test or debug SROA.
122static cl::opt<bool> SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(false),
123 cl::Hidden);
124} // namespace llvm
125
126namespace {
127
128class AllocaSliceRewriter;
129class AllocaSlices;
130class Partition;
131
132class SelectHandSpeculativity {
133 unsigned char Storage = 0; // None are speculatable by default.
134 using TrueVal = Bitfield::Element<bool, 0, 1>; // Low 0'th bit.
135 using FalseVal = Bitfield::Element<bool, 1, 1>; // Low 1'th bit.
136public:
137 SelectHandSpeculativity() = default;
138 SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal);
139 bool isSpeculatable(bool isTrueVal) const;
140 bool areAllSpeculatable() const;
141 bool areAnySpeculatable() const;
142 bool areNoneSpeculatable() const;
143 // For interop as int half of PointerIntPair.
144 explicit operator intptr_t() const { return static_cast<intptr_t>(Storage); }
145 explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {}
146};
147static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char));
148
149using PossiblySpeculatableLoad =
151using UnspeculatableStore = StoreInst *;
152using RewriteableMemOp =
153 std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
154using RewriteableMemOps = SmallVector<RewriteableMemOp, 2>;
155
156/// An optimization pass providing Scalar Replacement of Aggregates.
157///
158/// This pass takes allocations which can be completely analyzed (that is, they
159/// don't escape) and tries to turn them into scalar SSA values. There are
160/// a few steps to this process.
161///
162/// 1) It takes allocations of aggregates and analyzes the ways in which they
163/// are used to try to split them into smaller allocations, ideally of
164/// a single scalar data type. It will split up memcpy and memset accesses
165/// as necessary and try to isolate individual scalar accesses.
166/// 2) It will transform accesses into forms which are suitable for SSA value
167/// promotion. This can be replacing a memset with a scalar store of an
168/// integer value, or it can involve speculating operations on a PHI or
169/// select to be a PHI or select of the results.
170/// 3) Finally, this will try to detect a pattern of accesses which map cleanly
171/// onto insert and extract operations on a vector value, and convert them to
172/// this form. By doing so, it will enable promotion of vector aggregates to
173/// SSA vector values.
174class SROA {
175 LLVMContext *const C;
176 DomTreeUpdater *const DTU;
177 AssumptionCache *const AC;
178 const bool PreserveCFG;
179 const bool AggregateToVector;
180
181 /// Worklist of alloca instructions to simplify.
182 ///
183 /// Each alloca in the function is added to this. Each new alloca formed gets
184 /// added to it as well to recursively simplify unless that alloca can be
185 /// directly promoted. Finally, each time we rewrite a use of an alloca other
186 /// the one being actively rewritten, we add it back onto the list if not
187 /// already present to ensure it is re-visited.
188 SmallSetVector<AllocaInst *, 16> Worklist;
189
190 /// A collection of instructions to delete.
191 /// We try to batch deletions to simplify code and make things a bit more
192 /// efficient. We also make sure there is no dangling pointers.
193 SmallVector<WeakVH, 8> DeadInsts;
194
195 /// Post-promotion worklist.
196 ///
197 /// Sometimes we discover an alloca which has a high probability of becoming
198 /// viable for SROA after a round of promotion takes place. In those cases,
199 /// the alloca is enqueued here for re-processing.
200 ///
201 /// Note that we have to be very careful to clear allocas out of this list in
202 /// the event they are deleted.
203 SmallSetVector<AllocaInst *, 16> PostPromotionWorklist;
204
205 /// A collection of alloca instructions we can directly promote.
206 SetVector<AllocaInst *, SmallVector<AllocaInst *>,
207 SmallPtrSet<AllocaInst *, 16>, 16>
208 PromotableAllocas;
209
210 /// A worklist of PHIs to speculate prior to promoting allocas.
211 ///
212 /// All of these PHIs have been checked for the safety of speculation and by
213 /// being speculated will allow promoting allocas currently in the promotable
214 /// queue.
215 SmallSetVector<PHINode *, 8> SpeculatablePHIs;
216
217 /// A worklist of PHIs whose stores should be predicated onto incoming edges.
218 ///
219 /// All of these PHIs have been checked so that rewriting their stores will
220 /// allow the corresponding allocas to be promoted. Edges which require
221 /// splitting are only present when CFG modification is allowed.
222 SmallSetVector<PHINode *, 8> PHIsWithStoreToRewrite;
223
224 /// A worklist of select instructions to rewrite prior to promoting
225 /// allocas.
226 SmallMapVector<SelectInst *, RewriteableMemOps, 8> SelectsToRewrite;
227
228 /// Select instructions that use an alloca and are subsequently loaded can be
229 /// rewritten to load both input pointers and then select between the result,
230 /// allowing the load of the alloca to be promoted.
231 /// From this:
232 /// %P2 = select i1 %cond, ptr %Alloca, ptr %Other
233 /// %V = load <type>, ptr %P2
234 /// to:
235 /// %V1 = load <type>, ptr %Alloca -> will be mem2reg'd
236 /// %V2 = load <type>, ptr %Other
237 /// %V = select i1 %cond, <type> %V1, <type> %V2
238 ///
239 /// We can do this to a select if its only uses are loads
240 /// and if either the operand to the select can be loaded unconditionally,
241 /// or if we are allowed to perform CFG modifications.
242 static std::optional<RewriteableMemOps>
243 isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG);
244
245public:
246 SROA(LLVMContext *C, DomTreeUpdater *DTU, AssumptionCache *AC,
247 SROAOptions Options)
248 : C(C), DTU(DTU), AC(AC),
249 PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
250 AggregateToVector(Options.AggregateToVector) {}
251
252 /// Main run method used by both the SROAPass and by the legacy pass.
253 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runSROA(Function &F);
254
255private:
256 friend class AllocaSliceRewriter;
257
258 bool presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS);
259 std::pair<AllocaInst *, uint64_t>
260 rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P);
261 bool splitAlloca(AllocaInst &AI, AllocaSlices &AS);
262 bool propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS);
263 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runOnAlloca(AllocaInst &AI);
264 void clobberUse(Use &U);
265 bool deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas);
266 bool promoteAllocas();
267};
268
269} // end anonymous namespace
270
271/// Calculate the fragment of a variable to use when slicing a store
272/// based on the slice dimensions, existing fragment, and base storage
273/// fragment.
274/// Results:
275/// UseFrag - Use Target as the new fragment.
276/// UseNoFrag - The new slice already covers the whole variable.
277/// Skip - The new alloca slice doesn't include this variable.
278/// FIXME: Can we use calculateFragmentIntersect instead?
279namespace {
280enum FragCalcResult { UseFrag, UseNoFrag, Skip };
281}
282static FragCalcResult
284 uint64_t NewStorageSliceOffsetInBits,
285 uint64_t NewStorageSliceSizeInBits,
286 std::optional<DIExpression::FragmentInfo> StorageFragment,
287 std::optional<DIExpression::FragmentInfo> CurrentFragment,
289 // If the base storage describes part of the variable apply the offset and
290 // the size constraint.
291 if (StorageFragment) {
292 Target.SizeInBits =
293 std::min(NewStorageSliceSizeInBits, StorageFragment->SizeInBits);
294 Target.OffsetInBits =
295 NewStorageSliceOffsetInBits + StorageFragment->OffsetInBits;
296 } else {
297 Target.SizeInBits = NewStorageSliceSizeInBits;
298 Target.OffsetInBits = NewStorageSliceOffsetInBits;
299 }
300
301 // If this slice extracts the entirety of an independent variable from a
302 // larger alloca, do not produce a fragment expression, as the variable is
303 // not fragmented.
304 if (!CurrentFragment) {
305 if (auto Size = Variable->getSizeInBits()) {
306 // Treat the current fragment as covering the whole variable.
307 CurrentFragment = DIExpression::FragmentInfo(*Size, 0);
308 if (Target == CurrentFragment)
309 return UseNoFrag;
310 }
311 }
312
313 // No additional work to do if there isn't a fragment already, or there is
314 // but it already exactly describes the new assignment.
315 if (!CurrentFragment || *CurrentFragment == Target)
316 return UseFrag;
317
318 // Reject the target fragment if it doesn't fit wholly within the current
319 // fragment. TODO: We could instead chop up the target to fit in the case of
320 // a partial overlap.
321 if (Target.startInBits() < CurrentFragment->startInBits() ||
322 Target.endInBits() > CurrentFragment->endInBits())
323 return Skip;
324
325 // Target fits within the current fragment, return it.
326 return UseFrag;
327}
328
330 return DebugVariable(DVR->getVariable(), std::nullopt,
331 DVR->getDebugLoc().getInlinedAt());
332}
333
334/// Find linked dbg.assign and generate a new one with the correct
335/// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the
336/// value component is copied from the old dbg.assign to the new.
337/// \param OldAlloca Alloca for the variable before splitting.
338/// \param IsSplit True if the store (not necessarily alloca)
339/// is being split.
340/// \param OldAllocaOffsetInBits Offset of the slice taken from OldAlloca.
341/// \param SliceSizeInBits New number of bits being written to.
342/// \param OldInst Instruction that is being split.
343/// \param Inst New instruction performing this part of the
344/// split store.
345/// \param Dest Store destination.
346/// \param Value Stored value.
347/// \param DL Datalayout.
348static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
349 uint64_t OldAllocaOffsetInBits,
350 uint64_t SliceSizeInBits, Instruction *OldInst,
351 Instruction *Inst, Value *Dest, Value *Value,
352 const DataLayout &DL) {
353 // If we want allocas to be migrated using this helper then we need to ensure
354 // that the BaseFragments map code still works. A simple solution would be
355 // to choose to always clone alloca dbg_assigns (rather than sometimes
356 // "stealing" them).
357 assert(!isa<AllocaInst>(Inst) && "Unexpected alloca");
358
359 auto DVRAssignMarkerRange = at::getDVRAssignmentMarkers(OldInst);
360 // Nothing to do if OldInst has no linked dbg.assign intrinsics.
361 if (DVRAssignMarkerRange.empty())
362 return;
363
364 LLVM_DEBUG(dbgs() << " migrateDebugInfo\n");
365 LLVM_DEBUG(dbgs() << " OldAlloca: " << *OldAlloca << "\n");
366 LLVM_DEBUG(dbgs() << " IsSplit: " << IsSplit << "\n");
367 LLVM_DEBUG(dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits
368 << "\n");
369 LLVM_DEBUG(dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n");
370 LLVM_DEBUG(dbgs() << " OldInst: " << *OldInst << "\n");
371 LLVM_DEBUG(dbgs() << " Inst: " << *Inst << "\n");
372 LLVM_DEBUG(dbgs() << " Dest: " << *Dest << "\n");
373 if (Value)
374 LLVM_DEBUG(dbgs() << " Value: " << *Value << "\n");
375
376 /// Map of aggregate variables to their fragment associated with OldAlloca.
378 BaseFragments;
379 for (auto *DVR : at::getDVRAssignmentMarkers(OldAlloca))
380 BaseFragments[getAggregateVariable(DVR)] =
381 DVR->getExpression()->getFragmentInfo();
382
383 // The new inst needs a DIAssignID unique metadata tag (if OldInst has
384 // one). It shouldn't already have one: assert this assumption.
385 assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
386 DIAssignID *NewID = nullptr;
387 auto &Ctx = Inst->getContext();
388 DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false);
389 assert(OldAlloca->isStaticAlloca());
390
391 auto MigrateDbgAssign = [&](DbgVariableRecord *DbgAssign) {
392 LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign
393 << "\n");
394 auto *Expr = DbgAssign->getExpression();
395 bool SetKillLocation = false;
396
397 if (IsSplit) {
398 std::optional<DIExpression::FragmentInfo> BaseFragment;
399 {
400 auto R = BaseFragments.find(getAggregateVariable(DbgAssign));
401 if (R == BaseFragments.end())
402 return;
403 BaseFragment = R->second;
404 }
405 std::optional<DIExpression::FragmentInfo> CurrentFragment =
406 Expr->getFragmentInfo();
407 DIExpression::FragmentInfo NewFragment;
408 FragCalcResult Result = calculateFragment(
409 DbgAssign->getVariable(), OldAllocaOffsetInBits, SliceSizeInBits,
410 BaseFragment, CurrentFragment, NewFragment);
411
412 if (Result == Skip)
413 return;
414 if (Result == UseFrag && !(NewFragment == CurrentFragment)) {
415 if (CurrentFragment) {
416 // Rewrite NewFragment to be relative to the existing one (this is
417 // what createFragmentExpression wants). CalculateFragment has
418 // already resolved the size for us. FIXME: Should it return the
419 // relative fragment too?
420 NewFragment.OffsetInBits -= CurrentFragment->OffsetInBits;
421 }
422 // Add the new fragment info to the existing expression if possible.
424 Expr, NewFragment.OffsetInBits, NewFragment.SizeInBits)) {
425 Expr = *E;
426 } else {
427 // Otherwise, add the new fragment info to an empty expression and
428 // discard the value component of this dbg.assign as the value cannot
429 // be computed with the new fragment.
431 DIExpression::get(Expr->getContext(), {}),
432 NewFragment.OffsetInBits, NewFragment.SizeInBits);
433 SetKillLocation = true;
434 }
435 }
436 }
437
438 // If we haven't created a DIAssignID ID do that now and attach it to Inst.
439 if (!NewID) {
440 NewID = DIAssignID::getDistinct(Ctx);
441 Inst->setMetadata(LLVMContext::MD_DIAssignID, NewID);
442 }
443
444 DbgVariableRecord *NewAssign;
445 if (IsSplit) {
446 ::Value *NewValue = Value ? Value : DbgAssign->getValue();
448 Inst, NewValue, DbgAssign->getVariable(), Expr, Dest,
449 DIExpression::get(Expr->getContext(), {}), DbgAssign->getDebugLoc()));
450 } else {
451 // The store is not split, simply steal the existing dbg_assign.
452 NewAssign = DbgAssign;
453 NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
454 NewAssign->setAddress(Dest);
455 if (Value)
456 NewAssign->replaceVariableLocationOp(0u, Value);
457 assert(Expr == NewAssign->getExpression());
458 }
459
460 // If we've updated the value but the original dbg.assign has an arglist
461 // then kill it now - we can't use the requested new value.
462 // We can't replace the DIArgList with the new value as it'd leave
463 // the DIExpression in an invalid state (DW_OP_LLVM_arg operands without
464 // an arglist). And we can't keep the DIArgList in case the linked store
465 // is being split - in which case the DIArgList + expression may no longer
466 // be computing the correct value.
467 // This should be a very rare situation as it requires the value being
468 // stored to differ from the dbg.assign (i.e., the value has been
469 // represented differently in the debug intrinsic for some reason).
470 SetKillLocation |=
471 Value && (DbgAssign->hasArgList() ||
472 !DbgAssign->getExpression()->isSingleLocationExpression());
473 if (SetKillLocation)
474 NewAssign->setKillLocation();
475
476 // We could use more precision here at the cost of some additional (code)
477 // complexity - if the original dbg.assign was adjacent to its store, we
478 // could position this new dbg.assign adjacent to its store rather than the
479 // old dbg.assgn. That would result in interleaved dbg.assigns rather than
480 // what we get now:
481 // split store !1
482 // split store !2
483 // dbg.assign !1
484 // dbg.assign !2
485 // This (current behaviour) results results in debug assignments being
486 // noted as slightly offset (in code) from the store. In practice this
487 // should have little effect on the debugging experience due to the fact
488 // that all the split stores should get the same line number.
489 if (NewAssign != DbgAssign) {
490 NewAssign->moveBefore(DbgAssign->getIterator());
491 NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
492 }
493 LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
494 };
495
496 for_each(DVRAssignMarkerRange, MigrateDbgAssign);
497}
498
499namespace {
500
501/// A custom IRBuilder inserter which prefixes all names, but only in
502/// Assert builds.
503class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter {
504 std::string Prefix;
505
506 Twine getNameWithPrefix(const Twine &Name) const {
507 return Name.isTriviallyEmpty() ? Name : Prefix + Name;
508 }
509
510public:
511 void SetNamePrefix(const Twine &P) { Prefix = P.str(); }
512
513 void InsertHelper(Instruction *I, const Twine &Name,
514 BasicBlock::iterator InsertPt) const override {
515 IRBuilderDefaultInserter::InsertHelper(I, getNameWithPrefix(Name),
516 InsertPt);
517 }
518};
519
520/// Provide a type for IRBuilder that drops names in release builds.
522
523/// A used slice of an alloca.
524///
525/// This structure represents a slice of an alloca used by some instruction. It
526/// stores both the begin and end offsets of this use, a pointer to the use
527/// itself, and a flag indicating whether we can classify the use as splittable
528/// or not when forming partitions of the alloca.
529class Slice {
530 /// The beginning offset of the range.
531 uint64_t BeginOffset = 0;
532
533 /// The ending offset, not included in the range.
534 uint64_t EndOffset = 0;
535
536 /// Storage for both the use of this slice and whether it can be
537 /// split.
538 PointerIntPair<Use *, 1, bool> UseAndIsSplittable;
539
540public:
541 Slice() = default;
542
543 Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable)
544 : BeginOffset(BeginOffset), EndOffset(EndOffset),
545 UseAndIsSplittable(U, IsSplittable) {}
546
547 uint64_t beginOffset() const { return BeginOffset; }
548 uint64_t endOffset() const { return EndOffset; }
549
550 bool isSplittable() const { return UseAndIsSplittable.getInt(); }
551 void makeUnsplittable() { UseAndIsSplittable.setInt(false); }
552
553 Use *getUse() const { return UseAndIsSplittable.getPointer(); }
554
555 bool isDead() const { return getUse() == nullptr; }
556 void kill() { UseAndIsSplittable.setPointer(nullptr); }
557
558 /// Support for ordering ranges.
559 ///
560 /// This provides an ordering over ranges such that start offsets are
561 /// always increasing, and within equal start offsets, the end offsets are
562 /// decreasing. Thus the spanning range comes first in a cluster with the
563 /// same start position.
564 bool operator<(const Slice &RHS) const {
565 if (beginOffset() < RHS.beginOffset())
566 return true;
567 if (beginOffset() > RHS.beginOffset())
568 return false;
569 if (isSplittable() != RHS.isSplittable())
570 return !isSplittable();
571 if (endOffset() > RHS.endOffset())
572 return true;
573 return false;
574 }
575
576 /// Support comparison with a single offset to allow binary searches.
577 [[maybe_unused]] friend bool operator<(const Slice &LHS, uint64_t RHSOffset) {
578 return LHS.beginOffset() < RHSOffset;
579 }
580 [[maybe_unused]] friend bool operator<(uint64_t LHSOffset, const Slice &RHS) {
581 return LHSOffset < RHS.beginOffset();
582 }
583
584 bool operator==(const Slice &RHS) const {
585 return isSplittable() == RHS.isSplittable() &&
586 beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset();
587 }
588 bool operator!=(const Slice &RHS) const { return !operator==(RHS); }
589};
590
591/// Representation of the alloca slices.
592///
593/// This class represents the slices of an alloca which are formed by its
594/// various uses. If a pointer escapes, we can't fully build a representation
595/// for the slices used and we reflect that in this structure. The uses are
596/// stored, sorted by increasing beginning offset and with unsplittable slices
597/// starting at a particular offset before splittable slices.
598class AllocaSlices {
599public:
600 /// Construct the slices of a particular alloca.
601 AllocaSlices(const DataLayout &DL, AllocaInst &AI);
602
603 /// Test whether a pointer to the allocation escapes our analysis.
604 ///
605 /// If this is true, the slices are never fully built and should be
606 /// ignored.
607 bool isEscaped() const { return PointerEscapingInstr; }
608 bool isEscapedReadOnly() const { return PointerEscapingInstrReadOnly; }
609
610 /// Support for iterating over the slices.
611 /// @{
612 using iterator = SmallVectorImpl<Slice>::iterator;
613 using range = iterator_range<iterator>;
614
615 iterator begin() { return Slices.begin(); }
616 iterator end() { return Slices.end(); }
617
618 using const_iterator = SmallVectorImpl<Slice>::const_iterator;
619 using const_range = iterator_range<const_iterator>;
620
621 const_iterator begin() const { return Slices.begin(); }
622 const_iterator end() const { return Slices.end(); }
623 /// @}
624
625 /// Erase a range of slices.
626 void erase(iterator Start, iterator Stop) { Slices.erase(Start, Stop); }
627
628 /// Insert new slices for this alloca.
629 ///
630 /// This moves the slices into the alloca's slices collection, and re-sorts
631 /// everything so that the usual ordering properties of the alloca's slices
632 /// hold.
633 void insert(ArrayRef<Slice> NewSlices) {
634 int OldSize = Slices.size();
635 Slices.append(NewSlices.begin(), NewSlices.end());
636 auto SliceI = Slices.begin() + OldSize;
637 std::stable_sort(SliceI, Slices.end());
638 std::inplace_merge(Slices.begin(), SliceI, Slices.end());
639 }
640
641 // Forward declare the iterator and range accessor for walking the
642 // partitions.
643 class partition_iterator;
645
646 /// Access the dead users for this alloca.
647 ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; }
648
649 /// Access Uses that should be dropped if the alloca is promotable.
650 ArrayRef<Use *> getDeadUsesIfPromotable() const {
651 return DeadUseIfPromotable;
652 }
653
654 /// Access the dead operands referring to this alloca.
655 ///
656 /// These are operands which have cannot actually be used to refer to the
657 /// alloca as they are outside its range and the user doesn't correct for
658 /// that. These mostly consist of PHI node inputs and the like which we just
659 /// need to replace with undef.
660 ArrayRef<Use *> getDeadOperands() const { return DeadOperands; }
661
662#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
663 void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const;
664 void printSlice(raw_ostream &OS, const_iterator I,
665 StringRef Indent = " ") const;
666 void printUse(raw_ostream &OS, const_iterator I,
667 StringRef Indent = " ") const;
668 void print(raw_ostream &OS) const;
669 void dump(const_iterator I) const;
670 void dump() const;
671#endif
672
673private:
674 template <typename DerivedT, typename RetT = void> class BuilderBase;
675 class SliceBuilder;
676
677 friend class AllocaSlices::SliceBuilder;
678
679#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
680 /// Handle to alloca instruction to simplify method interfaces.
681 AllocaInst &AI;
682#endif
683
684 /// The instruction responsible for this alloca not having a known set
685 /// of slices.
686 ///
687 /// When an instruction (potentially) escapes the pointer to the alloca, we
688 /// store a pointer to that here and abort trying to form slices of the
689 /// alloca. This will be null if the alloca slices are analyzed successfully.
690 Instruction *PointerEscapingInstr;
691 Instruction *PointerEscapingInstrReadOnly;
692
693 /// The slices of the alloca.
694 ///
695 /// We store a vector of the slices formed by uses of the alloca here. This
696 /// vector is sorted by increasing begin offset, and then the unsplittable
697 /// slices before the splittable ones. See the Slice inner class for more
698 /// details.
700
701 /// Instructions which will become dead if we rewrite the alloca.
702 ///
703 /// Note that these are not separated by slice. This is because we expect an
704 /// alloca to be completely rewritten or not rewritten at all. If rewritten,
705 /// all these instructions can simply be removed and replaced with poison as
706 /// they come from outside of the allocated space.
707 SmallVector<Instruction *, 8> DeadUsers;
708
709 /// Uses which will become dead if can promote the alloca.
710 SmallVector<Use *, 8> DeadUseIfPromotable;
711
712 /// Operands which will become dead if we rewrite the alloca.
713 ///
714 /// These are operands that in their particular use can be replaced with
715 /// poison when we rewrite the alloca. These show up in out-of-bounds inputs
716 /// to PHI nodes and the like. They aren't entirely dead (there might be
717 /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
718 /// want to swap this particular input for poison to simplify the use lists of
719 /// the alloca.
720 SmallVector<Use *, 8> DeadOperands;
721};
722
723/// A partition of the slices.
724///
725/// An ephemeral representation for a range of slices which can be viewed as
726/// a partition of the alloca. This range represents a span of the alloca's
727/// memory which cannot be split, and provides access to all of the slices
728/// overlapping some part of the partition.
729///
730/// Objects of this type are produced by traversing the alloca's slices, but
731/// are only ephemeral and not persistent.
732class Partition {
733private:
734 friend class AllocaSlices;
735 friend class AllocaSlices::partition_iterator;
736
737 using iterator = AllocaSlices::iterator;
738
739 /// The beginning and ending offsets of the alloca for this
740 /// partition.
741 uint64_t BeginOffset = 0, EndOffset = 0;
742
743 /// The start and end iterators of this partition.
744 iterator SI, SJ;
745
746 /// A collection of split slice tails overlapping the partition.
747 SmallVector<Slice *, 4> SplitTails;
748
749 /// Raw constructor builds an empty partition starting and ending at
750 /// the given iterator.
751 Partition(iterator SI) : SI(SI), SJ(SI) {}
752
753public:
754 /// The start offset of this partition.
755 ///
756 /// All of the contained slices start at or after this offset.
757 uint64_t beginOffset() const { return BeginOffset; }
758
759 /// The end offset of this partition.
760 ///
761 /// All of the contained slices end at or before this offset.
762 uint64_t endOffset() const { return EndOffset; }
763
764 /// The size of the partition.
765 ///
766 /// Note that this can never be zero.
767 uint64_t size() const {
768 assert(BeginOffset < EndOffset && "Partitions must span some bytes!");
769 return EndOffset - BeginOffset;
770 }
771
772 /// Test whether this partition contains no slices, and merely spans
773 /// a region occupied by split slices.
774 bool empty() const { return SI == SJ; }
775
776 /// \name Iterate slices that start within the partition.
777 /// These may be splittable or unsplittable. They have a begin offset >= the
778 /// partition begin offset.
779 /// @{
780 // FIXME: We should probably define a "concat_iterator" helper and use that
781 // to stitch together pointee_iterators over the split tails and the
782 // contiguous iterators of the partition. That would give a much nicer
783 // interface here. We could then additionally expose filtered iterators for
784 // split, unsplit, and unsplittable splices based on the usage patterns.
785 iterator begin() const { return SI; }
786 iterator end() const { return SJ; }
787 /// @}
788
789 /// Get the sequence of split slice tails.
790 ///
791 /// These tails are of slices which start before this partition but are
792 /// split and overlap into the partition. We accumulate these while forming
793 /// partitions.
794 ArrayRef<Slice *> splitSliceTails() const { return SplitTails; }
795};
796
797} // end anonymous namespace
798
799/// An iterator over partitions of the alloca's slices.
800///
801/// This iterator implements the core algorithm for partitioning the alloca's
802/// slices. It is a forward iterator as we don't support backtracking for
803/// efficiency reasons, and re-use a single storage area to maintain the
804/// current set of split slices.
805///
806/// It is templated on the slice iterator type to use so that it can operate
807/// with either const or non-const slice iterators.
809 : public iterator_facade_base<partition_iterator, std::forward_iterator_tag,
810 Partition> {
811 friend class AllocaSlices;
812
813 /// Most of the state for walking the partitions is held in a class
814 /// with a nice interface for examining them.
815 Partition P;
816
817 /// We need to keep the end of the slices to know when to stop.
818 AllocaSlices::iterator SE;
819
820 /// We also need to keep track of the maximum split end offset seen.
821 /// FIXME: Do we really?
822 uint64_t MaxSplitSliceEndOffset = 0;
823
824 /// Sets the partition to be empty at given iterator, and sets the
825 /// end iterator.
826 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
827 : P(SI), SE(SE) {
828 // If not already at the end, advance our state to form the initial
829 // partition.
830 if (SI != SE)
831 advance();
832 }
833
834 /// Advance the iterator to the next partition.
835 ///
836 /// Requires that the iterator not be at the end of the slices.
837 void advance() {
838 assert((P.SI != SE || !P.SplitTails.empty()) &&
839 "Cannot advance past the end of the slices!");
840
841 // Clear out any split uses which have ended.
842 if (!P.SplitTails.empty()) {
843 if (P.EndOffset >= MaxSplitSliceEndOffset) {
844 // If we've finished all splits, this is easy.
845 P.SplitTails.clear();
846 MaxSplitSliceEndOffset = 0;
847 } else {
848 // Remove the uses which have ended in the prior partition. This
849 // cannot change the max split slice end because we just checked that
850 // the prior partition ended prior to that max.
851 llvm::erase_if(P.SplitTails,
852 [&](Slice *S) { return S->endOffset() <= P.EndOffset; });
853 assert(llvm::any_of(P.SplitTails,
854 [&](Slice *S) {
855 return S->endOffset() == MaxSplitSliceEndOffset;
856 }) &&
857 "Could not find the current max split slice offset!");
858 assert(llvm::all_of(P.SplitTails,
859 [&](Slice *S) {
860 return S->endOffset() <= MaxSplitSliceEndOffset;
861 }) &&
862 "Max split slice end offset is not actually the max!");
863 }
864 }
865
866 // If P.SI is already at the end, then we've cleared the split tail and
867 // now have an end iterator.
868 if (P.SI == SE) {
869 assert(P.SplitTails.empty() && "Failed to clear the split slices!");
870 return;
871 }
872
873 // If we had a non-empty partition previously, set up the state for
874 // subsequent partitions.
875 if (P.SI != P.SJ) {
876 // Accumulate all the splittable slices which started in the old
877 // partition into the split list.
878 for (Slice &S : P)
879 if (S.isSplittable() && S.endOffset() > P.EndOffset) {
880 P.SplitTails.push_back(&S);
881 MaxSplitSliceEndOffset =
882 std::max(S.endOffset(), MaxSplitSliceEndOffset);
883 }
884
885 // Start from the end of the previous partition.
886 P.SI = P.SJ;
887
888 // If P.SI is now at the end, we at most have a tail of split slices.
889 if (P.SI == SE) {
890 P.BeginOffset = P.EndOffset;
891 P.EndOffset = MaxSplitSliceEndOffset;
892 return;
893 }
894
895 // If the we have split slices and the next slice is after a gap and is
896 // not splittable immediately form an empty partition for the split
897 // slices up until the next slice begins.
898 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
899 !P.SI->isSplittable()) {
900 P.BeginOffset = P.EndOffset;
901 P.EndOffset = P.SI->beginOffset();
902 return;
903 }
904 }
905
906 // OK, we need to consume new slices. Set the end offset based on the
907 // current slice, and step SJ past it. The beginning offset of the
908 // partition is the beginning offset of the next slice unless we have
909 // pre-existing split slices that are continuing, in which case we begin
910 // at the prior end offset.
911 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
912 P.EndOffset = P.SI->endOffset();
913 ++P.SJ;
914
915 // There are two strategies to form a partition based on whether the
916 // partition starts with an unsplittable slice or a splittable slice.
917 if (!P.SI->isSplittable()) {
918 // When we're forming an unsplittable region, it must always start at
919 // the first slice and will extend through its end.
920 assert(P.BeginOffset == P.SI->beginOffset());
921
922 // Form a partition including all of the overlapping slices with this
923 // unsplittable slice.
924 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
925 if (!P.SJ->isSplittable())
926 P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset());
927 ++P.SJ;
928 }
929
930 // We have a partition across a set of overlapping unsplittable
931 // partitions.
932 return;
933 }
934
935 // If we're starting with a splittable slice, then we need to form
936 // a synthetic partition spanning it and any other overlapping splittable
937 // splices.
938 assert(P.SI->isSplittable() && "Forming a splittable partition!");
939
940 // Collect all of the overlapping splittable slices.
941 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset &&
942 P.SJ->isSplittable()) {
943 P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset());
944 ++P.SJ;
945 }
946
947 // Back upiP.EndOffset if we ended the span early when encountering an
948 // unsplittable slice. This synthesizes the early end offset of
949 // a partition spanning only splittable slices.
950 if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
951 assert(!P.SJ->isSplittable());
952 P.EndOffset = P.SJ->beginOffset();
953 }
954 }
955
956public:
957 bool operator==(const partition_iterator &RHS) const {
958 assert(SE == RHS.SE &&
959 "End iterators don't match between compared partition iterators!");
960
961 // The observed positions of partitions is marked by the P.SI iterator and
962 // the emptiness of the split slices. The latter is only relevant when
963 // P.SI == SE, as the end iterator will additionally have an empty split
964 // slices list, but the prior may have the same P.SI and a tail of split
965 // slices.
966 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
967 assert(P.SJ == RHS.P.SJ &&
968 "Same set of slices formed two different sized partitions!");
969 assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&
970 "Same slice position with differently sized non-empty split "
971 "slice tails!");
972 return true;
973 }
974 return false;
975 }
976
977 partition_iterator &operator++() {
978 advance();
979 return *this;
980 }
981
982 Partition &operator*() { return P; }
983};
984
985/// A forward range over the partitions of the alloca's slices.
986///
987/// This accesses an iterator range over the partitions of the alloca's
988/// slices. It computes these partitions on the fly based on the overlapping
989/// offsets of the slices and the ability to split them. It will visit "empty"
990/// partitions to cover regions of the alloca only accessed via split
991/// slices.
992iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() {
993 return make_range(partition_iterator(begin(), end()),
994 partition_iterator(end(), end()));
995}
996
998 // If the condition being selected on is a constant or the same value is
999 // being selected between, fold the select. Yes this does (rarely) happen
1000 // early on.
1001 if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition()))
1002 return SI.getOperand(1 + CI->isZero());
1003 if (SI.getOperand(1) == SI.getOperand(2))
1004 return SI.getOperand(1);
1005
1006 return nullptr;
1007}
1008
1009/// A helper that folds a PHI node or a select.
1011 if (PHINode *PN = dyn_cast<PHINode>(&I)) {
1012 // If PN merges together the same value, return that value.
1013 return PN->hasConstantValue();
1014 }
1016}
1017
1018/// Builder for the alloca slices.
1019///
1020/// This class builds a set of alloca slices by recursively visiting the uses
1021/// of an alloca and making a slice for each load and store at each offset.
1022class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
1023 friend class PtrUseVisitor<SliceBuilder>;
1024 friend class InstVisitor<SliceBuilder>;
1025
1026 using Base = PtrUseVisitor<SliceBuilder>;
1027
1028 const uint64_t AllocSize;
1029 AllocaSlices &AS;
1030
1031 SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap;
1033
1034 /// Set to de-duplicate dead instructions found in the use walk.
1035 SmallPtrSet<Instruction *, 4> VisitedDeadInsts;
1036
1037public:
1038 SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
1040 AllocSize(AI.getAllocationSize(DL)->getFixedValue()), AS(AS) {}
1041
1042private:
1043 void markAsDead(Instruction &I) {
1044 if (VisitedDeadInsts.insert(&I).second)
1045 AS.DeadUsers.push_back(&I);
1046 }
1047
1048 void insertUse(Instruction &I, const APInt &Offset, uint64_t Size,
1049 bool IsSplittable = false) {
1050 // Completely skip uses which have a zero size or start either before or
1051 // past the end of the allocation.
1052 if (Size == 0 || Offset.uge(AllocSize)) {
1053 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"
1054 << Offset
1055 << " which has zero size or starts outside of the "
1056 << AllocSize << " byte alloca:\n"
1057 << " alloca: " << AS.AI << "\n"
1058 << " use: " << I << "\n");
1059 return markAsDead(I);
1060 }
1061
1062 uint64_t BeginOffset = Offset.getZExtValue();
1063 uint64_t EndOffset = BeginOffset + Size;
1064
1065 // Clamp the end offset to the end of the allocation. Note that this is
1066 // formulated to handle even the case where "BeginOffset + Size" overflows.
1067 // This may appear superficially to be something we could ignore entirely,
1068 // but that is not so! There may be widened loads or PHI-node uses where
1069 // some instructions are dead but not others. We can't completely ignore
1070 // them, and so have to record at least the information here.
1071 assert(AllocSize >= BeginOffset); // Established above.
1072 if (Size > AllocSize - BeginOffset) {
1073 LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"
1074 << Offset << " to remain within the " << AllocSize
1075 << " byte alloca:\n"
1076 << " alloca: " << AS.AI << "\n"
1077 << " use: " << I << "\n");
1078 EndOffset = AllocSize;
1079 }
1080
1081 AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable));
1082 }
1083
1084 void visitBitCastInst(BitCastInst &BC) {
1085 if (BC.use_empty())
1086 return markAsDead(BC);
1087
1088 return Base::visitBitCastInst(BC);
1089 }
1090
1091 void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
1092 if (ASC.use_empty())
1093 return markAsDead(ASC);
1094
1095 return Base::visitAddrSpaceCastInst(ASC);
1096 }
1097
1098 void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
1099 if (GEPI.use_empty())
1100 return markAsDead(GEPI);
1101
1102 return Base::visitGetElementPtrInst(GEPI);
1103 }
1104
1105 void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset,
1106 uint64_t Size, bool IsVolatile) {
1107 // We allow splitting of non-volatile loads and stores where the type is an
1108 // integer type. These may be used to implement 'memcpy' or other "transfer
1109 // of bits" patterns.
1110 bool IsSplittable =
1111 Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty);
1112
1113 insertUse(I, Offset, Size, IsSplittable);
1114 }
1115
1116 void visitLoadInst(LoadInst &LI) {
1117 assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&
1118 "All simple FCA loads should have been pre-split");
1119
1120 // If there is a load with an unknown offset, we can still perform store
1121 // to load forwarding for other known-offset loads.
1122 if (!IsOffsetKnown)
1123 return PI.setEscapedReadOnly(&LI);
1124
1125 TypeSize Size = DL.getTypeStoreSize(LI.getType());
1126 if (Size.isScalable()) {
1127 unsigned VScale = LI.getFunction()->getVScaleValue();
1128 if (!VScale)
1129 return PI.setAborted(&LI);
1130
1131 Size = TypeSize::getFixed(Size.getKnownMinValue() * VScale);
1132 }
1133
1134 return handleLoadOrStore(LI.getType(), LI, Offset, Size.getFixedValue(),
1135 LI.isVolatile());
1136 }
1137
1138 void visitStoreInst(StoreInst &SI) {
1139 Value *ValOp = SI.getValueOperand();
1140 if (ValOp == *U)
1141 return PI.setEscapedAndAborted(&SI);
1142 if (!IsOffsetKnown)
1143 return PI.setAborted(&SI);
1144
1145 TypeSize StoreSize = DL.getTypeStoreSize(ValOp->getType());
1146 if (StoreSize.isScalable()) {
1147 unsigned VScale = SI.getFunction()->getVScaleValue();
1148 if (!VScale)
1149 return PI.setAborted(&SI);
1150
1151 StoreSize = TypeSize::getFixed(StoreSize.getKnownMinValue() * VScale);
1152 }
1153
1154 uint64_t Size = StoreSize.getFixedValue();
1155
1156 // If this memory access can be shown to *statically* extend outside the
1157 // bounds of the allocation, it's behavior is undefined, so simply
1158 // ignore it. Note that this is more strict than the generic clamping
1159 // behavior of insertUse. We also try to handle cases which might run the
1160 // risk of overflow.
1161 // FIXME: We should instead consider the pointer to have escaped if this
1162 // function is being instrumented for addressing bugs or race conditions.
1163 if (Size > AllocSize || Offset.ugt(AllocSize - Size)) {
1164 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"
1165 << Offset << " which extends past the end of the "
1166 << AllocSize << " byte alloca:\n"
1167 << " alloca: " << AS.AI << "\n"
1168 << " use: " << SI << "\n");
1169 return markAsDead(SI);
1170 }
1171
1172 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&
1173 "All simple FCA stores should have been pre-split");
1174 handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile());
1175 }
1176
1177 void visitMemSetInst(MemSetInst &II) {
1178 assert(II.getRawDest() == *U && "Pointer use is not the destination?");
1179 ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
1180 if ((Length && Length->getValue() == 0) ||
1181 (IsOffsetKnown && Offset.uge(AllocSize)))
1182 // Zero-length mem transfer intrinsics can be ignored entirely.
1183 return markAsDead(II);
1184
1185 if (!IsOffsetKnown)
1186 return PI.setAborted(&II);
1187
1188 insertUse(II, Offset,
1189 Length ? Length->getLimitedValue()
1190 : AllocSize - Offset.getLimitedValue(),
1191 (bool)Length);
1192 }
1193
1194 void visitMemTransferInst(MemTransferInst &II) {
1195 ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
1196 if (Length && Length->getValue() == 0)
1197 // Zero-length mem transfer intrinsics can be ignored entirely.
1198 return markAsDead(II);
1199
1200 // Because we can visit these intrinsics twice, also check to see if the
1201 // first time marked this instruction as dead. If so, skip it.
1202 if (VisitedDeadInsts.count(&II))
1203 return;
1204
1205 if (!IsOffsetKnown)
1206 return PI.setAborted(&II);
1207
1208 // This side of the transfer is completely out-of-bounds, and so we can
1209 // nuke the entire transfer. However, we also need to nuke the other side
1210 // if already added to our partitions.
1211 // FIXME: Yet another place we really should bypass this when
1212 // instrumenting for ASan.
1213 if (Offset.uge(AllocSize)) {
1214 auto MTPI = MemTransferSliceMap.find(&II);
1215 if (MTPI != MemTransferSliceMap.end())
1216 AS.Slices[MTPI->second].kill();
1217 return markAsDead(II);
1218 }
1219
1220 uint64_t RawOffset = Offset.getLimitedValue();
1221 uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset;
1222
1223 // Check for the special case where the same exact value is used for both
1224 // source and dest.
1225 if (*U == II.getRawDest() && *U == II.getRawSource()) {
1226 // For non-volatile transfers this is a no-op.
1227 if (!II.isVolatile())
1228 return markAsDead(II);
1229
1230 return insertUse(II, Offset, Size, /*IsSplittable=*/false);
1231 }
1232
1233 // If we have seen both source and destination for a mem transfer, then
1234 // they both point to the same alloca.
1235 bool Inserted;
1236 SmallDenseMap<Instruction *, unsigned>::iterator MTPI;
1237 std::tie(MTPI, Inserted) =
1238 MemTransferSliceMap.insert(std::make_pair(&II, AS.Slices.size()));
1239 unsigned PrevIdx = MTPI->second;
1240 if (!Inserted) {
1241 Slice &PrevP = AS.Slices[PrevIdx];
1242
1243 // Check if the begin offsets match and this is a non-volatile transfer.
1244 // In that case, we can completely elide the transfer.
1245 if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) {
1246 PrevP.kill();
1247 return markAsDead(II);
1248 }
1249
1250 // Otherwise we have an offset transfer within the same alloca. We can't
1251 // split those.
1252 PrevP.makeUnsplittable();
1253 }
1254
1255 // Insert the use now that we've fixed up the splittable nature.
1256 insertUse(II, Offset, Size, /*IsSplittable=*/Inserted && Length);
1257
1258 // Check that we ended up with a valid index in the map.
1259 assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&
1260 "Map index doesn't point back to a slice with this user.");
1261 }
1262
1263 // Disable SRoA for any intrinsics except for lifetime invariants.
1264 // FIXME: What about debug intrinsics? This matches old behavior, but
1265 // doesn't make sense.
1266 void visitIntrinsicInst(IntrinsicInst &II) {
1267 if (II.isDroppable()) {
1268 AS.DeadUseIfPromotable.push_back(U);
1269 return;
1270 }
1271
1272 if (!IsOffsetKnown)
1273 return PI.setAborted(&II);
1274
1275 if (II.isLifetimeStartOrEnd()) {
1276 insertUse(II, Offset, AllocSize, true);
1277 return;
1278 }
1279
1280 Base::visitIntrinsicInst(II);
1281 }
1282
1283 Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) {
1284 // We consider any PHI or select that results in a direct load or store of
1285 // the same offset to be a viable use for slicing purposes. These uses
1286 // are considered unsplittable and the size is the maximum loaded or stored
1287 // size.
1288 SmallPtrSet<Instruction *, 4> Visited;
1290 Visited.insert(Root);
1291 Uses.push_back(std::make_pair(cast<Instruction>(*U), Root));
1292 const DataLayout &DL = Root->getDataLayout();
1293 // If there are no loads or stores, the access is dead. We mark that as
1294 // a size zero access.
1295 Size = 0;
1296 do {
1297 Instruction *I, *UsedI;
1298 std::tie(UsedI, I) = Uses.pop_back_val();
1299
1300 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
1301 TypeSize LoadSize = DL.getTypeStoreSize(LI->getType());
1302 if (LoadSize.isScalable()) {
1303 PI.setAborted(LI);
1304 return nullptr;
1305 }
1306 Size = std::max(Size, LoadSize.getFixedValue());
1307 continue;
1308 }
1309 if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
1310 Value *Op = SI->getOperand(0);
1311 if (Op == UsedI)
1312 return SI;
1313 TypeSize StoreSize = DL.getTypeStoreSize(Op->getType());
1314 if (StoreSize.isScalable()) {
1315 PI.setAborted(SI);
1316 return nullptr;
1317 }
1318 Size = std::max(Size, StoreSize.getFixedValue());
1319 continue;
1320 }
1321
1322 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) {
1323 if (!GEP->hasAllZeroIndices())
1324 return GEP;
1325 } else if (!isa<BitCastInst>(I) && !isa<PHINode>(I) &&
1327 return I;
1328 }
1329
1330 for (User *U : I->users())
1331 if (Visited.insert(cast<Instruction>(U)).second)
1332 Uses.push_back(std::make_pair(I, cast<Instruction>(U)));
1333 } while (!Uses.empty());
1334
1335 return nullptr;
1336 }
1337
1338 void visitPHINodeOrSelectInst(Instruction &I) {
1340 if (I.use_empty())
1341 return markAsDead(I);
1342
1343 // If this is a PHI node before a catchswitch, we cannot insert any non-PHI
1344 // instructions in this BB, which may be required during rewriting. Bail out
1345 // on these cases.
1346 if (isa<PHINode>(I) && !I.getParent()->hasInsertionPt())
1347 return PI.setAborted(&I);
1348
1349 // TODO: We could use simplifyInstruction here to fold PHINodes and
1350 // SelectInsts. However, doing so requires to change the current
1351 // dead-operand-tracking mechanism. For instance, suppose neither loading
1352 // from %U nor %other traps. Then "load (select undef, %U, %other)" does not
1353 // trap either. However, if we simply replace %U with undef using the
1354 // current dead-operand-tracking mechanism, "load (select undef, undef,
1355 // %other)" may trap because the select may return the first operand
1356 // "undef".
1357 if (Value *Result = foldPHINodeOrSelectInst(I)) {
1358 if (Result == *U)
1359 // If the result of the constant fold will be the pointer, recurse
1360 // through the PHI/select as if we had RAUW'ed it.
1361 enqueueUsers(I);
1362 else
1363 // Otherwise the operand to the PHI/select is dead, and we can replace
1364 // it with poison.
1365 AS.DeadOperands.push_back(U);
1366
1367 return;
1368 }
1369
1370 if (!IsOffsetKnown)
1371 return PI.setAborted(&I);
1372
1373 // See if we already have computed info on this node.
1374 uint64_t &Size = PHIOrSelectSizes[&I];
1375 if (!Size) {
1376 // This is a new PHI/Select, check for an unsafe use of it.
1377 if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(&I, Size))
1378 return PI.setAborted(UnsafeI);
1379 }
1380
1381 // For PHI and select operands outside the alloca, we can't nuke the entire
1382 // phi or select -- the other side might still be relevant, so we special
1383 // case them here and use a separate structure to track the operands
1384 // themselves which should be replaced with poison.
1385 // FIXME: This should instead be escaped in the event we're instrumenting
1386 // for address sanitization.
1387 if (Offset.uge(AllocSize)) {
1388 AS.DeadOperands.push_back(U);
1389 return;
1390 }
1391
1392 insertUse(I, Offset, Size);
1393 }
1394
1395 void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(PN); }
1396
1397 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); }
1398
1399 /// Disable SROA entirely if there are unhandled users of the alloca.
1400 void visitInstruction(Instruction &I) { PI.setAborted(&I); }
1401
1402 void visitCallBase(CallBase &CB) {
1403 // If the call operand is read-only and only does a read-only or address
1404 // capture, then we mark it as EscapedReadOnly.
1405 if (CB.isDataOperand(U) &&
1406 !capturesFullProvenance(CB.getCaptureInfo(U->getOperandNo())) &&
1407 CB.onlyReadsMemory(U->getOperandNo())) {
1408 PI.setEscapedReadOnly(&CB);
1409 return;
1410 }
1411
1412 Base::visitCallBase(CB);
1413 }
1414};
1415
1416AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
1417 :
1418#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1419 AI(AI),
1420#endif
1421 PointerEscapingInstr(nullptr), PointerEscapingInstrReadOnly(nullptr) {
1422 SliceBuilder PB(DL, AI, *this);
1423 SliceBuilder::PtrInfo PtrI = PB.visitPtr(AI);
1424 if (PtrI.isEscaped() || PtrI.isAborted()) {
1425 // FIXME: We should sink the escape vs. abort info into the caller nicely,
1426 // possibly by just storing the PtrInfo in the AllocaSlices.
1427 PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst()
1428 : PtrI.getAbortingInst();
1429 assert(PointerEscapingInstr && "Did not track a bad instruction");
1430 return;
1431 }
1432 PointerEscapingInstrReadOnly = PtrI.getEscapedReadOnlyInst();
1433
1434 llvm::erase_if(Slices, [](const Slice &S) { return S.isDead(); });
1435
1436 // Sort the uses. This arranges for the offsets to be in ascending order,
1437 // and the sizes to be in descending order.
1438 llvm::stable_sort(Slices);
1439}
1440
1441#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1442
1443void AllocaSlices::print(raw_ostream &OS, const_iterator I,
1444 StringRef Indent) const {
1445 printSlice(OS, I, Indent);
1446 OS << "\n";
1447 printUse(OS, I, Indent);
1448}
1449
1450void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I,
1451 StringRef Indent) const {
1452 OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")"
1453 << " slice #" << (I - begin())
1454 << (I->isSplittable() ? " (splittable)" : "");
1455}
1456
1457void AllocaSlices::printUse(raw_ostream &OS, const_iterator I,
1458 StringRef Indent) const {
1459 OS << Indent << " used by: " << *I->getUse()->getUser() << "\n";
1460}
1461
1462void AllocaSlices::print(raw_ostream &OS) const {
1463 if (PointerEscapingInstr) {
1464 OS << "Can't analyze slices for alloca: " << AI << "\n"
1465 << " A pointer to this alloca escaped by:\n"
1466 << " " << *PointerEscapingInstr << "\n";
1467 return;
1468 }
1469
1470 if (PointerEscapingInstrReadOnly)
1471 OS << "Escapes into ReadOnly: " << *PointerEscapingInstrReadOnly << "\n";
1472
1473 OS << "Slices of alloca: " << AI << "\n";
1474 for (const_iterator I = begin(), E = end(); I != E; ++I)
1475 print(OS, I);
1476}
1477
1478LLVM_DUMP_METHOD void AllocaSlices::dump(const_iterator I) const {
1479 print(dbgs(), I);
1480}
1481LLVM_DUMP_METHOD void AllocaSlices::dump() const { print(dbgs()); }
1482
1483#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1484
1485/// Find a common load/store type used through a pointer PHI or select.
1486///
1487/// Look through a PHI or select to see if all of its users are loads or stores
1488/// of one common type. Whether those accesses can be speculated does not affect
1489/// the type they use and is checked separately when attempting promotion.
1491 assert((isa<PHINode, SelectInst>(I)) && "expected a PHI or select");
1492 Type *Ty = nullptr;
1493
1494 for (User *U : I.users()) {
1495 Type *UserTy = nullptr;
1496 if (auto *LI = dyn_cast<LoadInst>(U))
1497 UserTy = LI->getType();
1498 else if (auto *Store = dyn_cast<StoreInst>(U))
1499 // Slice building rejects stores of the PHI-or-select-derived pointer, so
1500 // it must be the store's pointer operand here.
1501 UserTy = Store->getValueOperand()->getType();
1502
1503 if (!UserTy || (Ty && Ty != UserTy))
1504 return nullptr;
1505 Ty = UserTy;
1506 }
1507
1508 return Ty;
1509}
1510
1511/// Walk the range of a partitioning looking for a common type to cover this
1512/// sequence of slices.
1513static std::pair<Type *, IntegerType *>
1514findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E,
1515 uint64_t EndOffset) {
1516 Type *Ty = nullptr;
1517 bool TyIsCommon = true;
1518 IntegerType *ITy = nullptr;
1519
1520 // Note that we need to look at *every* alloca slice's Use to ensure we
1521 // always get consistent results regardless of the order of slices.
1522 for (AllocaSlices::const_iterator I = B; I != E; ++I) {
1523 Use *U = I->getUse();
1524 if (isa<IntrinsicInst>(*U->getUser()))
1525 continue;
1526 if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset)
1527 continue;
1528
1529 Type *UserTy = nullptr;
1530 if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
1531 UserTy = LI->getType();
1532 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
1533 UserTy = SI->getValueOperand()->getType();
1534 } else if (isa<PHINode, SelectInst>(U->getUser())) {
1535 UserTy =
1537 }
1538
1539 if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(UserTy)) {
1540 // If the type is larger than the partition, skip it. We only encounter
1541 // this for split integer operations where we want to use the type of the
1542 // entity causing the split. Also skip if the type is not a byte width
1543 // multiple.
1544 if (UserITy->getBitWidth() % 8 != 0 ||
1545 UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset()))
1546 continue;
1547
1548 // Track the largest bitwidth integer type used in this way in case there
1549 // is no common type.
1550 if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth())
1551 ITy = UserITy;
1552 }
1553
1554 // To avoid depending on the order of slices, Ty and TyIsCommon must not
1555 // depend on types skipped above.
1556 if (!UserTy || (Ty && Ty != UserTy))
1557 TyIsCommon = false; // Give up on anything but an iN type.
1558 else
1559 Ty = UserTy;
1560 }
1561
1562 return {TyIsCommon ? Ty : nullptr, ITy};
1563}
1564
1565/// PHI instructions that use an alloca and are subsequently loaded can be
1566/// rewritten to load both input pointers in the pred blocks and then PHI the
1567/// results, allowing the load of the alloca to be promoted.
1568/// From this:
1569/// %P2 = phi [i32* %Alloca, i32* %Other]
1570/// %V = load i32* %P2
1571/// to:
1572/// %V1 = load i32* %Alloca -> will be mem2reg'd
1573/// ...
1574/// %V2 = load i32* %Other
1575/// ...
1576/// %V = phi [i32 %V1, i32 %V2]
1577///
1578/// We can do this to a PHI if its only uses are loads and if any loads moved
1579/// across other outgoing edges can be executed unconditionally.
1580///
1581/// FIXME: This should be hoisted into a generic utility, likely in
1582/// Transforms/Util/Local.h
1584 const DataLayout &DL = PN.getDataLayout();
1585
1586 // For now, we can only do this promotion if the load is in the same block
1587 // as the PHI, and if there are no stores between the phi and load.
1588 // TODO: Allow recursive phi users.
1589 BasicBlock *BB = PN.getParent();
1590 Align MaxAlign;
1591 uint64_t APWidth = DL.getIndexTypeSizeInBits(PN.getType());
1592 Type *LoadType = nullptr;
1593 for (User *U : PN.users()) {
1595 if (!LI || !LI->isSimple())
1596 return false;
1597
1598 // For now we only allow loads in the same block as the PHI. This is
1599 // a common case that happens when instcombine merges two loads through
1600 // a PHI.
1601 if (LI->getParent() != BB)
1602 return false;
1603
1604 if (LoadType) {
1605 if (LoadType != LI->getType())
1606 return false;
1607 } else {
1608 LoadType = LI->getType();
1609 }
1610
1611 // Ensure that there are no instructions between the PHI and the load that
1612 // could store.
1613 for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI)
1614 if (BBI->mayWriteToMemory())
1615 return false;
1616
1617 MaxAlign = std::max(MaxAlign, LI->getAlign());
1618 }
1619
1620 if (!LoadType)
1621 return false;
1622
1623 APInt LoadSize =
1624 APInt(APWidth, DL.getTypeStoreSize(LoadType).getFixedValue());
1625
1626 // We can only transform this if it is safe to push the loads into the
1627 // predecessor blocks. The only thing to watch out for is that we can't put
1628 // a possibly trapping load in the predecessor if it is a critical edge.
1629 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1631 Value *InVal = PN.getIncomingValue(Idx);
1632
1633 // If the value is produced by the terminator of the predecessor (an
1634 // invoke) or it has side-effects, there is no valid place to put a load
1635 // in the predecessor.
1636 if (TI == InVal || TI->mayHaveSideEffects())
1637 return false;
1638
1639 // If the predecessor has a single successor, then the edge isn't
1640 // critical.
1641 if (TI->getNumSuccessors() == 1)
1642 continue;
1643
1644 // If this pointer is always safe to load, or if we can prove that there
1645 // is already a load in the block, then we can move the load to the pred
1646 // block.
1647 if (isSafeToLoadUnconditionally(InVal, MaxAlign, LoadSize,
1648 SimplifyQuery(DL, TI)))
1649 continue;
1650
1651 return false;
1652 }
1653
1654 return true;
1655}
1656
1657/// Check whether a single store through PN can be moved onto each incoming
1658/// edge.
1659static StoreInst *getPHIStoreToRewrite(PHINode &PN, bool PreserveCFG,
1660 DominatorTree &DT) {
1661 // TODO: Support multiple stores and mixed load/store users.
1662 // TODO: Look through other instructions, such as other phis or addrspacecasts
1663 if (!PN.hasOneUse())
1664 return nullptr;
1665
1666 auto *SI = dyn_cast<StoreInst>(PN.user_back());
1667 if (!SI || SI->getPointerOperand() != &PN)
1668 return nullptr;
1669
1670 if (SI->isVolatile())
1671 return nullptr;
1672
1673 BasicBlock *BB = PN.getParent();
1674 // TODO: Allow a harmless prefix between the PHIs and the store.
1675 if (&*BB->getFirstNonPHIOrDbg() != SI)
1676 return nullptr;
1677
1678 Value *StoredValue = SI->getValueOperand();
1680 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1681 BasicBlock *Pred = PN.getIncomingBlock(Idx);
1682
1683 // Only one store is needed for duplicate edges from the same predecessor.
1684 if (!SeenPreds.insert(Pred).second)
1685 continue;
1686
1687 // TODO: Support other terminators.
1688 Instruction *TI = Pred->getTerminator();
1690 return nullptr;
1691
1692 if (Pred == BB)
1693 return nullptr;
1694
1695 // TODO: If StoredValue is another PHI in BB, use its corresponding
1696 // incoming value instead of requiring it to dominate every predecessor.
1697 if (!DT.dominates(StoredValue, TI))
1698 return nullptr;
1699
1700 if (TI->getNumSuccessors() == 1)
1701 continue;
1702
1703 // If the predecessor has more than one successor, then we will need to
1704 // split it so that we can insert the store only on the path to this BB.
1705 if (PreserveCFG || !BB->canSplitPredecessors())
1706 return nullptr;
1707 }
1708
1709 return SI;
1710}
1711
1712static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) {
1713 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
1714
1715 LoadInst *SomeLoad = cast<LoadInst>(PN.user_back());
1716 Type *LoadTy = SomeLoad->getType();
1717 IRB.SetInsertPoint(&PN);
1718 PHINode *NewPN = IRB.CreatePHI(LoadTy, PN.getNumIncomingValues(),
1719 PN.getName() + ".sroa.speculated");
1720
1721 // Get the AA tags and alignment to use from one of the loads. It does not
1722 // matter which one we get and if any differ.
1723 AAMDNodes AATags = SomeLoad->getAAMetadata();
1724 Align Alignment = SomeLoad->getAlign();
1725
1726 // Rewrite all loads of the PN to use the new PHI.
1727 while (!PN.use_empty()) {
1728 LoadInst *LI = cast<LoadInst>(PN.user_back());
1729 LI->replaceAllUsesWith(NewPN);
1730 LI->eraseFromParent();
1731 }
1732
1733 // Inject loads into all of the pred blocks.
1734 DenseMap<BasicBlock *, Value *> InjectedLoads;
1735 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1736 BasicBlock *Pred = PN.getIncomingBlock(Idx);
1737 Value *InVal = PN.getIncomingValue(Idx);
1738
1739 // A PHI node is allowed to have multiple (duplicated) entries for the same
1740 // basic block, as long as the value is the same. So if we already injected
1741 // a load in the predecessor, then we should reuse the same load for all
1742 // duplicated entries.
1743 if (Value *V = InjectedLoads.lookup(Pred)) {
1744 NewPN->addIncoming(V, Pred);
1745 continue;
1746 }
1747
1748 Instruction *TI = Pred->getTerminator();
1749 IRB.SetInsertPoint(TI);
1750
1751 LoadInst *Load = IRB.CreateAlignedLoad(
1752 LoadTy, InVal, Alignment,
1753 (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
1754 ++NumLoadsSpeculated;
1755 if (AATags)
1756 Load->setAAMetadata(AATags);
1757 NewPN->addIncoming(Load, Pred);
1758 InjectedLoads[Pred] = Load;
1759 }
1760
1761 LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n");
1762 PN.eraseFromParent();
1763}
1764
1765/// Move a store through a pointer PHI onto each of the PHI's incoming edges.
1766/// Returns whether this required modifying the CFG.
1769 LLVM_DEBUG(dbgs() << " original: " << PN << "\n"
1770 << " " << SI << "\n");
1771
1772 // Splitting one edge rewrites all PHIs in the destination block. Snapshot
1773 // the original predecessor/value pairs before making any CFG changes.
1776 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1777 BasicBlock *Pred = PN.getIncomingBlock(Idx);
1778 if (!SeenPreds.insert(Pred).second)
1779 continue;
1780 Value *InVal = PN.getIncomingValue(Idx);
1781 IncomingValues.emplace_back(Pred, InVal);
1782
1783 // Revisit every alloca exposed by removing the pointer PHI.
1784 if (auto *AI = dyn_cast<AllocaInst>(getUnderlyingObject(InVal)))
1785 Worklist.insert(AI);
1786 }
1787
1788 bool CFGChanged = false;
1789 BasicBlock *BB = PN.getParent();
1790 for (auto [Pred, InVal] : IncomingValues) {
1791 BasicBlock *StoreBB = Pred;
1792 if (Pred->getTerminator()->getNumSuccessors() != 1) {
1793 StoreBB = SplitBlockPredecessors(BB, {Pred}, ".sroa.store", &DTU);
1794 assert(StoreBB && "store edge was not checked for splitting");
1795 CFGChanged = true;
1796 }
1797
1798 auto *NewStore = cast<StoreInst>(SI.clone());
1799 NewStore->setOperand(StoreInst::getPointerOperandIndex(), InVal);
1800 NewStore->insertBefore(StoreBB->getTerminator()->getIterator());
1801 ++NumStoresPredicated;
1802 LLVM_DEBUG(dbgs() << " to: " << *NewStore << "\n");
1803 }
1804
1805 SI.eraseFromParent();
1806 PN.eraseFromParent();
1807 return CFGChanged;
1808}
1809
1810SelectHandSpeculativity &
1811SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) {
1812 if (isTrueVal)
1814 else
1816 return *this;
1817}
1818
1819bool SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const {
1820 return isTrueVal ? Bitfield::get<SelectHandSpeculativity::TrueVal>(Storage)
1821 : Bitfield::get<SelectHandSpeculativity::FalseVal>(Storage);
1822}
1823
1824bool SelectHandSpeculativity::areAllSpeculatable() const {
1825 return isSpeculatable(/*isTrueVal=*/true) &&
1826 isSpeculatable(/*isTrueVal=*/false);
1827}
1828
1829bool SelectHandSpeculativity::areAnySpeculatable() const {
1830 return isSpeculatable(/*isTrueVal=*/true) ||
1831 isSpeculatable(/*isTrueVal=*/false);
1832}
1833bool SelectHandSpeculativity::areNoneSpeculatable() const {
1834 return !areAnySpeculatable();
1835}
1836
1837static SelectHandSpeculativity
1839 assert(LI.isSimple() && "Only for simple loads");
1840 SelectHandSpeculativity Spec;
1841
1842 const DataLayout &DL = SI.getDataLayout();
1843 for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()})
1845 SimplifyQuery(DL, &LI)))
1846 Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue());
1847 else if (PreserveCFG)
1848 return Spec;
1849
1850 return Spec;
1851}
1852
1853std::optional<RewriteableMemOps>
1854SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) {
1855 RewriteableMemOps Ops;
1856
1857 for (User *U : SI.users()) {
1858 if (auto *Store = dyn_cast<StoreInst>(U)) {
1859 // Note that atomic stores can be transformed; atomic semantics do not
1860 // have any meaning for a local alloca. Stores are not speculatable,
1861 // however, so if we can't turn it into a predicated store, we are done.
1862 if (Store->isVolatile() || PreserveCFG)
1863 return {}; // Give up on this `select`.
1864 Ops.emplace_back(Store);
1865 continue;
1866 }
1867
1868 auto *LI = dyn_cast<LoadInst>(U);
1869
1870 // Note that atomic loads can be transformed;
1871 // atomic semantics do not have any meaning for a local alloca.
1872 if (!LI || LI->isVolatile())
1873 return {}; // Give up on this `select`.
1874
1875 PossiblySpeculatableLoad Load(LI);
1876 if (!LI->isSimple()) {
1877 // If the `load` is not simple, we can't speculatively execute it,
1878 // but we could handle this via a CFG modification. But can we?
1879 if (PreserveCFG)
1880 return {}; // Give up on this `select`.
1881 Ops.emplace_back(Load);
1882 continue;
1883 }
1884
1885 SelectHandSpeculativity Spec =
1886 isSafeLoadOfSelectToSpeculate(*LI, SI, PreserveCFG);
1887 if (PreserveCFG && !Spec.areAllSpeculatable())
1888 return {}; // Give up on this `select`.
1889
1890 Load.setInt(Spec);
1891 Ops.emplace_back(Load);
1892 }
1893
1894 return Ops;
1895}
1896
1898 IRBuilderTy &IRB) {
1899 LLVM_DEBUG(dbgs() << " original load: " << SI << "\n");
1900
1901 Value *TV = SI.getTrueValue();
1902 Value *FV = SI.getFalseValue();
1903 // Replace the given load of the select with a select of two loads.
1904
1905 assert(LI.isSimple() && "We only speculate simple loads");
1906
1907 IRB.SetInsertPoint(&LI);
1908
1909 LoadInst *TL =
1910 IRB.CreateAlignedLoad(LI.getType(), TV, LI.getAlign(),
1911 LI.getName() + ".sroa.speculate.load.true");
1912 LoadInst *FL =
1913 IRB.CreateAlignedLoad(LI.getType(), FV, LI.getAlign(),
1914 LI.getName() + ".sroa.speculate.load.false");
1915 NumLoadsSpeculated += 2;
1916
1917 // Transfer alignment and AA info if present.
1918 TL->setAlignment(LI.getAlign());
1919 FL->setAlignment(LI.getAlign());
1920
1921 AAMDNodes Tags = LI.getAAMetadata();
1922 if (Tags) {
1923 TL->setAAMetadata(Tags);
1924 FL->setAAMetadata(Tags);
1925 }
1926
1927 Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL,
1928 LI.getName() + ".sroa.speculated", &SI);
1929
1930 LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n");
1931 LI.replaceAllUsesWith(V);
1932}
1933
1934template <typename T>
1936 SelectHandSpeculativity Spec,
1937 DomTreeUpdater &DTU) {
1938 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Only for load and store!");
1939 LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n");
1940 BasicBlock *Head = I.getParent();
1941 Instruction *ThenTerm = nullptr;
1942 Instruction *ElseTerm = nullptr;
1943 if (Spec.areNoneSpeculatable())
1944 SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm,
1945 SI.getMetadata(LLVMContext::MD_prof), &DTU);
1946 else {
1947 SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false,
1948 SI.getMetadata(LLVMContext::MD_prof), &DTU,
1949 /*LI=*/nullptr, /*ThenBlock=*/nullptr);
1950 if (Spec.isSpeculatable(/*isTrueVal=*/true))
1951 cast<CondBrInst>(Head->getTerminator())->swapSuccessors();
1952 }
1953 auto *HeadBI = cast<CondBrInst>(Head->getTerminator());
1954 Spec = {}; // Do not use `Spec` beyond this point.
1955 BasicBlock *Tail = I.getParent();
1956 Tail->setName(Head->getName() + ".cont");
1957 PHINode *PN;
1958 if (isa<LoadInst>(I))
1959 PN = PHINode::Create(I.getType(), 2, "", I.getIterator());
1960 for (BasicBlock *SuccBB : successors(Head)) {
1961 bool IsThen = SuccBB == HeadBI->getSuccessor(0);
1962 int SuccIdx = IsThen ? 0 : 1;
1963 auto *NewMemOpBB = SuccBB == Tail ? Head : SuccBB;
1964 auto &CondMemOp = cast<T>(*I.clone());
1965 if (NewMemOpBB != Head) {
1966 NewMemOpBB->setName(Head->getName() + (IsThen ? ".then" : ".else"));
1967 if (isa<LoadInst>(I))
1968 ++NumLoadsPredicated;
1969 else
1970 ++NumStoresPredicated;
1971 } else {
1972 CondMemOp.dropUBImplyingAttrsAndMetadata();
1973 ++NumLoadsSpeculated;
1974 }
1975 CondMemOp.insertBefore(NewMemOpBB->getTerminator()->getIterator());
1976 Value *Ptr = SI.getOperand(1 + SuccIdx);
1977 CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr);
1978 if (isa<LoadInst>(I)) {
1979 CondMemOp.setName(I.getName() + (IsThen ? ".then" : ".else") + ".val");
1980 PN->addIncoming(&CondMemOp, NewMemOpBB);
1981 } else
1982 LLVM_DEBUG(dbgs() << " to: " << CondMemOp << "\n");
1983 }
1984 if (isa<LoadInst>(I)) {
1985 PN->takeName(&I);
1986 LLVM_DEBUG(dbgs() << " to: " << *PN << "\n");
1987 I.replaceAllUsesWith(PN);
1988 }
1989}
1990
1992 SelectHandSpeculativity Spec,
1993 DomTreeUpdater &DTU) {
1994 if (auto *LI = dyn_cast<LoadInst>(&I))
1995 rewriteMemOpOfSelect(SelInst, *LI, Spec, DTU);
1996 else if (auto *SI = dyn_cast<StoreInst>(&I))
1997 rewriteMemOpOfSelect(SelInst, *SI, Spec, DTU);
1998 else
1999 llvm_unreachable_internal("Only for load and store.");
2000}
2001
2003 const RewriteableMemOps &Ops,
2004 IRBuilderTy &IRB, DomTreeUpdater *DTU) {
2005 bool CFGChanged = false;
2006 LLVM_DEBUG(dbgs() << " original select: " << SI << "\n");
2007
2008 for (const RewriteableMemOp &Op : Ops) {
2009 SelectHandSpeculativity Spec;
2010 Instruction *I;
2011 if (auto *const *US = std::get_if<UnspeculatableStore>(&Op)) {
2012 I = *US;
2013 } else {
2014 auto PSL = std::get<PossiblySpeculatableLoad>(Op);
2015 I = PSL.getPointer();
2016 Spec = PSL.getInt();
2017 }
2018 if (Spec.areAllSpeculatable()) {
2020 } else {
2021 assert(DTU && "Should not get here when not allowed to modify the CFG!");
2022 rewriteMemOpOfSelect(SI, *I, Spec, *DTU);
2023 CFGChanged = true;
2024 }
2025 I->eraseFromParent();
2026 }
2027
2028 for (User *U : make_early_inc_range(SI.users()))
2029 cast<BitCastInst>(U)->eraseFromParent();
2030 SI.eraseFromParent();
2031 return CFGChanged;
2032}
2033
2034/// Compute an adjusted pointer from Ptr by Offset bytes where the
2035/// resulting pointer has PointerTy.
2036static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr,
2038 const Twine &NamePrefix) {
2039 if (Offset != 0)
2040 Ptr = IRB.CreateInBoundsPtrAdd(Ptr, IRB.getInt(Offset),
2041 NamePrefix + "sroa_idx");
2042 return IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, PointerTy,
2043 NamePrefix + "sroa_cast");
2044}
2045
2046/// Compute the adjusted alignment for a load or store from an offset.
2050
2051/// Test whether we can convert a value from the old to the new type.
2052///
2053/// This predicate should be used to guard calls to convertValue in order to
2054/// ensure that we only try to convert viable values. The strategy is that we
2055/// will peel off single element struct and array wrappings to get to an
2056/// underlying value, and convert that value.
2057static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy,
2058 unsigned VScale = 0) {
2059 if (OldTy == NewTy)
2060 return true;
2061
2062 // For integer types, we can't handle any bit-width differences. This would
2063 // break both vector conversions with extension and introduce endianness
2064 // issues when in conjunction with loads and stores.
2065 if (isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) {
2067 cast<IntegerType>(NewTy)->getBitWidth() &&
2068 "We can't have the same bitwidth for different int types");
2069 return false;
2070 }
2071
2072 TypeSize NewSize = DL.getTypeSizeInBits(NewTy);
2073 TypeSize OldSize = DL.getTypeSizeInBits(OldTy);
2074
2075 if ((isa<ScalableVectorType>(NewTy) && isa<FixedVectorType>(OldTy)) ||
2076 (isa<ScalableVectorType>(OldTy) && isa<FixedVectorType>(NewTy))) {
2077 // Conversion is only possible when the size of scalable vectors is known.
2078 if (!VScale)
2079 return false;
2080
2081 // For ptr-to-int and int-to-ptr casts, the pointer side is resolved within
2082 // a single domain (either fixed or scalable). Any additional conversion
2083 // between fixed and scalable types is handled through integer types.
2084 auto OldVTy = OldTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(OldTy) : OldTy;
2085 auto NewVTy = NewTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(NewTy) : NewTy;
2086
2087 if (isa<ScalableVectorType>(NewTy)) {
2089 return false;
2090
2091 NewSize = TypeSize::getFixed(NewSize.getKnownMinValue() * VScale);
2092 } else {
2094 return false;
2095
2096 OldSize = TypeSize::getFixed(OldSize.getKnownMinValue() * VScale);
2097 }
2098 }
2099
2100 if (NewSize != OldSize)
2101 return false;
2102 if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
2103 return false;
2104
2105 // We can convert pointers to integers and vice-versa. Same for vectors
2106 // of pointers and integers.
2107 OldTy = OldTy->getScalarType();
2108 NewTy = NewTy->getScalarType();
2109 if (NewTy->isPointerTy() || OldTy->isPointerTy()) {
2110 if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
2111 unsigned OldAS = OldTy->getPointerAddressSpace();
2112 unsigned NewAS = NewTy->getPointerAddressSpace();
2113 // Convert pointers if they are pointers from the same address space or
2114 // different integral (not non-integral) address spaces with the same
2115 // pointer size.
2116 return OldAS == NewAS ||
2117 (!DL.isNonIntegralAddressSpace(OldAS) &&
2118 !DL.isNonIntegralAddressSpace(NewAS) &&
2119 DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS));
2120 }
2121
2122 // We can convert integers to integral pointers, but not to non-integral
2123 // pointers.
2124 if (OldTy->isIntegerTy())
2125 return !DL.isNonIntegralPointerType(NewTy);
2126
2127 // We can convert integral pointers to integers, but non-integral pointers
2128 // need to remain pointers.
2129 if (!DL.isNonIntegralPointerType(OldTy))
2130 return NewTy->isIntegerTy();
2131
2132 return false;
2133 }
2134
2135 if (OldTy->isTargetExtTy() || NewTy->isTargetExtTy())
2136 return false;
2137
2138 return true;
2139}
2140
2141/// Test whether the given slice use can be promoted to a vector.
2142///
2143/// This function is called to test each entry in a partition which is slated
2144/// for a single slice.
2145static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
2146 VectorType *Ty,
2147 uint64_t ElementSize,
2148 const DataLayout &DL,
2149 unsigned VScale) {
2150 // First validate the slice offsets.
2151 uint64_t BeginOffset =
2152 std::max(S.beginOffset(), P.beginOffset()) - P.beginOffset();
2153 uint64_t BeginIndex = BeginOffset / ElementSize;
2154 if (BeginIndex * ElementSize != BeginOffset ||
2155 BeginIndex >= cast<FixedVectorType>(Ty)->getNumElements())
2156 return false;
2157 uint64_t EndOffset = std::min(S.endOffset(), P.endOffset()) - P.beginOffset();
2158 uint64_t EndIndex = EndOffset / ElementSize;
2159 if (EndIndex * ElementSize != EndOffset ||
2160 EndIndex > cast<FixedVectorType>(Ty)->getNumElements())
2161 return false;
2162
2163 assert(EndIndex > BeginIndex && "Empty vector!");
2164 uint64_t NumElements = EndIndex - BeginIndex;
2165 Type *SliceTy = (NumElements == 1)
2166 ? Ty->getElementType()
2167 : FixedVectorType::get(Ty->getElementType(), NumElements);
2168
2169 Type *SplitIntTy =
2170 Type::getIntNTy(Ty->getContext(), NumElements * ElementSize * 8);
2171
2172 Use *U = S.getUse();
2173
2174 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) {
2175 if (MI->isVolatile())
2176 return false;
2177 if (!S.isSplittable())
2178 return false; // Skip any unsplittable intrinsics.
2179 if (isa<MemSetInst>(MI)) {
2180 Type *SplatTy = Type::getIntNTy(Ty->getContext(), ElementSize * 8);
2181 if (!canConvertValue(DL, SplatTy, Ty->getElementType(), VScale))
2182 return false;
2183 }
2184 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
2185 if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
2186 return false;
2187 } else if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
2188 if (LI->isVolatile())
2189 return false;
2190 Type *LTy = LI->getType();
2191 // Disable vector promotion when there are loads or stores of an FCA.
2192 if (LTy->isStructTy())
2193 return false;
2194 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2195 assert(LTy->isIntegerTy());
2196 LTy = SplitIntTy;
2197 }
2198 if (!canConvertValue(DL, SliceTy, LTy, VScale))
2199 return false;
2200 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
2201 if (SI->isVolatile())
2202 return false;
2203 Type *STy = SI->getValueOperand()->getType();
2204 // Disable vector promotion when there are loads or stores of an FCA.
2205 if (STy->isStructTy())
2206 return false;
2207 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2208 assert(STy->isIntegerTy());
2209 STy = SplitIntTy;
2210 }
2211 if (!canConvertValue(DL, STy, SliceTy, VScale))
2212 return false;
2213 } else {
2214 return false;
2215 }
2216
2217 return true;
2218}
2219
2220/// Test whether any vector type in \p CandidateTys is viable for promotion.
2221///
2222/// This implements the necessary checking for \c isVectorPromotionViable over
2223/// all slices of the alloca for the given VectorType.
2224static VectorType *
2226 SmallVectorImpl<VectorType *> &CandidateTys,
2227 bool HaveCommonEltTy, Type *CommonEltTy,
2228 bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
2229 VectorType *CommonVecPtrTy, unsigned VScale) {
2230 // If we didn't find a vector type, nothing to do here.
2231 if (CandidateTys.empty())
2232 return nullptr;
2233
2234 // Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
2235 // then we should choose it, not some other alternative.
2236 // But, we can't perform a no-op pointer address space change via bitcast,
2237 // so if we didn't have a common pointer element type, bail.
2238 if (HaveVecPtrTy && !HaveCommonVecPtrTy)
2239 return nullptr;
2240
2241 // Try to pick the "best" element type out of the choices.
2242 if (!HaveCommonEltTy && HaveVecPtrTy) {
2243 // If there was a pointer element type, there's really only one choice.
2244 CandidateTys.clear();
2245 CandidateTys.push_back(CommonVecPtrTy);
2246 } else if (!HaveCommonEltTy && !HaveVecPtrTy) {
2247 // Integer-ify vector types.
2248 for (VectorType *&VTy : CandidateTys) {
2249 if (!VTy->getElementType()->isIntegerTy())
2250 VTy = cast<VectorType>(VTy->getWithNewType(IntegerType::getIntNTy(
2251 VTy->getContext(), VTy->getScalarSizeInBits())));
2252 }
2253
2254 // Rank the remaining candidate vector types. This is easy because we know
2255 // they're all integer vectors. We sort by ascending number of elements.
2256 auto RankVectorTypesComp = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2257 (void)DL;
2258 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2259 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2260 "Cannot have vector types of different sizes!");
2261 assert(RHSTy->getElementType()->isIntegerTy() &&
2262 "All non-integer types eliminated!");
2263 assert(LHSTy->getElementType()->isIntegerTy() &&
2264 "All non-integer types eliminated!");
2265 return cast<FixedVectorType>(RHSTy)->getNumElements() <
2266 cast<FixedVectorType>(LHSTy)->getNumElements();
2267 };
2268 auto RankVectorTypesEq = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2269 (void)DL;
2270 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2271 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2272 "Cannot have vector types of different sizes!");
2273 assert(RHSTy->getElementType()->isIntegerTy() &&
2274 "All non-integer types eliminated!");
2275 assert(LHSTy->getElementType()->isIntegerTy() &&
2276 "All non-integer types eliminated!");
2277 return cast<FixedVectorType>(RHSTy)->getNumElements() ==
2278 cast<FixedVectorType>(LHSTy)->getNumElements();
2279 };
2280 llvm::sort(CandidateTys, RankVectorTypesComp);
2281 CandidateTys.erase(llvm::unique(CandidateTys, RankVectorTypesEq),
2282 CandidateTys.end());
2283 } else {
2284// The only way to have the same element type in every vector type is to
2285// have the same vector type. Check that and remove all but one.
2286#ifndef NDEBUG
2287 for (VectorType *VTy : CandidateTys) {
2288 assert(VTy->getElementType() == CommonEltTy &&
2289 "Unaccounted for element type!");
2290 assert(VTy == CandidateTys[0] &&
2291 "Different vector types with the same element type!");
2292 }
2293#endif
2294 CandidateTys.resize(1);
2295 }
2296
2297 // FIXME: hack. Do we have a named constant for this?
2298 // SDAG SDNode can't have more than 65535 operands.
2299 llvm::erase_if(CandidateTys, [](VectorType *VTy) {
2300 return cast<FixedVectorType>(VTy)->getNumElements() >
2301 std::numeric_limits<unsigned short>::max();
2302 });
2303
2304 // Find a vector type viable for promotion by iterating over all slices.
2305 auto *VTy = llvm::find_if(CandidateTys, [&](VectorType *VTy) -> bool {
2306 uint64_t ElementSize =
2307 DL.getTypeSizeInBits(VTy->getElementType()).getFixedValue();
2308
2309 // While the definition of LLVM vectors is bitpacked, we don't support sizes
2310 // that aren't byte sized.
2311 if (ElementSize % 8)
2312 return false;
2313 assert((DL.getTypeSizeInBits(VTy).getFixedValue() % 8) == 0 &&
2314 "vector size not a multiple of element size?");
2315 ElementSize /= 8;
2316
2317 for (const Slice &S : P)
2318 if (!isVectorPromotionViableForSlice(P, S, VTy, ElementSize, DL, VScale))
2319 return false;
2320
2321 for (const Slice *S : P.splitSliceTails())
2322 if (!isVectorPromotionViableForSlice(P, *S, VTy, ElementSize, DL, VScale))
2323 return false;
2324
2325 return true;
2326 });
2327 return VTy != CandidateTys.end() ? *VTy : nullptr;
2328}
2329
2331 SetVector<Type *> &OtherTys, ArrayRef<VectorType *> CandidateTysCopy,
2332 function_ref<void(Type *)> CheckCandidateType, Partition &P,
2333 const DataLayout &DL, SmallVectorImpl<VectorType *> &CandidateTys,
2334 bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy,
2335 bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy, unsigned VScale) {
2336 [[maybe_unused]] VectorType *OriginalElt =
2337 CandidateTysCopy.size() ? CandidateTysCopy[0] : nullptr;
2338 // Consider additional vector types where the element type size is a
2339 // multiple of load/store element size.
2340 for (Type *Ty : OtherTys) {
2342 continue;
2343 unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
2344 // Make a copy of CandidateTys and iterate through it, because we
2345 // might append to CandidateTys in the loop.
2346 for (VectorType *const VTy : CandidateTysCopy) {
2347 // The elements in the copy should remain invariant throughout the loop
2348 assert(CandidateTysCopy[0] == OriginalElt && "Different Element");
2349 unsigned VectorSize = DL.getTypeSizeInBits(VTy).getFixedValue();
2350 unsigned ElementSize =
2351 DL.getTypeSizeInBits(VTy->getElementType()).getFixedValue();
2352 if (TypeSize != VectorSize && TypeSize != ElementSize &&
2353 VectorSize % TypeSize == 0) {
2354 VectorType *NewVTy = VectorType::get(Ty, VectorSize / TypeSize, false);
2355 CheckCandidateType(NewVTy);
2356 }
2357 }
2358 }
2359
2361 P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2362 HaveCommonVecPtrTy, CommonVecPtrTy, VScale);
2363}
2364
2365/// Test whether the given alloca partitioning and range of slices can be
2366/// promoted to a vector.
2367///
2368/// This is a quick test to check whether we can rewrite a particular alloca
2369/// partition (and its newly formed alloca) into a vector alloca with only
2370/// whole-vector loads and stores such that it could be promoted to a vector
2371/// SSA value. We only can ensure this for a limited set of operations, and we
2372/// don't want to do the rewrites unless we are confident that the result will
2373/// be promotable, so we have an early test here.
2375 unsigned VScale) {
2376 // Collect the candidate types for vector-based promotion. Also track whether
2377 // we have different element types.
2378 SmallVector<VectorType *, 4> CandidateTys;
2379 SetVector<Type *> LoadStoreTys;
2380 SetVector<Type *> DeferredTys;
2381 Type *CommonEltTy = nullptr;
2382 VectorType *CommonVecPtrTy = nullptr;
2383 bool HaveVecPtrTy = false;
2384 bool HaveCommonEltTy = true;
2385 bool HaveCommonVecPtrTy = true;
2386 auto CheckCandidateType = [&](Type *Ty) {
2387 if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
2388 // Return if bitcast to vectors is different for total size in bits.
2389 if (!CandidateTys.empty()) {
2390 VectorType *V = CandidateTys[0];
2391 if (DL.getTypeSizeInBits(VTy).getFixedValue() !=
2392 DL.getTypeSizeInBits(V).getFixedValue()) {
2393 CandidateTys.clear();
2394 return;
2395 }
2396 }
2397 CandidateTys.push_back(VTy);
2398 Type *EltTy = VTy->getElementType();
2399
2400 if (!CommonEltTy)
2401 CommonEltTy = EltTy;
2402 else if (CommonEltTy != EltTy)
2403 HaveCommonEltTy = false;
2404
2405 if (EltTy->isPointerTy()) {
2406 HaveVecPtrTy = true;
2407 if (!CommonVecPtrTy)
2408 CommonVecPtrTy = VTy;
2409 else if (CommonVecPtrTy != VTy)
2410 HaveCommonVecPtrTy = false;
2411 }
2412 }
2413 };
2414
2415 // Put load and store types into a set for de-duplication.
2416 for (const Slice &S : P) {
2417 Type *Ty;
2418 if (auto *LI = dyn_cast<LoadInst>(S.getUse()->getUser()))
2419 Ty = LI->getType();
2420 else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser()))
2421 Ty = SI->getValueOperand()->getType();
2422 else
2423 continue;
2424
2425 auto CandTy = Ty->getScalarType();
2426 if (CandTy->isPointerTy() && (S.beginOffset() != P.beginOffset() ||
2427 S.endOffset() != P.endOffset())) {
2428 DeferredTys.insert(Ty);
2429 continue;
2430 }
2431
2432 LoadStoreTys.insert(Ty);
2433 // Consider any loads or stores that are the exact size of the slice.
2434 if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset())
2435 CheckCandidateType(Ty);
2436 }
2437
2438 SmallVector<VectorType *, 4> CandidateTysCopy = CandidateTys;
2440 LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL,
2441 CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2442 HaveCommonVecPtrTy, CommonVecPtrTy, VScale))
2443 return VTy;
2444
2445 CandidateTys.clear();
2447 DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys,
2448 HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy,
2449 CommonVecPtrTy, VScale);
2450}
2451
2452/// Test whether a slice of an alloca is valid for integer widening.
2453///
2454/// This implements the necessary checking for the \c isIntegerWideningViable
2455/// test below on a single slice of the alloca.
2456static bool isIntegerWideningViableForSlice(const Slice &S,
2457 uint64_t AllocBeginOffset,
2458 Type *AllocaTy,
2459 const DataLayout &DL,
2460 bool &WholeAllocaOp) {
2461 uint64_t Size = DL.getTypeStoreSize(AllocaTy).getFixedValue();
2462
2463 uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
2464 uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
2465
2466 Use *U = S.getUse();
2467
2468 // Lifetime intrinsics operate over the whole alloca whose sizes are usually
2469 // larger than other load/store slices (RelEnd > Size). But lifetime are
2470 // always promotable and should not impact other slices' promotability of the
2471 // partition.
2472 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
2473 if (II->isLifetimeStartOrEnd() || II->isDroppable())
2474 return true;
2475 }
2476
2477 // We can't reasonably handle cases where the load or store extends past
2478 // the end of the alloca's type and into its padding.
2479 if (RelEnd > Size)
2480 return false;
2481
2482 if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
2483 if (LI->isVolatile())
2484 return false;
2485 // We can't handle loads that extend past the allocated memory.
2486 TypeSize LoadSize = DL.getTypeStoreSize(LI->getType());
2487 if (!LoadSize.isFixed() || LoadSize.getFixedValue() > Size)
2488 return false;
2489 // So far, AllocaSliceRewriter does not support widening split slice tails
2490 // in rewriteIntegerLoad.
2491 if (S.beginOffset() < AllocBeginOffset)
2492 return false;
2493 // Note that we don't count vector loads or stores as whole-alloca
2494 // operations which enable integer widening because we would prefer to use
2495 // vector widening instead.
2496 if (!isa<VectorType>(LI->getType()) && RelBegin == 0 && RelEnd == Size)
2497 WholeAllocaOp = true;
2498 if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) {
2499 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedValue())
2500 return false;
2501 } else if (RelBegin != 0 || RelEnd != Size ||
2502 !canConvertValue(DL, AllocaTy, LI->getType())) {
2503 // Non-integer loads need to be convertible from the alloca type so that
2504 // they are promotable.
2505 return false;
2506 }
2507 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
2508 Type *ValueTy = SI->getValueOperand()->getType();
2509 if (SI->isVolatile())
2510 return false;
2511 // We can't handle stores that extend past the allocated memory.
2512 TypeSize StoreSize = DL.getTypeStoreSize(ValueTy);
2513 if (!StoreSize.isFixed() || StoreSize.getFixedValue() > Size)
2514 return false;
2515 // So far, AllocaSliceRewriter does not support widening split slice tails
2516 // in rewriteIntegerStore.
2517 if (S.beginOffset() < AllocBeginOffset)
2518 return false;
2519 // Note that we don't count vector loads or stores as whole-alloca
2520 // operations which enable integer widening because we would prefer to use
2521 // vector widening instead.
2522 if (!isa<VectorType>(ValueTy) && RelBegin == 0 && RelEnd == Size)
2523 WholeAllocaOp = true;
2524 if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) {
2525 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedValue())
2526 return false;
2527 } else if (RelBegin != 0 || RelEnd != Size ||
2528 !canConvertValue(DL, ValueTy, AllocaTy)) {
2529 // Non-integer stores need to be convertible to the alloca type so that
2530 // they are promotable.
2531 return false;
2532 }
2533 } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) {
2534 if (MI->isVolatile() || !isa<Constant>(MI->getLength()))
2535 return false;
2536 if (!S.isSplittable())
2537 return false; // Skip any unsplittable intrinsics.
2538 } else {
2539 return false;
2540 }
2541
2542 return true;
2543}
2544
2545/// Test whether the given alloca partition's integer operations can be
2546/// widened to promotable ones.
2547///
2548/// This is a quick test to check whether we can rewrite the integer loads and
2549/// stores to a particular alloca into wider loads and stores and be able to
2550/// promote the resulting alloca.
2551static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
2552 const DataLayout &DL) {
2553 uint64_t SizeInBits = DL.getTypeSizeInBits(AllocaTy).getFixedValue();
2554 // Don't create integer types larger than the maximum bitwidth.
2555 if (SizeInBits > IntegerType::MAX_INT_BITS)
2556 return false;
2557
2558 // Don't try to handle allocas with bit-padding.
2559 if (SizeInBits != DL.getTypeStoreSizeInBits(AllocaTy).getFixedValue())
2560 return false;
2561
2562 // We need to ensure that an integer type with the appropriate bitwidth can
2563 // be converted to the alloca type, whatever that is. We don't want to force
2564 // the alloca itself to have an integer type if there is a more suitable one.
2565 Type *IntTy = Type::getIntNTy(AllocaTy->getContext(), SizeInBits);
2566 if (!canConvertValue(DL, AllocaTy, IntTy) ||
2567 !canConvertValue(DL, IntTy, AllocaTy))
2568 return false;
2569
2570 // While examining uses, we ensure that the alloca has a covering load or
2571 // store. We don't want to widen the integer operations only to fail to
2572 // promote due to some other unsplittable entry (which we may make splittable
2573 // later). However, if there are only splittable uses, go ahead and assume
2574 // that we cover the alloca.
2575 // FIXME: We shouldn't consider split slices that happen to start in the
2576 // partition here...
2577 bool WholeAllocaOp = P.empty() && DL.isLegalInteger(SizeInBits);
2578
2579 for (const Slice &S : P)
2580 if (!isIntegerWideningViableForSlice(S, P.beginOffset(), AllocaTy, DL,
2581 WholeAllocaOp))
2582 return false;
2583
2584 for (const Slice *S : P.splitSliceTails())
2585 if (!isIntegerWideningViableForSlice(*S, P.beginOffset(), AllocaTy, DL,
2586 WholeAllocaOp))
2587 return false;
2588
2589 return WholeAllocaOp;
2590}
2591
2592static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
2594 const Twine &Name) {
2595 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2596 IntegerType *IntTy = cast<IntegerType>(V->getType());
2597 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2598 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2599 "Element extends past full value");
2600 uint64_t ShAmt = 8 * Offset;
2601 if (DL.isBigEndian())
2602 ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedValue() -
2603 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2604 if (ShAmt) {
2605 V = IRB.CreateLShr(V, ShAmt, Name + ".shift");
2606 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2607 }
2608 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2609 "Cannot extract to a larger integer!");
2610 if (Ty != IntTy) {
2611 V = IRB.CreateTrunc(V, Ty, Name + ".trunc");
2612 LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n");
2613 }
2614 return V;
2615}
2616
2617static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old,
2618 Value *V, uint64_t Offset, const Twine &Name) {
2619 IntegerType *IntTy = cast<IntegerType>(Old->getType());
2620 IntegerType *Ty = cast<IntegerType>(V->getType());
2621 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2622 "Cannot insert a larger integer!");
2623 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2624 if (Ty != IntTy) {
2625 V = IRB.CreateZExt(V, IntTy, Name + ".ext");
2626 LLVM_DEBUG(dbgs() << " extended: " << *V << "\n");
2627 }
2628 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2629 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2630 "Element store outside of alloca store");
2631 uint64_t ShAmt = 8 * Offset;
2632 if (DL.isBigEndian())
2633 ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedValue() -
2634 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2635 if (ShAmt) {
2636 V = IRB.CreateShl(V, ShAmt, Name + ".shift");
2637 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2638 }
2639
2640 if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) {
2641 APInt Mask = ~Ty->getMask().zext(IntTy->getBitWidth()).shl(ShAmt);
2642 Old = IRB.CreateAnd(Old, Mask, Name + ".mask");
2643 LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n");
2644 V = IRB.CreateOr(Old, V, Name + ".insert");
2645 LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n");
2646 }
2647 return V;
2648}
2649
2650static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex,
2651 unsigned EndIndex, const Twine &Name) {
2652 auto *VecTy = cast<FixedVectorType>(V->getType());
2653 unsigned NumElements = EndIndex - BeginIndex;
2654 assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2655
2656 if (NumElements == VecTy->getNumElements())
2657 return V;
2658
2659 if (NumElements == 1) {
2660 V = IRB.CreateExtractElement(V, BeginIndex, Name + ".extract");
2661 LLVM_DEBUG(dbgs() << " extract: " << *V << "\n");
2662 return V;
2663 }
2664
2665 auto Mask = llvm::to_vector<8>(llvm::seq<int>(BeginIndex, EndIndex));
2666 V = IRB.CreateShuffleVector(V, Mask, Name + ".extract");
2667 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2668 return V;
2669}
2670
2671static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V,
2672 unsigned BeginIndex, const Twine &Name) {
2673 VectorType *VecTy = cast<VectorType>(Old->getType());
2674 assert(VecTy && "Can only insert a vector into a vector");
2675
2676 VectorType *Ty = dyn_cast<VectorType>(V->getType());
2677 if (!Ty) {
2678 // Single element to insert.
2679 V = IRB.CreateInsertElement(Old, V, BeginIndex, Name + ".insert");
2680 LLVM_DEBUG(dbgs() << " insert: " << *V << "\n");
2681 return V;
2682 }
2683
2684 unsigned NumSubElements = cast<FixedVectorType>(Ty)->getNumElements();
2685 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
2686
2687 assert(NumSubElements <= NumElements && "Too many elements!");
2688 if (NumSubElements == NumElements) {
2689 assert(V->getType() == VecTy && "Vector type mismatch");
2690 return V;
2691 }
2692 unsigned EndIndex = BeginIndex + NumSubElements;
2693
2694 // When inserting a smaller vector into the larger to store, we first
2695 // use a shuffle vector to widen it with undef elements, and then
2696 // a second shuffle vector to select between the loaded vector and the
2697 // incoming vector.
2699 Mask.reserve(NumElements);
2700 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2701 if (Idx >= BeginIndex && Idx < EndIndex)
2702 Mask.push_back(Idx - BeginIndex);
2703 else
2704 Mask.push_back(-1);
2705 V = IRB.CreateShuffleVector(V, Mask, Name + ".expand");
2706 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2707
2708 Mask.clear();
2709 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2710 if (Idx >= BeginIndex && Idx < EndIndex)
2711 Mask.push_back(Idx);
2712 else
2713 Mask.push_back(Idx + NumElements);
2714 V = IRB.CreateShuffleVector(V, Old, Mask, Name + "blend");
2715 LLVM_DEBUG(dbgs() << " blend: " << *V << "\n");
2716 return V;
2717}
2718
2719/// This function takes two vector values and combines them into a single vector
2720/// by concatenating their elements. The function handles:
2721///
2722/// 1. Element type mismatch: If either vector's element type differs from
2723/// NewAIEltType, the function bitcasts the vector to use NewAIEltType while
2724/// preserving the total bit width (adjusting the number of elements
2725/// accordingly).
2726///
2727/// 2. Size mismatch: After transforming the vectors to have the desired element
2728/// type, if the two vectors have different numbers of elements, the smaller
2729/// vector is extended with poison values to match the size of the larger
2730/// vector before concatenation.
2731///
2732/// 3. Concatenation: The vectors are merged using a shuffle operation that
2733/// places all elements of V0 first, followed by all elements of V1.
2734///
2735/// \param V0 The first vector to merge (must be a vector type)
2736/// \param V1 The second vector to merge (must be a vector type)
2737/// \param DL The data layout for size calculations
2738/// \param NewAIEltTy The desired element type for the result vector
2739/// \param Builder IRBuilder for creating new instructions
2740/// \return A new vector containing all elements from V0 followed by all
2741/// elements from V1
2743 Type *NewAIEltTy, IRBuilder<> &Builder) {
2744 // V0 and V1 are vectors
2745 // Create a new vector type with combined elements
2746 // Use ShuffleVector to concatenate the vectors
2747 auto *VecType0 = cast<FixedVectorType>(V0->getType());
2748 auto *VecType1 = cast<FixedVectorType>(V1->getType());
2749
2750 // If V0/V1 element types are different from NewAllocaElementType,
2751 // we need to introduce bitcasts before merging them
2752 auto BitcastIfNeeded = [&](Value *&V, FixedVectorType *&VecType,
2753 const char *DebugName) {
2754 Type *EltType = VecType->getElementType();
2755 if (EltType != NewAIEltTy) {
2756 // Calculate new number of elements to maintain same bit width
2757 unsigned TotalBits =
2758 VecType->getNumElements() * DL.getTypeSizeInBits(EltType);
2759 unsigned NewNumElts = TotalBits / DL.getTypeSizeInBits(NewAIEltTy);
2760
2761 auto *NewVecType = FixedVectorType::get(NewAIEltTy, NewNumElts);
2762 V = Builder.CreateBitCast(V, NewVecType);
2763 VecType = NewVecType;
2764 LLVM_DEBUG(dbgs() << " bitcast " << DebugName << ": " << *V << "\n");
2765 }
2766 };
2767
2768 BitcastIfNeeded(V0, VecType0, "V0");
2769 BitcastIfNeeded(V1, VecType1, "V1");
2770
2771 unsigned NumElts0 = VecType0->getNumElements();
2772 unsigned NumElts1 = VecType1->getNumElements();
2773
2774 SmallVector<int, 16> ShuffleMask;
2775
2776 if (NumElts0 == NumElts1) {
2777 for (unsigned i = 0; i < NumElts0 + NumElts1; ++i)
2778 ShuffleMask.push_back(i);
2779 } else {
2780 // If two vectors have different sizes, we need to extend
2781 // the smaller vector to the size of the larger vector.
2782 unsigned SmallSize = std::min(NumElts0, NumElts1);
2783 unsigned LargeSize = std::max(NumElts0, NumElts1);
2784 bool IsV0Smaller = NumElts0 < NumElts1;
2785 Value *&ExtendedVec = IsV0Smaller ? V0 : V1;
2786 SmallVector<int, 16> ExtendMask;
2787 for (unsigned i = 0; i < SmallSize; ++i)
2788 ExtendMask.push_back(i);
2789 for (unsigned i = SmallSize; i < LargeSize; ++i)
2790 ExtendMask.push_back(PoisonMaskElem);
2791 ExtendedVec = Builder.CreateShuffleVector(
2792 ExtendedVec, PoisonValue::get(ExtendedVec->getType()), ExtendMask);
2793 LLVM_DEBUG(dbgs() << " shufflevector: " << *ExtendedVec << "\n");
2794 for (unsigned i = 0; i < NumElts0; ++i)
2795 ShuffleMask.push_back(i);
2796 for (unsigned i = 0; i < NumElts1; ++i)
2797 ShuffleMask.push_back(LargeSize + i);
2798 }
2799
2800 return Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2801}
2802
2803namespace {
2804
2805/// Visitor to rewrite instructions using p particular slice of an alloca
2806/// to use a new alloca.
2807///
2808/// Also implements the rewriting to vector-based accesses when the partition
2809/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
2810/// lives here.
2811class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
2812 // Befriend the base class so it can delegate to private visit methods.
2813 friend class InstVisitor<AllocaSliceRewriter, bool>;
2814
2815 using Base = InstVisitor<AllocaSliceRewriter, bool>;
2816
2817 const DataLayout &DL;
2818 AllocaSlices &AS;
2819 SROA &Pass;
2820 AllocaInst &OldAI, &NewAI;
2821 const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
2822 Type *NewAllocaTy;
2823
2824 // This is a convenience and flag variable that will be null unless the new
2825 // alloca's integer operations should be widened to this integer type due to
2826 // passing isIntegerWideningViable above. If it is non-null, the desired
2827 // integer type will be stored here for easy access during rewriting.
2828 IntegerType *IntTy;
2829
2830 // If we are rewriting an alloca partition which can be written as pure
2831 // vector operations, we stash extra information here. When VecTy is
2832 // non-null, we have some strict guarantees about the rewritten alloca:
2833 // - The new alloca is exactly the size of the vector type here.
2834 // - The accesses all either map to the entire vector or to a single
2835 // element.
2836 // - The set of accessing instructions is only one of those handled above
2837 // in isVectorPromotionViable. Generally these are the same access kinds
2838 // which are promotable via mem2reg.
2839 VectorType *VecTy;
2840 Type *ElementTy;
2841 uint64_t ElementSize;
2842
2843 // The original offset of the slice currently being rewritten relative to
2844 // the original alloca.
2845 uint64_t BeginOffset = 0;
2846 uint64_t EndOffset = 0;
2847
2848 // The new offsets of the slice currently being rewritten relative to the
2849 // original alloca.
2850 uint64_t NewBeginOffset = 0, NewEndOffset = 0;
2851
2852 uint64_t SliceSize = 0;
2853 bool IsSplittable = false;
2854 bool IsSplit = false;
2855 Use *OldUse = nullptr;
2856 Instruction *OldPtr = nullptr;
2857
2858 // Track post-rewrite users which are PHI nodes and Selects.
2859 SmallSetVector<PHINode *, 8> &PHIUsers;
2860 SmallSetVector<SelectInst *, 8> &SelectUsers;
2861
2862 // Utility IR builder, whose name prefix is setup for each visited use, and
2863 // the insertion point is set to point to the user.
2864 IRBuilderTy IRB;
2865
2866 // Return the new alloca, addrspacecasted if required to avoid changing the
2867 // addrspace of a volatile access.
2868 Value *getPtrToNewAI(unsigned AddrSpace, bool IsVolatile) {
2869 if (!IsVolatile || AddrSpace == NewAI.getType()->getPointerAddressSpace())
2870 return &NewAI;
2871
2872 Type *AccessTy = IRB.getPtrTy(AddrSpace);
2873 return IRB.CreateAddrSpaceCast(&NewAI, AccessTy);
2874 }
2875
2876public:
2877 AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass,
2878 AllocaInst &OldAI, AllocaInst &NewAI, Type *NewAllocaTy,
2879 uint64_t NewAllocaBeginOffset,
2880 uint64_t NewAllocaEndOffset, bool IsIntegerPromotable,
2881 VectorType *PromotableVecTy,
2882 SmallSetVector<PHINode *, 8> &PHIUsers,
2883 SmallSetVector<SelectInst *, 8> &SelectUsers)
2884 : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI),
2885 NewAllocaBeginOffset(NewAllocaBeginOffset),
2886 NewAllocaEndOffset(NewAllocaEndOffset), NewAllocaTy(NewAllocaTy),
2887 IntTy(IsIntegerPromotable
2888 ? Type::getIntNTy(
2889 NewAI.getContext(),
2890 DL.getTypeSizeInBits(NewAllocaTy).getFixedValue())
2891 : nullptr),
2892 VecTy(PromotableVecTy),
2893 ElementTy(VecTy ? VecTy->getElementType() : nullptr),
2894 ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy).getFixedValue() / 8
2895 : 0),
2896 PHIUsers(PHIUsers), SelectUsers(SelectUsers),
2897 IRB(NewAI.getContext(), ConstantFolder()) {
2898 if (VecTy) {
2899 assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 &&
2900 "Only multiple-of-8 sized vector elements are viable");
2901 ++NumVectorized;
2902 }
2903 assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy));
2904 }
2905
2906 bool visit(AllocaSlices::const_iterator I) {
2907 bool CanSROA = true;
2908 BeginOffset = I->beginOffset();
2909 EndOffset = I->endOffset();
2910 IsSplittable = I->isSplittable();
2911 IsSplit =
2912 BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset;
2913 LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""));
2914 LLVM_DEBUG(AS.printSlice(dbgs(), I, ""));
2915 LLVM_DEBUG(dbgs() << "\n");
2916
2917 // Compute the intersecting offset range.
2918 assert(BeginOffset < NewAllocaEndOffset);
2919 assert(EndOffset > NewAllocaBeginOffset);
2920 NewBeginOffset = std::max(BeginOffset, NewAllocaBeginOffset);
2921 NewEndOffset = std::min(EndOffset, NewAllocaEndOffset);
2922
2923 SliceSize = NewEndOffset - NewBeginOffset;
2924 LLVM_DEBUG(dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset
2925 << ") NewBegin:(" << NewBeginOffset << ", "
2926 << NewEndOffset << ") NewAllocaBegin:("
2927 << NewAllocaBeginOffset << ", " << NewAllocaEndOffset
2928 << ")\n");
2929 assert(IsSplit || NewBeginOffset == BeginOffset);
2930 OldUse = I->getUse();
2931 OldPtr = cast<Instruction>(OldUse->get());
2932
2933 Instruction *OldUserI = cast<Instruction>(OldUse->getUser());
2934 IRB.SetInsertPoint(OldUserI);
2935 IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc());
2936 // Avoid materializing the name prefix when it is discarded anyway.
2937 if (!IRB.getContext().shouldDiscardValueNames())
2938 IRB.getInserter().SetNamePrefix(Twine(NewAI.getName()) + "." +
2939 Twine(BeginOffset) + ".");
2940
2941 CanSROA &= visit(cast<Instruction>(OldUse->getUser()));
2942 if (VecTy || IntTy)
2943 assert(CanSROA);
2944 return CanSROA;
2945 }
2946
2947 /// Attempts to rewrite a partition using tree-structured merge optimization.
2948 ///
2949 /// This function handles two patterns. Both produce an O(log n) tree of
2950 /// shufflevectors in place of the linear expand+blend chain that SROA would
2951 /// otherwise emit for each partial store.
2952 ///
2953 /// Pattern 1 (stores-only):
2954 /// Multiple non-overlapping partial stores completely fill the alloca
2955 /// and there is exactly one full-width load coming after the stores.
2956 /// The stores are tree-merged into a single vector and stored once.
2957 ///
2958 /// Example transformation:
2959 /// Before: (stores do not have to be in order)
2960 /// %alloca = alloca <8 x float>
2961 /// store <2 x float> %val0, ptr %alloca ; offset 0-1
2962 /// store <2 x float> %val2, ptr %alloca+16 ; offset 4-5
2963 /// store <2 x float> %val1, ptr %alloca+8 ; offset 2-3
2964 /// store <2 x float> %val3, ptr %alloca+24 ; offset 6-7
2965 /// %r = load <8 x float>, ptr %alloca
2966 ///
2967 /// After: tree of shufflevectors producing <8 x float> directly.
2968 ///
2969 /// Pattern 2 (init + RMW, possibly multi-round):
2970 /// A single full-width init store, followed by partial loads and
2971 /// partial stores that read-modify-write the alloca one or more
2972 /// times, optionally followed by a full-width load. The only
2973 /// structural requirement is that the distinct [begin, end) ranges
2974 /// touched by the partial loads and stores, taken together, tile
2975 /// the alloca disjointly.
2976 ///
2977 /// We keep a map from each slice range to the SSA value that
2978 /// currently lives there, `SliceValues[r] -> Value*`:
2979 /// - initialize each entry to the corresponding piece of the
2980 /// init store's value (via a shufflevector picking the
2981 /// range's elements out of the init value),
2982 /// - walk partial loads and stores in block order,
2983 /// - for a partial load at range r: RAUW with `SliceValues[r]`,
2984 /// - for a partial store at range r: update `SliceValues[r]` to
2985 /// the stored value and drop the store.
2986 /// At the end, the final `SliceValues[r]` entries are tree-merged
2987 /// (in range order) into a single store to the alloca, and the
2988 /// optional full-width load is replaced by a load of the alloca.
2989 ///
2990 /// Because the ranges are disjoint by construction, a store at one
2991 /// range cannot affect another range's tracked value, so a single
2992 /// block-order walk correctly tracks the memory state at each
2993 /// range. The algorithm handles multi-round RMW, partial loads
2994 /// and stores interleaved in any order, read-only slices (the
2995 /// tracked value stays at the init extract), and write-only
2996 /// slices (the tracked value never flows into a load).
2997 ///
2998 /// \param P The partition to analyze and potentially rewrite
2999 /// \return An optional vector of values that were deleted during the
3000 /// rewrite, or std::nullopt if the partition cannot be optimized.
3001 std::optional<SmallVector<Value *, 4>>
3002 rewriteTreeStructuredMerge(Partition &P) {
3003 // No tail slices that overlap with the partition
3004 if (P.splitSliceTails().size() > 0)
3005 return std::nullopt;
3006
3007 // Structure to hold store information
3008 struct StoreInfo {
3009 StoreInst *Store;
3010 uint64_t BeginOffset;
3011 uint64_t EndOffset;
3012 Value *StoredValue;
3013 StoreInfo(StoreInst *SI, uint64_t Begin, uint64_t End, Value *Val)
3014 : Store(SI), BeginOffset(Begin), EndOffset(End), StoredValue(Val) {}
3015 };
3016 struct LoadInfo {
3017 LoadInst *Load;
3018 uint64_t BeginOffset;
3019 uint64_t EndOffset;
3020 };
3021
3022 SmallVector<StoreInfo, 4> StoreInfos; // partial stores only
3023 SmallVector<LoadInfo, 4> LoadInfos; // partial loads only
3024 LoadInst *FullLoad = nullptr; // optional full-width load
3025 StoreInst *InitStore = nullptr; // optional full-width init store
3026
3027 // If the new alloca is a fixed vector type, we use its element type as the
3028 // allocated element type, otherwise we use i8 as the allocated element
3029 Type *AllocatedEltTy =
3030 isa<FixedVectorType>(NewAllocaTy)
3031 ? cast<FixedVectorType>(NewAllocaTy)->getElementType()
3032 : Type::getInt8Ty(NewAI.getContext());
3033 unsigned AllocatedEltTySize = DL.getTypeSizeInBits(AllocatedEltTy);
3034
3035 // Helper to check if a type is
3036 // 1. A fixed vector type
3037 // 2. The element type is not a pointer
3038 // 3. The element type size is byte-aligned
3039 // We only handle the cases that the ld/st meet these conditions
3040 auto IsTypeValidForTreeStructuredMerge = [&](Type *Ty) -> bool {
3041 auto *FixedVecTy = dyn_cast<FixedVectorType>(Ty);
3042 return FixedVecTy &&
3043 DL.getTypeSizeInBits(FixedVecTy->getElementType()) % 8 == 0 &&
3044 !FixedVecTy->getElementType()->isPointerTy();
3045 };
3046
3047 for (Slice &S : P) {
3048 auto *User = cast<Instruction>(S.getUse()->getUser());
3049 // A "full-width" slice spans the entire alloca; it's either the single
3050 // init store (Pattern 2) or the single final load (both patterns).
3051 bool IsFullWidth = (S.beginOffset() == NewAllocaBeginOffset &&
3052 S.endOffset() == NewAllocaEndOffset);
3053 if (auto *LI = dyn_cast<LoadInst>(User)) {
3054 // Only handle simple (non-volatile, non-atomic) loads.
3055 if (!LI->isSimple() ||
3056 !IsTypeValidForTreeStructuredMerge(LI->getType()))
3057 return std::nullopt;
3058 if (IsFullWidth) {
3059 // We accept at most one full-width load (the "final" load, after
3060 // all the partial stores).
3061 if (FullLoad)
3062 return std::nullopt;
3063 FullLoad = LI;
3064 } else {
3065 // Partial load (RMW pattern only).
3066 LoadInfos.push_back({LI, S.beginOffset(), S.endOffset()});
3067 }
3068 } else if (auto *SI = dyn_cast<StoreInst>(User)) {
3069 // Do not handle the case if
3070 // 1. The store does not meet the conditions in the helper function
3071 // 2. The store is not simple — we drop stores as part of the
3072 // rewrite, so volatile stores (which must be kept) and atomic
3073 // stores (which carry memory-ordering semantics) are unsound
3074 // to replace with SSA bookkeeping.
3075 // 3. The total store size is not a multiple of the allocated
3076 // element type size (required so the tree merge can produce a
3077 // vector whose element type matches the alloca).
3078 if (!SI->isSimple() || !IsTypeValidForTreeStructuredMerge(
3079 SI->getValueOperand()->getType()))
3080 return std::nullopt;
3081 auto *StVecTy = cast<FixedVectorType>(SI->getValueOperand()->getType());
3082 unsigned NumElts = StVecTy->getNumElements();
3083 unsigned EltSize = DL.getTypeSizeInBits(StVecTy->getElementType());
3084 if (NumElts * EltSize % AllocatedEltTySize != 0)
3085 return std::nullopt;
3086 if (IsFullWidth) {
3087 // At most one full-width store is allowed — it's the init store
3088 // for the RMW pattern.
3089 if (InitStore)
3090 return std::nullopt;
3091 InitStore = SI;
3092 } else {
3093 StoreInfos.emplace_back(SI, S.beginOffset(), S.endOffset(),
3094 SI->getValueOperand());
3095 }
3096 } else {
3097 // If we have instructions other than load and store, we cannot do
3098 // the tree structured merge.
3099 return std::nullopt;
3100 }
3101 }
3102
3103 // Need at least two partial stores to benefit from tree-merging; a
3104 // single store is already optimal as-is. This applies to both patterns
3105 // below, so check it before classifying.
3106 if (StoreInfos.size() < 2)
3107 return std::nullopt;
3108
3109 // Classify the pattern by looking at what we collected:
3110 // Pattern 1 (stores-only): only partial stores + exactly one full load.
3111 // Pattern 2 (RMW): one full init store + partial loads + partial stores
3112 // (+ optional full final load). RMW also needs VecTy to be set
3113 // because we use getIndex() to convert byte offsets to element
3114 // indices, which requires a promoted vector alloca.
3115 bool IsRMWPattern = InitStore && VecTy && !LoadInfos.empty();
3116 bool IsStoresOnlyPattern = !InitStore && FullLoad && LoadInfos.empty();
3117 if (!IsRMWPattern && !IsStoresOnlyPattern)
3118 return std::nullopt;
3119
3120 // All partial stores must live in the same basic block — the tree merge
3121 // is built in a single BB using block-order ordering (comesBefore).
3122 BasicBlock *StoreBB = StoreInfos[0].Store->getParent();
3123 for (auto &Info : StoreInfos)
3124 if (Info.Store->getParent() != StoreBB)
3125 return std::nullopt;
3126
3127 SmallVector<Value *, 4> DeletedValues;
3128
3129 // Helper: pairwise tree-merge a list of vectors into a single vector.
3130 // At each iteration we merge each adjacent pair via mergeTwoVectors,
3131 // collect the merged values into Next, and (if Vals had odd length)
3132 // carry the trailing element through unchanged. Loop until one value
3133 // remains — the fully-merged vector.
3134 auto TreeMerge = [&](SmallVectorImpl<Value *> &Vals,
3135 IRBuilder<> &B) -> Value * {
3136 LLVM_DEBUG(dbgs() << " Rewrite stores into shufflevectors:\n");
3137 while (Vals.size() > 1) {
3138 SmallVector<Value *, 8> Next;
3139 for (unsigned I = 0, E = Vals.size(); I + 1 < E; I += 2) {
3140 Value *M =
3141 mergeTwoVectors(Vals[I], Vals[I + 1], DL, AllocatedEltTy, B);
3142 LLVM_DEBUG(dbgs() << " shufflevector: " << *M << "\n");
3143 Next.push_back(M);
3144 }
3145 if (Vals.size() % 2 == 1)
3146 Next.push_back(Vals.back());
3147 Vals = std::move(Next);
3148 }
3149 return Vals[0];
3150 };
3151
3152 // Replace a full-width load with a load of the freshly-merged alloca.
3153 // The merge stored a value of type Merged->getType() into NewAI; we load
3154 // that same type back so every access to NewAI stays consistently typed
3155 // (otherwise the alloca is no longer promotable).
3156 auto ReplaceFullLoad = [&](LoadInst *LoadToReplace, Value *Merged) {
3157 IRBuilder<> LoadBuilder(LoadToReplace);
3158 Value *NewLoad = LoadBuilder.CreateAlignedLoad(
3159 Merged->getType(), &NewAI, getSliceAlign(),
3160 LoadToReplace->isVolatile(),
3161 LoadToReplace->getName() + ".sroa.new.load");
3162 if (NewLoad->getType() != LoadToReplace->getType())
3163 NewLoad = LoadBuilder.CreateBitCast(NewLoad, LoadToReplace->getType());
3164 LoadToReplace->replaceAllUsesWith(NewLoad);
3165 DeletedValues.push_back(LoadToReplace);
3166 };
3167
3168 if (IsStoresOnlyPattern) {
3169 // Stores should not overlap and should cover the whole alloca.
3170 // Sort by begin offset to verify this with a single linear scan.
3171 llvm::sort(StoreInfos, [](const StoreInfo &A, const StoreInfo &B) {
3172 return A.BeginOffset < B.BeginOffset;
3173 });
3174 // Check for gap or overlap: each begin offset must equal the previous
3175 // end offset, i.e. the store ranges must tile [NewAllocaBeginOffset,
3176 // NewAllocaEndOffset) exactly.
3177 uint64_t Expected = NewAllocaBeginOffset;
3178 for (auto &Info : StoreInfos) {
3179 if (Info.BeginOffset != Expected)
3180 return std::nullopt;
3181 Expected = Info.EndOffset;
3182 }
3183 // Stores cover the entire alloca (no trailing gap either).
3184 if (Expected != NewAllocaEndOffset)
3185 return std::nullopt;
3186
3187 // The load should not be in the middle of the stores.
3188 // Note:
3189 // If the load is in a different basic block from the stores, we can
3190 // still do the tree-structured merge. We don't have store->load
3191 // forwarding here — the merged vector is stored back to NewAI and
3192 // the new load loads from NewAI. The forwarding will be handled
3193 // later when NewAI is promoted.
3194 BasicBlock *LoadBB = FullLoad->getParent();
3195 if (LoadBB == StoreBB) {
3196 for (auto &Info : StoreInfos)
3197 if (!Info.Store->comesBefore(FullLoad))
3198 return std::nullopt;
3199 }
3200
3201 LLVM_DEBUG({
3202 dbgs() << "Tree structured merge rewrite (stores-only):\n";
3203 dbgs() << " Load: " << *FullLoad << "\n Ordered stores:\n";
3204 for (auto [I, Info] : enumerate(StoreInfos)) {
3205 dbgs() << " [" << I << "] Range[" << Info.BeginOffset << ", "
3206 << Info.EndOffset << ") \tStore: " << *Info.Store
3207 << "\tValue: " << *Info.StoredValue << "\n";
3208 }
3209 });
3210
3211 // StoreInfos is sorted by offset, not by block order. Anchoring to
3212 // StoreInfos.back().Store (last by offset) can place shuffles before
3213 // operands that appear later in the block (invalid SSA). Insert before
3214 // FullLoad when it shares the store block (after all stores, before
3215 // any later IR in that block). Otherwise insert before the store
3216 // block's terminator so the merge runs after every store and any
3217 // trailing instructions in that block.
3218 IRBuilder<> Builder(LoadBB == StoreBB ? cast<Instruction>(FullLoad)
3219 : StoreBB->getTerminator());
3220 SmallVector<Value *, 8> Vals;
3221 for (const auto &Info : StoreInfos) {
3222 DeletedValues.push_back(Info.Store);
3223 Vals.push_back(Info.StoredValue);
3224 }
3225 // Merge all stored values and store the merged value into the alloca.
3226 Value *Merged = TreeMerge(Vals, Builder);
3227 Builder.CreateAlignedStore(Merged, &NewAI, getSliceAlign());
3228
3229 // Replace the original load with a load of the newly-merged alloca.
3230 ReplaceFullLoad(FullLoad, Merged);
3231 return DeletedValues;
3232 }
3233
3234 // RMW pattern handling starts from here.
3235 // Like StoreBB above: keep the init store, all partial loads and all
3236 // partial stores in one basic block so we can reason about ordering
3237 // with comesBefore and build SSA without PHIs.
3238 if (InitStore->getParent() != StoreBB)
3239 return std::nullopt;
3240 if (any_of(LoadInfos, [&](const LoadInfo &I) {
3241 return I.Load->getParent() != StoreBB;
3242 }))
3243 return std::nullopt;
3244 // FullLoad (if any) is allowed to live in a different basic block. See
3245 // the note on the stores-only path: we don't do store->load forwarding
3246 // directly — the merged vector is stored to NewAI and the new load
3247 // loads from NewAI, so cross-BB ordering is resolved later when NewAI
3248 // is promoted.
3249
3250 // Collect the combined partial-load/partial-store accesses sorted
3251 // by block order. Used both for ordering checks and for the rewrite
3252 // walk below.
3253 struct Access {
3254 Instruction *Inst;
3255 uint64_t BeginOffset, EndOffset;
3256 bool IsStore;
3257 };
3259 Accesses.reserve(LoadInfos.size() + StoreInfos.size());
3260 for (const auto &L : LoadInfos)
3261 Accesses.push_back({L.Load, L.BeginOffset, L.EndOffset, false});
3262 for (const auto &S : StoreInfos)
3263 Accesses.push_back({S.Store, S.BeginOffset, S.EndOffset, true});
3264 llvm::sort(Accesses, [](const Access &A, const Access &B) {
3265 return A.Inst->comesBefore(B.Inst);
3266 });
3267
3268 // Ordering constraint 1: InitStore must come before every partial
3269 // access — they read/write the RMW state initialised by InitStore.
3270 // Accesses is sorted by block order, so the first element is the
3271 // earliest; checking it is enough.
3272 if (!InitStore->comesBefore(Accesses.front().Inst))
3273 return std::nullopt;
3274 // Ordering constraint 2: when FullLoad shares the block with the
3275 // partial accesses, it must come after every one of them — otherwise
3276 // it could read a stale value. Accesses is sorted, so the last
3277 // element is the latest; checking it is enough. If FullLoad is in
3278 // another block, mem2reg forwards the merged store to it.
3279 if (FullLoad && FullLoad->getParent() == StoreBB &&
3280 !Accesses.back().Inst->comesBefore(FullLoad))
3281 return std::nullopt;
3282
3283 // Coverage check: the distinct [begin, end) ranges touched by the
3284 // partial loads and stores must tile the alloca disjointly. That is
3285 // the only precondition the per-range SliceValues tracking below
3286 // needs — a disjoint tile guarantees the entries don't alias each
3287 // other. We don't check per-range load/store counts: a range with
3288 // only loads ends with SliceValues[r] = the init extract
3289 // (contributed to the final tree-merge), and a range with only
3290 // stores ends with SliceValues[r] = its last stored value. Both are
3291 // correct.
3292 using SliceRange = std::pair<uint64_t, uint64_t>;
3293 SmallVector<SliceRange, 8> SortedRanges;
3294 SortedRanges.reserve(Accesses.size());
3295 for (auto &Acc : Accesses)
3296 SortedRanges.emplace_back(Acc.BeginOffset, Acc.EndOffset);
3297 llvm::sort(SortedRanges);
3298 SortedRanges.erase(llvm::unique(SortedRanges), SortedRanges.end());
3299 // Disjoint + contiguous tile of the whole alloca.
3300 uint64_t Expected = NewAllocaBeginOffset;
3301 for (auto &Range : SortedRanges) {
3302 if (Range.first != Expected)
3303 return std::nullopt;
3304 Expected = Range.second;
3305 }
3306 if (Expected != NewAllocaEndOffset)
3307 return std::nullopt;
3308
3309 LLVM_DEBUG({
3310 dbgs() << "Tree structured merge rewrite (RMW):\n";
3311 dbgs() << " Init store: " << *InitStore << "\n";
3312 if (FullLoad)
3313 dbgs() << " Final load: " << *FullLoad << "\n";
3314 dbgs() << " Slice ranges (" << SortedRanges.size() << "):\n";
3315 for (auto &Range : SortedRanges)
3316 dbgs() << " [" << Range.first << ", " << Range.second << ")\n";
3317 });
3318
3319 // Initialize SliceValues: one SSA value per slice range, tracking
3320 // the value the alloca currently holds at that range. Each entry
3321 // starts at the corresponding piece of the init store, obtained by
3322 // bitcasting the init value to the alloca's vector type (if needed)
3323 // and extracting the slice's sub-range.
3324 IRB.SetInsertPoint(InitStore->getNextNode());
3325 Value *InitVec = InitStore->getValueOperand();
3326 if (InitVec->getType() != NewAllocaTy)
3327 InitVec = IRB.CreateBitCast(InitVec, NewAllocaTy, "init.cast");
3328 DenseMap<SliceRange, Value *> SliceValues;
3329 for (auto &Range : SortedRanges) {
3330 unsigned BeginIdx = getIndex(Range.first);
3331 unsigned EndIdx = getIndex(Range.second);
3332 SliceValues[Range] = IRB.CreateShuffleVector(
3333 InitVec, createSequentialMask(BeginIdx, EndIdx - BeginIdx, 0),
3334 "init.extract");
3335 }
3336 // The init store itself becomes dead — its value is consumed via the
3337 // extracts above.
3338 DeletedValues.push_back(InitStore);
3339
3340 // Walk accesses in block order:
3341 // - partial load at range r: replace with SliceValues[r] (bitcast
3342 // if the load's type differs from the current tracked value's
3343 // type, e.g. because a previous store wrote a vector with a
3344 // different element type);
3345 // - partial store at range r: update SliceValues[r] to the stored
3346 // value and drop the store.
3347 for (auto &Acc : Accesses) {
3348 SliceRange Range{Acc.BeginOffset, Acc.EndOffset};
3349 if (!Acc.IsStore) {
3350 Value *V = SliceValues[Range];
3351 if (V->getType() != Acc.Inst->getType()) {
3352 IRB.SetInsertPoint(cast<LoadInst>(Acc.Inst));
3353 V = IRB.CreateBitCast(V, Acc.Inst->getType());
3354 }
3355 Acc.Inst->replaceAllUsesWith(V);
3356 } else {
3357 SliceValues[Range] = cast<StoreInst>(Acc.Inst)->getValueOperand();
3358 }
3359 DeletedValues.push_back(Acc.Inst);
3360 }
3361
3362 // Tree-merge the final per-range values (in range order) into the
3363 // alloca's final vector value. Anchor the IRBuilder to FullLoad (when it
3364 // shares the partial-access block) or otherwise to the block's
3365 // terminator — never to a partial access, since those are queued for
3366 // deletion. Both anchors are guaranteed to dominate every SliceValues
3367 // entry: each one is either an init extract (before any access) or a
3368 // stored value defined before its (now-deleted) store.
3369 IRBuilder<> Builder(FullLoad && FullLoad->getParent() == StoreBB
3370 ? cast<Instruction>(FullLoad)
3371 : StoreBB->getTerminator());
3372 SmallVector<Value *, 8> Vals;
3373 for (auto &Range : SortedRanges)
3374 Vals.push_back(SliceValues[Range]);
3375 Value *Merged = TreeMerge(Vals, Builder);
3376 Builder.CreateAlignedStore(Merged, &NewAI, getSliceAlign());
3377
3378 // Replace the optional final full-width load with a load of the newly
3379 // merged alloca. Later promotion will forward the store above to it.
3380 if (FullLoad)
3381 ReplaceFullLoad(FullLoad, Merged);
3382
3383 return DeletedValues;
3384 }
3385
3386private:
3387 // Make sure the other visit overloads are visible.
3388 using Base::visit;
3389
3390 // Every instruction which can end up as a user must have a rewrite rule.
3391 bool visitInstruction(Instruction &I) {
3392 LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n");
3393 llvm_unreachable("No rewrite rule for this instruction!");
3394 }
3395
3396 Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) {
3397 // Note that the offset computation can use BeginOffset or NewBeginOffset
3398 // interchangeably for unsplit slices.
3399 assert(IsSplit || BeginOffset == NewBeginOffset);
3400 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3401
3402 StringRef OldName = OldPtr->getName();
3403 // Skip through the last '.sroa.' component of the name.
3404 size_t LastSROAPrefix = OldName.rfind(".sroa.");
3405 if (LastSROAPrefix != StringRef::npos) {
3406 OldName = OldName.substr(LastSROAPrefix + strlen(".sroa."));
3407 // Look for an SROA slice index.
3408 size_t IndexEnd = OldName.find_first_not_of("0123456789");
3409 if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') {
3410 // Strip the index and look for the offset.
3411 OldName = OldName.substr(IndexEnd + 1);
3412 size_t OffsetEnd = OldName.find_first_not_of("0123456789");
3413 if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.')
3414 // Strip the offset.
3415 OldName = OldName.substr(OffsetEnd + 1);
3416 }
3417 }
3418 // Strip any SROA suffixes as well.
3419 OldName = OldName.substr(0, OldName.find(".sroa_"));
3420
3421 return getAdjustedPtr(IRB, DL, &NewAI,
3422 APInt(DL.getIndexTypeSizeInBits(PointerTy), Offset),
3423 PointerTy, Twine(OldName) + ".");
3424 }
3425
3426 /// Compute suitable alignment to access this slice of the *new*
3427 /// alloca.
3428 ///
3429 /// You can optionally pass a type to this routine and if that type's ABI
3430 /// alignment is itself suitable, this will return zero.
3431 Align getSliceAlign() {
3432 return commonAlignment(NewAI.getAlign(),
3433 NewBeginOffset - NewAllocaBeginOffset);
3434 }
3435
3436 unsigned getIndex(uint64_t Offset) {
3437 assert(VecTy && "Can only call getIndex when rewriting a vector");
3438 uint64_t RelOffset = Offset - NewAllocaBeginOffset;
3439 assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
3440 uint32_t Index = RelOffset / ElementSize;
3441 assert(Index * ElementSize == RelOffset);
3442 return Index;
3443 }
3444
3445 void deleteIfTriviallyDead(Value *V) {
3448 Pass.DeadInsts.push_back(I);
3449 }
3450
3451 Value *rewriteVectorizedLoadInst(LoadInst &LI) {
3452 unsigned BeginIndex = getIndex(NewBeginOffset);
3453 unsigned EndIndex = getIndex(NewEndOffset);
3454 assert(EndIndex > BeginIndex && "Empty vector!");
3455
3456 LoadInst *Load =
3457 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
3458
3459 Load->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
3460 LLVMContext::MD_access_group});
3461 return extractVector(IRB, Load, BeginIndex, EndIndex, "vec");
3462 }
3463
3464 Value *rewriteIntegerLoad(LoadInst &LI) {
3465 assert(IntTy && "We cannot insert an integer to the alloca");
3466 assert(!LI.isVolatile());
3467 Value *V =
3468 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
3469 V = IRB.CreateBitPreservingCastChain(DL, V, IntTy);
3470 assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3471 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3472 if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) {
3473 IntegerType *ExtractTy = Type::getIntNTy(LI.getContext(), SliceSize * 8);
3474 V = extractInteger(DL, IRB, V, ExtractTy, Offset, "extract");
3475 }
3476 // It is possible that the extracted type is not the load type. This
3477 // happens if there is a load past the end of the alloca, and as
3478 // a consequence the slice is narrower but still a candidate for integer
3479 // lowering. To handle this case, we just zero extend the extracted
3480 // integer.
3481 assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&
3482 "Can only handle an extract for an overly wide load");
3483 if (cast<IntegerType>(LI.getType())->getBitWidth() > SliceSize * 8)
3484 V = IRB.CreateZExt(V, LI.getType());
3485 return V;
3486 }
3487
3488 bool visitLoadInst(LoadInst &LI) {
3489 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
3490 Value *OldOp = LI.getOperand(0);
3491 assert(OldOp == OldPtr);
3492
3493 AAMDNodes AATags = LI.getAAMetadata();
3494
3495 unsigned AS = LI.getPointerAddressSpace();
3496
3497 Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8)
3498 : LI.getType();
3499 bool IsPtrAdjusted = false;
3500 Value *V;
3501 if (VecTy) {
3502 V = rewriteVectorizedLoadInst(LI);
3503 } else if (IntTy && LI.getType()->isIntegerTy()) {
3504 V = rewriteIntegerLoad(LI);
3505 } else if (NewBeginOffset == NewAllocaBeginOffset &&
3506 NewEndOffset == NewAllocaEndOffset &&
3507 (canConvertValue(DL, NewAllocaTy, TargetTy) ||
3508 (NewAllocaTy->isIntegerTy() && TargetTy->isIntegerTy() &&
3509 DL.getTypeStoreSize(TargetTy).getFixedValue() > SliceSize &&
3510 !LI.isVolatile()))) {
3511 Value *NewPtr =
3512 getPtrToNewAI(LI.getPointerAddressSpace(), LI.isVolatile());
3513 LoadInst *NewLI = IRB.CreateAlignedLoad(
3514 NewAllocaTy, NewPtr, NewAI.getAlign(), LI.isVolatile(), LI.getName());
3515 if (LI.isVolatile())
3516 NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
3517 if (NewLI->isAtomic())
3518 NewLI->setAlignment(LI.getAlign());
3519
3520 // Copy any metadata that is valid for the new load. This may require
3521 // conversion to a different kind of metadata, e.g. !nonnull might change
3522 // to !range or vice versa.
3523 copyMetadataForLoad(*NewLI, LI);
3524
3525 // Do this after copyMetadataForLoad() to preserve the TBAA shift.
3526 if (AATags)
3527 NewLI->setAAMetadata(AATags.adjustForAccess(
3528 NewBeginOffset - BeginOffset, NewLI->getType(), DL));
3529
3530 // Try to preserve nonnull metadata
3531 V = NewLI;
3532
3533 // If this is an integer load past the end of the slice (which means the
3534 // bytes outside the slice are undef or this load is dead) just forcibly
3535 // fix the integer size with correct handling of endianness.
3536 if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy))
3537 if (auto *TITy = dyn_cast<IntegerType>(TargetTy))
3538 if (AITy->getBitWidth() < TITy->getBitWidth()) {
3539 V = IRB.CreateZExt(V, TITy, "load.ext");
3540 if (DL.isBigEndian())
3541 V = IRB.CreateShl(V, TITy->getBitWidth() - AITy->getBitWidth(),
3542 "endian_shift");
3543 }
3544 } else {
3545 Type *LTy = IRB.getPtrTy(AS);
3546 LoadInst *NewLI =
3547 IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy),
3548 getSliceAlign(), LI.isVolatile(), LI.getName());
3549
3550 if (AATags)
3551 NewLI->setAAMetadata(AATags.adjustForAccess(
3552 NewBeginOffset - BeginOffset, NewLI->getType(), DL));
3553
3554 if (LI.isVolatile())
3555 NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
3556 NewLI->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
3557 LLVMContext::MD_access_group});
3558
3559 V = NewLI;
3560 IsPtrAdjusted = true;
3561 }
3562 V = IRB.CreateBitPreservingCastChain(DL, V, TargetTy);
3563
3564 if (IsSplit) {
3565 assert(!LI.isVolatile());
3566 assert(LI.getType()->isIntegerTy() &&
3567 "Only integer type loads and stores are split");
3568 assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() &&
3569 "Split load isn't smaller than original load");
3570 assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&
3571 "Non-byte-multiple bit width");
3572 // Move the insertion point just past the load so that we can refer to it.
3573 BasicBlock::iterator LIIt = std::next(LI.getIterator());
3574 // Ensure the insertion point comes before any debug-info immediately
3575 // after the load, so that variable values referring to the load are
3576 // dominated by it.
3577 LIIt.setHeadBit(true);
3578 IRB.SetInsertPoint(LI.getParent(), LIIt);
3579 // Create a placeholder value with the same type as LI to use as the
3580 // basis for the new value. This allows us to replace the uses of LI with
3581 // the computed value, and then replace the placeholder with LI, leaving
3582 // LI only used for this computation.
3583 Value *Placeholder =
3584 new LoadInst(LI.getType(), PoisonValue::get(IRB.getPtrTy(AS)), "",
3585 false, Align(1));
3586 V = insertInteger(DL, IRB, Placeholder, V, NewBeginOffset - BeginOffset,
3587 "insert");
3588 LI.replaceAllUsesWith(V);
3589 Placeholder->replaceAllUsesWith(&LI);
3590 Placeholder->deleteValue();
3591 } else {
3592 LI.replaceAllUsesWith(V);
3593 }
3594
3595 Pass.DeadInsts.push_back(&LI);
3596 deleteIfTriviallyDead(OldOp);
3597 LLVM_DEBUG(dbgs() << " to: " << *V << "\n");
3598 return !LI.isVolatile() && !IsPtrAdjusted;
3599 }
3600
3601 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
3602 AAMDNodes AATags) {
3603 // Capture V for the purpose of debug-info accounting once it's converted
3604 // to a vector store.
3605 Value *OrigV = V;
3606 if (V->getType() != VecTy) {
3607 unsigned BeginIndex = getIndex(NewBeginOffset);
3608 unsigned EndIndex = getIndex(NewEndOffset);
3609 assert(EndIndex > BeginIndex && "Empty vector!");
3610 unsigned NumElements = EndIndex - BeginIndex;
3611 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3612 "Too many elements!");
3613 Type *SliceTy = (NumElements == 1)
3614 ? ElementTy
3615 : FixedVectorType::get(ElementTy, NumElements);
3616 if (V->getType() != SliceTy)
3617 V = IRB.CreateBitPreservingCastChain(DL, V, SliceTy);
3618
3619 // Mix in the existing elements.
3620 Value *Old =
3621 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
3622 V = insertVector(IRB, Old, V, BeginIndex, "vec");
3623 }
3624 StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
3625 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
3626 LLVMContext::MD_access_group});
3627 if (AATags)
3628 Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
3629 V->getType(), DL));
3630 Pass.DeadInsts.push_back(&SI);
3631
3632 // NOTE: Careful to use OrigV rather than V.
3633 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
3634 Store, Store->getPointerOperand(), OrigV, DL);
3635 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3636 return true;
3637 }
3638
3639 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
3640 assert(IntTy && "We cannot extract an integer from the alloca");
3641 assert(!SI.isVolatile());
3642 if (DL.getTypeSizeInBits(V->getType()).getFixedValue() !=
3643 IntTy->getBitWidth()) {
3644 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(),
3645 "oldload");
3646 Old = IRB.CreateBitPreservingCastChain(DL, Old, IntTy);
3647 assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3648 uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
3649 V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert");
3650 }
3651 V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
3652 StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
3653 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
3654 LLVMContext::MD_access_group});
3655 if (AATags)
3656 Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
3657 V->getType(), DL));
3658
3659 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
3660 Store, Store->getPointerOperand(),
3661 Store->getValueOperand(), DL);
3662
3663 Pass.DeadInsts.push_back(&SI);
3664 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3665 return true;
3666 }
3667
3668 bool visitStoreInst(StoreInst &SI) {
3669 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3670 Value *OldOp = SI.getOperand(1);
3671 assert(OldOp == OldPtr);
3672
3673 AAMDNodes AATags = SI.getAAMetadata();
3674 Value *V = SI.getValueOperand();
3675
3676 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3677 // alloca that should be re-examined after promoting this alloca.
3678 if (V->getType()->isPointerTy())
3679 if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets()))
3680 Pass.PostPromotionWorklist.insert(AI);
3681
3682 TypeSize StoreSize = DL.getTypeStoreSize(V->getType());
3683 if (StoreSize.isFixed() && SliceSize < StoreSize.getFixedValue()) {
3684 assert(!SI.isVolatile());
3685 assert(V->getType()->isIntegerTy() &&
3686 "Only integer type loads and stores are split");
3687 assert(DL.typeSizeEqualsStoreSize(V->getType()) &&
3688 "Non-byte-multiple bit width");
3689 IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8);
3690 V = extractInteger(DL, IRB, V, NarrowTy, NewBeginOffset - BeginOffset,
3691 "extract");
3692 }
3693
3694 if (VecTy)
3695 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
3696 if (IntTy && V->getType()->isIntegerTy())
3697 return rewriteIntegerStore(V, SI, AATags);
3698
3699 StoreInst *NewSI;
3700 if (NewBeginOffset == NewAllocaBeginOffset &&
3701 NewEndOffset == NewAllocaEndOffset &&
3702 canConvertValue(DL, V->getType(), NewAllocaTy)) {
3703 V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
3704 Value *NewPtr =
3705 getPtrToNewAI(SI.getPointerAddressSpace(), SI.isVolatile());
3706
3707 NewSI =
3708 IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), SI.isVolatile());
3709 } else {
3710 unsigned AS = SI.getPointerAddressSpace();
3711 Value *NewPtr = getNewAllocaSlicePtr(IRB, IRB.getPtrTy(AS));
3712 NewSI =
3713 IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(), SI.isVolatile());
3714 }
3715 NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
3716 LLVMContext::MD_access_group});
3717 if (AATags)
3718 NewSI->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
3719 V->getType(), DL));
3720 if (SI.isVolatile())
3721 NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID());
3722 if (NewSI->isAtomic())
3723 NewSI->setAlignment(SI.getAlign());
3724
3725 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
3726 NewSI, NewSI->getPointerOperand(),
3727 NewSI->getValueOperand(), DL);
3728
3729 Pass.DeadInsts.push_back(&SI);
3730 deleteIfTriviallyDead(OldOp);
3731
3732 LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n");
3733 return NewSI->getPointerOperand() == &NewAI &&
3734 NewSI->getValueOperand()->getType() == NewAllocaTy &&
3735 !SI.isVolatile();
3736 }
3737
3738 /// Compute an integer value from splatting an i8 across the given
3739 /// number of bytes.
3740 ///
3741 /// Note that this routine assumes an i8 is a byte. If that isn't true, don't
3742 /// call this routine.
3743 /// FIXME: Heed the advice above.
3744 ///
3745 /// \param V The i8 value to splat.
3746 /// \param Size The number of bytes in the output (assuming i8 is one byte)
3747 Value *getIntegerSplat(Value *V, unsigned Size) {
3748 assert(Size > 0 && "Expected a positive number of bytes.");
3749 IntegerType *VTy = cast<IntegerType>(V->getType());
3750 assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte");
3751 if (Size == 1)
3752 return V;
3753
3754 Type *SplatIntTy = Type::getIntNTy(VTy->getContext(), Size * 8);
3755 V = IRB.CreateMul(
3756 IRB.CreateZExt(V, SplatIntTy, "zext"),
3757 IRB.CreateUDiv(Constant::getAllOnesValue(SplatIntTy),
3758 IRB.CreateZExt(Constant::getAllOnesValue(V->getType()),
3759 SplatIntTy)),
3760 "isplat");
3761 return V;
3762 }
3763
3764 /// Compute a vector splat for a given element value.
3765 Value *getVectorSplat(Value *V, unsigned NumElements) {
3766 V = IRB.CreateVectorSplat(NumElements, V, "vsplat");
3767 LLVM_DEBUG(dbgs() << " splat: " << *V << "\n");
3768 return V;
3769 }
3770
3771 bool visitMemSetInst(MemSetInst &II) {
3772 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3773 assert(II.getRawDest() == OldPtr);
3774
3775 AAMDNodes AATags = II.getAAMetadata();
3776
3777 // If the memset has a variable size, it cannot be split, just adjust the
3778 // pointer to the new alloca.
3779 if (!isa<ConstantInt>(II.getLength())) {
3780 assert(!IsSplit);
3781 assert(NewBeginOffset == BeginOffset);
3782 II.setDest(getNewAllocaSlicePtr(IRB, OldPtr->getType()));
3783 II.setDestAlignment(getSliceAlign());
3784 // In theory we should call migrateDebugInfo here. However, we do not
3785 // emit dbg.assign intrinsics for mem intrinsics storing through non-
3786 // constant geps, or storing a variable number of bytes.
3788 "AT: Unexpected link to non-const GEP");
3789 deleteIfTriviallyDead(OldPtr);
3790 return false;
3791 }
3792
3793 // Record this instruction for deletion.
3794 Pass.DeadInsts.push_back(&II);
3795
3796 Type *ScalarTy = NewAllocaTy->getScalarType();
3797
3798 const bool CanContinue = [&]() {
3799 if (VecTy || IntTy)
3800 return true;
3801 if (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset)
3802 return false;
3803 // Length must be in range for FixedVectorType.
3804 auto *C = cast<ConstantInt>(II.getLength());
3805 const uint64_t Len = C->getLimitedValue();
3806 if (Len > std::numeric_limits<unsigned>::max())
3807 return false;
3808 auto *Int8Ty = IntegerType::getInt8Ty(NewAI.getContext());
3809 auto *SrcTy = FixedVectorType::get(Int8Ty, Len);
3810 return canConvertValue(DL, SrcTy, NewAllocaTy) &&
3811 DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy).getFixedValue());
3812 }();
3813
3814 // If this doesn't map cleanly onto the alloca type, and that type isn't
3815 // a single value type, just emit a memset.
3816 if (!CanContinue) {
3817 Type *SizeTy = II.getLength()->getType();
3818 unsigned Sz = NewEndOffset - NewBeginOffset;
3819 Constant *Size = ConstantInt::get(SizeTy, Sz);
3820 MemIntrinsic *New = cast<MemIntrinsic>(IRB.CreateMemSet(
3821 getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size,
3822 MaybeAlign(getSliceAlign()), II.isVolatile()));
3823 if (AATags)
3824 New->setAAMetadata(
3825 AATags.adjustForAccess(NewBeginOffset - BeginOffset, Sz));
3826
3827 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II,
3828 New, New->getRawDest(), nullptr, DL);
3829
3830 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3831 return false;
3832 }
3833
3834 // If we can represent this as a simple value, we have to build the actual
3835 // value to store, which requires expanding the byte present in memset to
3836 // a sensible representation for the alloca type. This is essentially
3837 // splatting the byte to a sufficiently wide integer, splatting it across
3838 // any desired vector width, and bitcasting to the final type.
3839 Value *V;
3840
3841 if (VecTy) {
3842 // If this is a memset of a vectorized alloca, insert it.
3843 assert(ElementTy == ScalarTy);
3844
3845 unsigned BeginIndex = getIndex(NewBeginOffset);
3846 unsigned EndIndex = getIndex(NewEndOffset);
3847 assert(EndIndex > BeginIndex && "Empty vector!");
3848 unsigned NumElements = EndIndex - BeginIndex;
3849 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3850 "Too many elements!");
3851
3852 Value *Splat = getIntegerSplat(
3853 II.getValue(), DL.getTypeSizeInBits(ElementTy).getFixedValue() / 8);
3854 Splat = IRB.CreateBitPreservingCastChain(DL, Splat, ElementTy);
3855 if (NumElements > 1)
3856 Splat = getVectorSplat(Splat, NumElements);
3857
3858 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(),
3859 "oldload");
3860 V = insertVector(IRB, Old, Splat, BeginIndex, "vec");
3861 } else if (IntTy) {
3862 // If this is a memset on an alloca where we can widen stores, insert the
3863 // set integer.
3864 assert(!II.isVolatile());
3865
3866 uint64_t Size = NewEndOffset - NewBeginOffset;
3867 V = getIntegerSplat(II.getValue(), Size);
3868
3869 if (IntTy && (NewBeginOffset != NewAllocaBeginOffset ||
3870 NewEndOffset != NewAllocaEndOffset)) {
3871 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI,
3872 NewAI.getAlign(), "oldload");
3873 Old = IRB.CreateBitPreservingCastChain(DL, Old, IntTy);
3874 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3875 V = insertInteger(DL, IRB, Old, V, Offset, "insert");
3876 } else {
3877 assert(V->getType() == IntTy &&
3878 "Wrong type for an alloca wide integer!");
3879 }
3880 V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
3881 } else {
3882 // Established these invariants above.
3883 assert(NewBeginOffset == NewAllocaBeginOffset);
3884 assert(NewEndOffset == NewAllocaEndOffset);
3885
3886 V = getIntegerSplat(II.getValue(),
3887 DL.getTypeSizeInBits(ScalarTy).getFixedValue() / 8);
3888 if (VectorType *AllocaVecTy = dyn_cast<VectorType>(NewAllocaTy))
3889 V = getVectorSplat(
3890 V, cast<FixedVectorType>(AllocaVecTy)->getNumElements());
3891
3892 V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
3893 }
3894
3895 Value *NewPtr = getPtrToNewAI(II.getDestAddressSpace(), II.isVolatile());
3896 StoreInst *New =
3897 IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), II.isVolatile());
3898 New->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
3899 LLVMContext::MD_access_group});
3900 if (AATags)
3901 New->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
3902 V->getType(), DL));
3903
3904 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II,
3905 New, New->getPointerOperand(), V, DL);
3906
3907 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3908 return !II.isVolatile();
3909 }
3910
3911 bool visitMemTransferInst(MemTransferInst &II) {
3912 // Rewriting of memory transfer instructions can be a bit tricky. We break
3913 // them into two categories: split intrinsics and unsplit intrinsics.
3914
3915 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3916
3917 AAMDNodes AATags = II.getAAMetadata();
3918
3919 bool IsDest = &II.getRawDestUse() == OldUse;
3920 assert((IsDest && II.getRawDest() == OldPtr) ||
3921 (!IsDest && II.getRawSource() == OldPtr));
3922
3923 Align SliceAlign = getSliceAlign();
3924 // For unsplit intrinsics, we simply modify the source and destination
3925 // pointers in place. This isn't just an optimization, it is a matter of
3926 // correctness. With unsplit intrinsics we may be dealing with transfers
3927 // within a single alloca before SROA ran, or with transfers that have
3928 // a variable length. We may also be dealing with memmove instead of
3929 // memcpy, and so simply updating the pointers is the necessary for us to
3930 // update both source and dest of a single call.
3931 if (!IsSplittable) {
3932 Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
3933 if (IsDest) {
3934 // Update the address component of linked dbg.assigns.
3935 for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(&II)) {
3936 if (llvm::is_contained(DbgAssign->location_ops(), II.getDest()) ||
3937 DbgAssign->getAddress() == II.getDest())
3938 DbgAssign->replaceVariableLocationOp(II.getDest(), AdjustedPtr);
3939 }
3940 II.setDest(AdjustedPtr);
3941 II.setDestAlignment(SliceAlign);
3942 } else {
3943 II.setSource(AdjustedPtr);
3944 II.setSourceAlignment(SliceAlign);
3945 }
3946
3947 LLVM_DEBUG(dbgs() << " to: " << II << "\n");
3948 deleteIfTriviallyDead(OldPtr);
3949 return false;
3950 }
3951 // For split transfer intrinsics we have an incredibly useful assurance:
3952 // the source and destination do not reside within the same alloca, and at
3953 // least one of them does not escape. This means that we can replace
3954 // memmove with memcpy, and we don't need to worry about all manner of
3955 // downsides to splitting and transforming the operations.
3956
3957 // If this doesn't map cleanly onto the alloca type, and that type isn't
3958 // a single value type, just emit a memcpy.
3959 bool EmitMemCpy =
3960 !VecTy && !IntTy &&
3961 (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset ||
3962 SliceSize != DL.getTypeStoreSize(NewAllocaTy).getFixedValue() ||
3963 !DL.typeSizeEqualsStoreSize(NewAllocaTy) ||
3964 !NewAllocaTy->isSingleValueType());
3965
3966 // If we're just going to emit a memcpy, the alloca hasn't changed, and the
3967 // size hasn't been shrunk based on analysis of the viable range, this is
3968 // a no-op.
3969 if (EmitMemCpy && &OldAI == &NewAI) {
3970 // Ensure the start lines up.
3971 assert(NewBeginOffset == BeginOffset);
3972
3973 // Rewrite the size as needed.
3974 if (NewEndOffset != EndOffset)
3975 II.setLength(NewEndOffset - NewBeginOffset);
3976 return false;
3977 }
3978 // Record this instruction for deletion.
3979 Pass.DeadInsts.push_back(&II);
3980
3981 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3982 // alloca that should be re-examined after rewriting this instruction.
3983 Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
3984 if (AllocaInst *AI =
3986 assert(AI != &OldAI && AI != &NewAI &&
3987 "Splittable transfers cannot reach the same alloca on both ends.");
3988 Pass.Worklist.insert(AI);
3989 }
3990
3991 Type *OtherPtrTy = OtherPtr->getType();
3992 unsigned OtherAS = OtherPtrTy->getPointerAddressSpace();
3993
3994 // Compute the relative offset for the other pointer within the transfer.
3995 unsigned OffsetWidth = DL.getIndexSizeInBits(OtherAS);
3996 APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
3997 Align OtherAlign =
3998 (IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne();
3999 OtherAlign =
4000 commonAlignment(OtherAlign, OtherOffset.zextOrTrunc(64).getZExtValue());
4001
4002 if (EmitMemCpy) {
4003 // Compute the other pointer, folding as much as possible to produce
4004 // a single, simple GEP in most cases.
4005 OtherPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy,
4006 OtherPtr->getName() + ".");
4007
4008 Value *OurPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
4009 Type *SizeTy = II.getLength()->getType();
4010 Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset);
4011
4012 Value *DestPtr, *SrcPtr;
4013 MaybeAlign DestAlign, SrcAlign;
4014 // Note: IsDest is true iff we're copying into the new alloca slice
4015 if (IsDest) {
4016 DestPtr = OurPtr;
4017 DestAlign = SliceAlign;
4018 SrcPtr = OtherPtr;
4019 SrcAlign = OtherAlign;
4020 } else {
4021 DestPtr = OtherPtr;
4022 DestAlign = OtherAlign;
4023 SrcPtr = OurPtr;
4024 SrcAlign = SliceAlign;
4025 }
4026 CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign,
4027 Size, II.isVolatile());
4028 if (AATags)
4029 New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
4030
4031 APInt Offset(DL.getIndexTypeSizeInBits(DestPtr->getType()), 0);
4032 if (IsDest) {
4033 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8,
4034 &II, New, DestPtr, nullptr, DL);
4035 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4037 DL, Offset, /*AllowNonInbounds*/ true))) {
4038 migrateDebugInfo(Base, IsSplit, Offset.getZExtValue() * 8,
4039 SliceSize * 8, &II, New, DestPtr, nullptr, DL);
4040 }
4041 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4042 return false;
4043 }
4044
4045 bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset &&
4046 NewEndOffset == NewAllocaEndOffset;
4047 uint64_t Size = NewEndOffset - NewBeginOffset;
4048 unsigned BeginIndex = VecTy ? getIndex(NewBeginOffset) : 0;
4049 unsigned EndIndex = VecTy ? getIndex(NewEndOffset) : 0;
4050 unsigned NumElements = EndIndex - BeginIndex;
4051 IntegerType *SubIntTy =
4052 IntTy ? Type::getIntNTy(IntTy->getContext(), Size * 8) : nullptr;
4053
4054 // Reset the other pointer type to match the register type we're going to
4055 // use, but using the address space of the original other pointer.
4056 Type *OtherTy;
4057 if (VecTy && !IsWholeAlloca) {
4058 if (NumElements == 1)
4059 OtherTy = VecTy->getElementType();
4060 else
4061 OtherTy = FixedVectorType::get(VecTy->getElementType(), NumElements);
4062 } else if (IntTy && !IsWholeAlloca) {
4063 OtherTy = SubIntTy;
4064 } else {
4065 OtherTy = NewAllocaTy;
4066 }
4067
4068 Value *AdjPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy,
4069 OtherPtr->getName() + ".");
4070 MaybeAlign SrcAlign = OtherAlign;
4071 MaybeAlign DstAlign = SliceAlign;
4072 if (!IsDest)
4073 std::swap(SrcAlign, DstAlign);
4074
4075 Value *SrcPtr;
4076 Value *DstPtr;
4077
4078 if (IsDest) {
4079 DstPtr = getPtrToNewAI(II.getDestAddressSpace(), II.isVolatile());
4080 SrcPtr = AdjPtr;
4081 } else {
4082 DstPtr = AdjPtr;
4083 SrcPtr = getPtrToNewAI(II.getSourceAddressSpace(), II.isVolatile());
4084 }
4085
4086 Value *Src;
4087 if (VecTy && !IsWholeAlloca && !IsDest) {
4088 Src =
4089 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
4090 Src = extractVector(IRB, Src, BeginIndex, EndIndex, "vec");
4091 } else if (IntTy && !IsWholeAlloca && !IsDest) {
4092 Src =
4093 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
4094 Src = IRB.CreateBitPreservingCastChain(DL, Src, IntTy);
4095 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4096 Src = extractInteger(DL, IRB, Src, SubIntTy, Offset, "extract");
4097 } else {
4098 LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign,
4099 II.isVolatile(), "copyload");
4100 Load->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
4101 LLVMContext::MD_access_group});
4102 if (AATags)
4103 Load->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
4104 Load->getType(), DL));
4105 Src = Load;
4106 }
4107
4108 if (VecTy && !IsWholeAlloca && IsDest) {
4109 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(),
4110 "oldload");
4111 Src = insertVector(IRB, Old, Src, BeginIndex, "vec");
4112 } else if (IntTy && !IsWholeAlloca && IsDest) {
4113 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(),
4114 "oldload");
4115 Old = IRB.CreateBitPreservingCastChain(DL, Old, IntTy);
4116 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4117 Src = insertInteger(DL, IRB, Old, Src, Offset, "insert");
4118 Src = IRB.CreateBitPreservingCastChain(DL, Src, NewAllocaTy);
4119 }
4120
4121 StoreInst *Store = cast<StoreInst>(
4122 IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile()));
4123 Store->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
4124 LLVMContext::MD_access_group});
4125 if (AATags)
4126 Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
4127 Src->getType(), DL));
4128
4129 APInt Offset(DL.getIndexTypeSizeInBits(DstPtr->getType()), 0);
4130 if (IsDest) {
4131
4132 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II,
4133 Store, DstPtr, Src, DL);
4134 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4136 DL, Offset, /*AllowNonInbounds*/ true))) {
4137 migrateDebugInfo(Base, IsSplit, Offset.getZExtValue() * 8, SliceSize * 8,
4138 &II, Store, DstPtr, Src, DL);
4139 }
4140
4141 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4142 return !II.isVolatile();
4143 }
4144
4145 bool visitIntrinsicInst(IntrinsicInst &II) {
4146 assert((II.isLifetimeStartOrEnd() || II.isDroppable()) &&
4147 "Unexpected intrinsic!");
4148 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
4149
4150 // Record this instruction for deletion.
4151 Pass.DeadInsts.push_back(&II);
4152
4153 if (II.isDroppable()) {
4154 assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
4155 // TODO For now we forget assumed information, this can be improved.
4156 OldPtr->dropDroppableUsesIn(II);
4157 return true;
4158 }
4159
4160 assert(II.getArgOperand(0) == OldPtr);
4161 Type *PointerTy = IRB.getPtrTy(OldPtr->getType()->getPointerAddressSpace());
4162 Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
4163 Value *New;
4164 if (II.getIntrinsicID() == Intrinsic::lifetime_start)
4165 New = IRB.CreateLifetimeStart(Ptr);
4166 else
4167 New = IRB.CreateLifetimeEnd(Ptr);
4168
4169 (void)New;
4170 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4171
4172 return true;
4173 }
4174
4175 void fixLoadStoreAlign(Instruction &Root) {
4176 // This algorithm implements the same visitor loop as
4177 // hasUnsafePHIOrSelectUse, and fixes the alignment of each load
4178 // or store found.
4179 SmallPtrSet<Instruction *, 4> Visited;
4180 SmallVector<Instruction *, 4> Uses;
4181 Visited.insert(&Root);
4182 Uses.push_back(&Root);
4183 do {
4184 Instruction *I = Uses.pop_back_val();
4185
4186 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
4187 LI->setAlignment(std::min(LI->getAlign(), getSliceAlign()));
4188 continue;
4189 }
4190 if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
4191 SI->setAlignment(std::min(SI->getAlign(), getSliceAlign()));
4192 continue;
4193 }
4194
4198 for (User *U : I->users())
4199 if (Visited.insert(cast<Instruction>(U)).second)
4200 Uses.push_back(cast<Instruction>(U));
4201 } while (!Uses.empty());
4202 }
4203
4204 bool visitPHINode(PHINode &PN) {
4205 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
4206 assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable");
4207 assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable");
4208
4209 // We would like to compute a new pointer in only one place, but have it be
4210 // as local as possible to the PHI. To do that, we re-use the location of
4211 // the old pointer, which necessarily must be in the right position to
4212 // dominate the PHI.
4213 IRBuilderBase::InsertPointGuard Guard(IRB);
4214 if (isa<PHINode>(OldPtr))
4215 IRB.SetInsertPoint(OldPtr->getParent(),
4216 OldPtr->getParent()->getFirstInsertionPt());
4217 else
4218 IRB.SetInsertPoint(OldPtr);
4219 IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc());
4220
4221 Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
4222 // Replace the operands which were using the old pointer.
4223 std::replace(PN.op_begin(), PN.op_end(), cast<Value>(OldPtr), NewPtr);
4224
4225 LLVM_DEBUG(dbgs() << " to: " << PN << "\n");
4226 deleteIfTriviallyDead(OldPtr);
4227
4228 // Fix the alignment of any loads or stores using this PHI node.
4229 fixLoadStoreAlign(PN);
4230
4231 // PHIs can't be promoted on their own, but often can be speculated. We
4232 // check the speculation outside of the rewriter so that we see the
4233 // fully-rewritten alloca.
4234 PHIUsers.insert(&PN);
4235 return true;
4236 }
4237
4238 bool visitSelectInst(SelectInst &SI) {
4239 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4240 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&
4241 "Pointer isn't an operand!");
4242 assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable");
4243 assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable");
4244
4245 Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
4246 // Replace the operands which were using the old pointer.
4247 if (SI.getOperand(1) == OldPtr)
4248 SI.setOperand(1, NewPtr);
4249 if (SI.getOperand(2) == OldPtr)
4250 SI.setOperand(2, NewPtr);
4251
4252 LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
4253 deleteIfTriviallyDead(OldPtr);
4254
4255 // Fix the alignment of any loads or stores using this select.
4256 fixLoadStoreAlign(SI);
4257
4258 // Selects can't be promoted on their own, but often can be speculated. We
4259 // check the speculation outside of the rewriter so that we see the
4260 // fully-rewritten alloca.
4261 SelectUsers.insert(&SI);
4262 return true;
4263 }
4264};
4265
4266/// Visitor to rewrite aggregate loads and stores as scalar.
4267///
4268/// This pass aggressively rewrites all aggregate loads and stores on
4269/// a particular pointer (or any pointer derived from it which we can identify)
4270/// with scalar loads and stores.
4271class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
4272 // Befriend the base class so it can delegate to private visit methods.
4273 friend class InstVisitor<AggLoadStoreRewriter, bool>;
4274
4275 /// Queue of pointer uses to analyze and potentially rewrite.
4277
4278 /// Set to prevent us from cycling with phi nodes and loops.
4279 SmallPtrSet<User *, 8> Visited;
4280
4281 /// The current pointer use being rewritten. This is used to dig up the used
4282 /// value (as opposed to the user).
4283 Use *U = nullptr;
4284
4285 /// Used to calculate offsets, and hence alignment, of subobjects.
4286 const DataLayout &DL;
4287
4288 IRBuilderTy &IRB;
4289
4290public:
4291 AggLoadStoreRewriter(const DataLayout &DL, IRBuilderTy &IRB)
4292 : DL(DL), IRB(IRB) {}
4293
4294 /// Rewrite loads and stores through a pointer and all pointers derived from
4295 /// it.
4296 bool rewrite(Instruction &I) {
4297 LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n");
4298 enqueueUsers(I);
4299 bool Changed = false;
4300 while (!Queue.empty()) {
4301 U = Queue.pop_back_val();
4302 Changed |= visit(cast<Instruction>(U->getUser()));
4303 }
4304 return Changed;
4305 }
4306
4307private:
4308 /// Enqueue all the users of the given instruction for further processing.
4309 /// This uses a set to de-duplicate users.
4310 void enqueueUsers(Instruction &I) {
4311 for (Use &U : I.uses())
4312 if (Visited.insert(U.getUser()).second)
4313 Queue.push_back(&U);
4314 }
4315
4316 // Conservative default is to not rewrite anything.
4317 bool visitInstruction(Instruction &I) { return false; }
4318
4319 /// Generic recursive split emission class.
4320 template <typename Derived> class OpSplitter {
4321 protected:
4322 /// The builder used to form new instructions.
4323 IRBuilderTy &IRB;
4324
4325 /// The indices which to be used with insert- or extractvalue to select the
4326 /// appropriate value within the aggregate.
4327 SmallVector<unsigned, 4> Indices;
4328
4329 /// The indices to a GEP instruction which will move Ptr to the correct slot
4330 /// within the aggregate.
4331 SmallVector<Value *, 4> GEPIndices;
4332
4333 /// The base pointer of the original op, used as a base for GEPing the
4334 /// split operations.
4335 Value *Ptr;
4336
4337 /// The base pointee type being GEPed into.
4338 Type *BaseTy;
4339
4340 /// Known alignment of the base pointer.
4341 Align BaseAlign;
4342
4343 /// To calculate offset of each component so we can correctly deduce
4344 /// alignments.
4345 const DataLayout &DL;
4346
4347 /// Initialize the splitter with an insertion point, Ptr and start with a
4348 /// single zero GEP index.
4349 OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4350 Align BaseAlign, const DataLayout &DL, IRBuilderTy &IRB)
4351 : IRB(IRB), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr), BaseTy(BaseTy),
4352 BaseAlign(BaseAlign), DL(DL) {
4353 IRB.SetInsertPoint(InsertionPoint);
4354 }
4355
4356 public:
4357 /// Generic recursive split emission routine.
4358 ///
4359 /// This method recursively splits an aggregate op (load or store) into
4360 /// scalar or vector ops. It splits recursively until it hits a single value
4361 /// and emits that single value operation via the template argument.
4362 ///
4363 /// The logic of this routine relies on GEPs and insertvalue and
4364 /// extractvalue all operating with the same fundamental index list, merely
4365 /// formatted differently (GEPs need actual values).
4366 ///
4367 /// \param Ty The type being split recursively into smaller ops.
4368 /// \param Agg The aggregate value being built up or stored, depending on
4369 /// whether this is splitting a load or a store respectively.
4370 void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) {
4371 if (Ty->isSingleValueType()) {
4372 unsigned Offset = DL.getIndexedOffsetInType(BaseTy, GEPIndices);
4373 return static_cast<Derived *>(this)->emitFunc(
4374 Ty, Agg, commonAlignment(BaseAlign, Offset), Name);
4375 }
4376
4377 if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
4378 unsigned OldSize = Indices.size();
4379 (void)OldSize;
4380 for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size;
4381 ++Idx) {
4382 assert(Indices.size() == OldSize && "Did not return to the old size");
4383 Indices.push_back(Idx);
4384 GEPIndices.push_back(IRB.getInt32(Idx));
4385 emitSplitOps(ATy->getElementType(), Agg, Name + "." + Twine(Idx));
4386 GEPIndices.pop_back();
4387 Indices.pop_back();
4388 }
4389 return;
4390 }
4391
4392 if (StructType *STy = dyn_cast<StructType>(Ty)) {
4393 unsigned OldSize = Indices.size();
4394 (void)OldSize;
4395 for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size;
4396 ++Idx) {
4397 assert(Indices.size() == OldSize && "Did not return to the old size");
4398 Indices.push_back(Idx);
4399 GEPIndices.push_back(IRB.getInt32(Idx));
4400 emitSplitOps(STy->getElementType(Idx), Agg, Name + "." + Twine(Idx));
4401 GEPIndices.pop_back();
4402 Indices.pop_back();
4403 }
4404 return;
4405 }
4406
4407 llvm_unreachable("Only arrays and structs are aggregate loadable types");
4408 }
4409 };
4410
4411 struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
4412 AAMDNodes AATags;
4413 // A vector to hold the split components that we want to emit
4414 // separate fake uses for.
4415 SmallVector<Value *, 4> Components;
4416 // A vector to hold all the fake uses of the struct that we are splitting.
4417 // Usually there should only be one, but we are handling the general case.
4419
4420 LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4421 AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
4422 IRBuilderTy &IRB)
4423 : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
4424 IRB),
4425 AATags(AATags) {}
4426
4427 /// Emit a leaf load of a single value. This is called at the leaves of the
4428 /// recursive emission to actually load values.
4429 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4431 // Load the single value and insert it using the indices.
4432 Value *GEP =
4433 IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
4434 LoadInst *Load =
4435 IRB.CreateAlignedLoad(Ty, GEP, Alignment, Name + ".load");
4436
4437 APInt Offset(
4438 DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
4439 if (AATags &&
4440 GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset))
4441 Load->setAAMetadata(
4442 AATags.adjustForAccess(Offset.getZExtValue(), Load->getType(), DL));
4443 // Record the load so we can generate a fake use for this aggregate
4444 // component.
4445 Components.push_back(Load);
4446
4447 Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert");
4448 LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
4449 }
4450
4451 // Stash the fake uses that use the value generated by this instruction.
4452 void recordFakeUses(LoadInst &LI) {
4453 for (Use &U : LI.uses())
4454 if (auto *II = dyn_cast<IntrinsicInst>(U.getUser()))
4455 if (II->getIntrinsicID() == Intrinsic::fake_use)
4456 FakeUses.push_back(II);
4457 }
4458
4459 // Replace all fake uses of the aggregate with a series of fake uses, one
4460 // for each split component.
4461 void emitFakeUses() {
4462 for (Instruction *I : FakeUses) {
4463 IRB.SetInsertPoint(I);
4464 for (auto *V : Components)
4465 IRB.CreateIntrinsic(Intrinsic::fake_use, {V});
4466 I->eraseFromParent();
4467 }
4468 }
4469 };
4470
4471 bool visitLoadInst(LoadInst &LI) {
4472 assert(LI.getPointerOperand() == *U);
4473 if (!LI.isSimple() || LI.getType()->isSingleValueType())
4474 return false;
4475
4476 // We have an aggregate being loaded, split it apart.
4477 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
4478 LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
4479 getAdjustedAlignment(&LI, 0), DL, IRB);
4480 Splitter.recordFakeUses(LI);
4482 Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca");
4483 Splitter.emitFakeUses();
4484 Visited.erase(&LI);
4485 LI.replaceAllUsesWith(V);
4486 LI.eraseFromParent();
4487 return true;
4488 }
4489
4490 struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
4491 StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4492 AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
4493 const DataLayout &DL, IRBuilderTy &IRB)
4494 : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
4495 DL, IRB),
4496 AATags(AATags), AggStore(AggStore) {}
4497 AAMDNodes AATags;
4498 StoreInst *AggStore;
4499 /// Emit a leaf store of a single value. This is called at the leaves of the
4500 /// recursive emission to actually produce stores.
4501 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4503 // Extract the single value and store it using the indices.
4504 //
4505 // The gep and extractvalue values are factored out of the CreateStore
4506 // call to make the output independent of the argument evaluation order.
4507 Value *ExtractValue =
4508 IRB.CreateExtractValue(Agg, Indices, Name + ".extract");
4509 Value *InBoundsGEP =
4510 IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
4511 StoreInst *Store =
4512 IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Alignment);
4513
4514 APInt Offset(
4515 DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
4516 GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset);
4517 if (AATags) {
4518 Store->setAAMetadata(AATags.adjustForAccess(
4519 Offset.getZExtValue(), ExtractValue->getType(), DL));
4520 }
4521
4522 // migrateDebugInfo requires the base Alloca. Walk to it from this gep.
4523 // If we cannot (because there's an intervening non-const or unbounded
4524 // gep) then we wouldn't expect to see dbg.assign intrinsics linked to
4525 // this instruction.
4527 if (auto *OldAI = dyn_cast<AllocaInst>(Base)) {
4528 uint64_t SizeInBits =
4529 DL.getTypeSizeInBits(Store->getValueOperand()->getType());
4530 migrateDebugInfo(OldAI, /*IsSplit*/ true, Offset.getZExtValue() * 8,
4531 SizeInBits, AggStore, Store,
4532 Store->getPointerOperand(), Store->getValueOperand(),
4533 DL);
4534 } else {
4536 "AT: unexpected debug.assign linked to store through "
4537 "unbounded GEP");
4538 }
4539 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4540 }
4541 };
4542
4543 bool visitStoreInst(StoreInst &SI) {
4544 if (!SI.isSimple() || SI.getPointerOperand() != *U)
4545 return false;
4546 Value *V = SI.getValueOperand();
4547 if (V->getType()->isSingleValueType())
4548 return false;
4549
4550 // We have an aggregate being stored, split it apart.
4551 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4552 StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
4553 getAdjustedAlignment(&SI, 0), DL, IRB);
4554 Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca");
4555 Visited.erase(&SI);
4556 // The stores replacing SI each have markers describing fragments of the
4557 // assignment so delete the assignment markers linked to SI.
4559 SI.eraseFromParent();
4560 return true;
4561 }
4562
4563 bool visitBitCastInst(BitCastInst &BC) {
4564 enqueueUsers(BC);
4565 return false;
4566 }
4567
4568 bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
4569 enqueueUsers(ASC);
4570 return false;
4571 }
4572
4573 // Unfold gep (select cond, ptr1, ptr2), idx
4574 // => select cond, gep(ptr1, idx), gep(ptr2, idx)
4575 // and gep ptr, (select cond, idx1, idx2)
4576 // => select cond, gep(ptr, idx1), gep(ptr, idx2)
4577 // We also allow for i1 zext indices, which are equivalent to selects.
4578 bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
4579 // Check whether the GEP has exactly one select operand and all indices
4580 // will become constant after the transform.
4582 for (Value *Op : GEPI.indices()) {
4583 if (auto *SI = dyn_cast<SelectInst>(Op)) {
4584 if (Sel)
4585 return false;
4586
4587 Sel = SI;
4588 if (!isa<ConstantInt>(SI->getTrueValue()) ||
4589 !isa<ConstantInt>(SI->getFalseValue()))
4590 return false;
4591 continue;
4592 }
4593 if (auto *ZI = dyn_cast<ZExtInst>(Op)) {
4594 if (Sel)
4595 return false;
4596 Sel = ZI;
4597 if (!ZI->getSrcTy()->isIntegerTy(1))
4598 return false;
4599 continue;
4600 }
4601
4602 if (!isa<ConstantInt>(Op))
4603 return false;
4604 }
4605
4606 if (!Sel)
4607 return false;
4608
4609 LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):\n";
4610 dbgs() << " original: " << *Sel << "\n";
4611 dbgs() << " " << GEPI << "\n";);
4612
4613 auto GetNewOps = [&](Value *SelOp) {
4614 SmallVector<Value *> NewOps;
4615 for (Value *Op : GEPI.operands())
4616 if (Op == Sel)
4617 NewOps.push_back(SelOp);
4618 else
4619 NewOps.push_back(Op);
4620 return NewOps;
4621 };
4622
4623 Value *Cond, *True, *False;
4624 Instruction *MDFrom = nullptr;
4625 if (auto *SI = dyn_cast<SelectInst>(Sel)) {
4626 Cond = SI->getCondition();
4627 True = SI->getTrueValue();
4628 False = SI->getFalseValue();
4629 MDFrom = SI;
4630 } else {
4631 Cond = Sel->getOperand(0);
4632 True = ConstantInt::get(Sel->getType(), 1);
4633 False = ConstantInt::get(Sel->getType(), 0);
4634 }
4635 SmallVector<Value *> TrueOps = GetNewOps(True);
4636 SmallVector<Value *> FalseOps = GetNewOps(False);
4637
4638 IRB.SetInsertPoint(&GEPI);
4639 GEPNoWrapFlags NW = GEPI.getNoWrapFlags();
4640
4641 Type *Ty = GEPI.getSourceElementType();
4642 Value *NTrue = IRB.CreateGEP(Ty, TrueOps[0], ArrayRef(TrueOps).drop_front(),
4643 True->getName() + ".sroa.gep", NW);
4644
4645 Value *NFalse =
4646 IRB.CreateGEP(Ty, FalseOps[0], ArrayRef(FalseOps).drop_front(),
4647 False->getName() + ".sroa.gep", NW);
4648
4649 Value *NSel = MDFrom
4650 ? IRB.CreateSelect(Cond, NTrue, NFalse,
4651 Sel->getName() + ".sroa.sel", MDFrom)
4652 : IRB.CreateSelectWithUnknownProfile(
4653 Cond, NTrue, NFalse, DEBUG_TYPE,
4654 Sel->getName() + ".sroa.sel");
4655 Visited.erase(&GEPI);
4656 GEPI.replaceAllUsesWith(NSel);
4657 GEPI.eraseFromParent();
4658 Instruction *NSelI = cast<Instruction>(NSel);
4659 Visited.insert(NSelI);
4660 enqueueUsers(*NSelI);
4661
4662 LLVM_DEBUG(dbgs() << " to: " << *NTrue << "\n";
4663 dbgs() << " " << *NFalse << "\n";
4664 dbgs() << " " << *NSel << "\n";);
4665
4666 return true;
4667 }
4668
4669 // Unfold gep (phi ptr1, ptr2), idx
4670 // => phi ((gep ptr1, idx), (gep ptr2, idx))
4671 // and gep ptr, (phi idx1, idx2)
4672 // => phi ((gep ptr, idx1), (gep ptr, idx2))
4673 bool unfoldGEPPhi(GetElementPtrInst &GEPI) {
4674 // To prevent infinitely expanding recursive phis, bail if the GEP pointer
4675 // operand (looking through the phi if it is the phi we want to unfold) is
4676 // an instruction besides a static alloca.
4677 PHINode *Phi = dyn_cast<PHINode>(GEPI.getPointerOperand());
4678 auto IsInvalidPointerOperand = [](Value *V) {
4679 if (!isa<Instruction>(V))
4680 return false;
4681 if (auto *AI = dyn_cast<AllocaInst>(V))
4682 return !AI->isStaticAlloca();
4683 return true;
4684 };
4685 if (Phi) {
4686 if (any_of(Phi->operands(), IsInvalidPointerOperand))
4687 return false;
4688 } else {
4689 if (IsInvalidPointerOperand(GEPI.getPointerOperand()))
4690 return false;
4691 }
4692 // Check whether the GEP has exactly one phi operand (including the pointer
4693 // operand) and all indices will become constant after the transform.
4694 for (Value *Op : GEPI.indices()) {
4695 if (auto *SI = dyn_cast<PHINode>(Op)) {
4696 if (Phi)
4697 return false;
4698
4699 Phi = SI;
4700 if (!all_of(Phi->incoming_values(),
4701 [](Value *V) { return isa<ConstantInt>(V); }))
4702 return false;
4703 continue;
4704 }
4705
4706 if (!isa<ConstantInt>(Op))
4707 return false;
4708 }
4709
4710 if (!Phi)
4711 return false;
4712
4713 LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):\n";
4714 dbgs() << " original: " << *Phi << "\n";
4715 dbgs() << " " << GEPI << "\n";);
4716
4717 auto GetNewOps = [&](Value *PhiOp) {
4718 SmallVector<Value *> NewOps;
4719 for (Value *Op : GEPI.operands())
4720 if (Op == Phi)
4721 NewOps.push_back(PhiOp);
4722 else
4723 NewOps.push_back(Op);
4724 return NewOps;
4725 };
4726
4727 IRB.SetInsertPoint(Phi);
4728 PHINode *NewPhi = IRB.CreatePHI(GEPI.getType(), Phi->getNumIncomingValues(),
4729 Phi->getName() + ".sroa.phi");
4730
4731 Type *SourceTy = GEPI.getSourceElementType();
4732 // We only handle arguments, constants, and static allocas here, so we can
4733 // insert GEPs at the end of the entry block.
4734 IRB.SetInsertPoint(GEPI.getFunction()->getEntryBlock().getTerminator());
4735 for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
4736 Value *Op = Phi->getIncomingValue(I);
4737 BasicBlock *BB = Phi->getIncomingBlock(I);
4738 Value *NewGEP;
4739 if (int NI = NewPhi->getBasicBlockIndex(BB); NI >= 0) {
4740 NewGEP = NewPhi->getIncomingValue(NI);
4741 } else {
4742 SmallVector<Value *> NewOps = GetNewOps(Op);
4743 NewGEP =
4744 IRB.CreateGEP(SourceTy, NewOps[0], ArrayRef(NewOps).drop_front(),
4745 Phi->getName() + ".sroa.gep", GEPI.getNoWrapFlags());
4746 }
4747 NewPhi->addIncoming(NewGEP, BB);
4748 }
4749
4750 Visited.erase(&GEPI);
4751 GEPI.replaceAllUsesWith(NewPhi);
4752 GEPI.eraseFromParent();
4753 Visited.insert(NewPhi);
4754 enqueueUsers(*NewPhi);
4755
4756 LLVM_DEBUG(dbgs() << " to: ";
4757 for (Value *In
4758 : NewPhi->incoming_values()) dbgs()
4759 << "\n " << *In;
4760 dbgs() << "\n " << *NewPhi << '\n');
4761
4762 return true;
4763 }
4764
4765 bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
4766 if (unfoldGEPSelect(GEPI))
4767 return true;
4768
4769 if (unfoldGEPPhi(GEPI))
4770 return true;
4771
4772 enqueueUsers(GEPI);
4773 return false;
4774 }
4775
4776 bool visitPHINode(PHINode &PN) {
4777 enqueueUsers(PN);
4778 return false;
4779 }
4780
4781 bool visitSelectInst(SelectInst &SI) {
4782 enqueueUsers(SI);
4783 return false;
4784 }
4785};
4786
4787} // end anonymous namespace
4788
4789/// Strip aggregate type wrapping.
4790///
4791/// This removes no-op aggregate types wrapping an underlying type. It will
4792/// strip as many layers of types as it can without changing either the type
4793/// size or the allocated size.
4795 if (Ty->isSingleValueType())
4796 return Ty;
4797
4798 uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedValue();
4799 uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
4800
4801 Type *InnerTy;
4802 if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) {
4803 InnerTy = ArrTy->getElementType();
4804 } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
4805 const StructLayout *SL = DL.getStructLayout(STy);
4806 unsigned Index = SL->getElementContainingOffset(0);
4807 InnerTy = STy->getElementType(Index);
4808 } else {
4809 return Ty;
4810 }
4811
4812 if (AllocSize > DL.getTypeAllocSize(InnerTy).getFixedValue() ||
4813 TypeSize > DL.getTypeSizeInBits(InnerTy).getFixedValue())
4814 return Ty;
4815
4816 return stripAggregateTypeWrapping(DL, InnerTy);
4817}
4818
4819/// Try to find a partition of the aggregate type passed in for a given
4820/// offset and size.
4821///
4822/// This recurses through the aggregate type and tries to compute a subtype
4823/// based on the offset and size. When the offset and size span a sub-section
4824/// of an array, it will even compute a new array type for that sub-section,
4825/// and the same for structs.
4826///
4827/// Note that this routine is very strict and tries to find a partition of the
4828/// type which produces the *exact* right offset and size. It is not forgiving
4829/// when the size or offset cause either end of type-based partition to be off.
4830/// Also, this is a best-effort routine. It is reasonable to give up and not
4831/// return a type if necessary.
4833 uint64_t Size) {
4834 if (Offset == 0 && DL.getTypeAllocSize(Ty).getFixedValue() == Size)
4835 return stripAggregateTypeWrapping(DL, Ty);
4836 if (Offset > DL.getTypeAllocSize(Ty).getFixedValue() ||
4837 (DL.getTypeAllocSize(Ty).getFixedValue() - Offset) < Size)
4838 return nullptr;
4839
4840 if (isa<ArrayType>(Ty) || isa<VectorType>(Ty)) {
4841 Type *ElementTy;
4842 uint64_t TyNumElements;
4843 if (auto *AT = dyn_cast<ArrayType>(Ty)) {
4844 ElementTy = AT->getElementType();
4845 TyNumElements = AT->getNumElements();
4846 } else {
4847 // FIXME: This isn't right for vectors with non-byte-sized or
4848 // non-power-of-two sized elements.
4849 auto *VT = cast<FixedVectorType>(Ty);
4850 ElementTy = VT->getElementType();
4851 TyNumElements = VT->getNumElements();
4852 }
4853 uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedValue();
4854 uint64_t NumSkippedElements = Offset / ElementSize;
4855 if (NumSkippedElements >= TyNumElements)
4856 return nullptr;
4857 Offset -= NumSkippedElements * ElementSize;
4858
4859 // First check if we need to recurse.
4860 if (Offset > 0 || Size < ElementSize) {
4861 // Bail if the partition ends in a different array element.
4862 if ((Offset + Size) > ElementSize)
4863 return nullptr;
4864 // Recurse through the element type trying to peel off offset bytes.
4865 return getTypePartition(DL, ElementTy, Offset, Size);
4866 }
4867 assert(Offset == 0);
4868
4869 if (Size == ElementSize)
4870 return stripAggregateTypeWrapping(DL, ElementTy);
4871 assert(Size > ElementSize);
4872 uint64_t NumElements = Size / ElementSize;
4873 if (NumElements * ElementSize != Size)
4874 return nullptr;
4875 return ArrayType::get(ElementTy, NumElements);
4876 }
4877
4879 if (!STy)
4880 return nullptr;
4881
4882 const StructLayout *SL = DL.getStructLayout(STy);
4883
4884 if (SL->getSizeInBits().isScalable())
4885 return nullptr;
4886
4887 if (Offset >= SL->getSizeInBytes())
4888 return nullptr;
4889 uint64_t EndOffset = Offset + Size;
4890 if (EndOffset > SL->getSizeInBytes())
4891 return nullptr;
4892
4893 unsigned Index = SL->getElementContainingOffset(Offset);
4894 Offset -= SL->getElementOffset(Index);
4895
4896 Type *ElementTy = STy->getElementType(Index);
4897 uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedValue();
4898 if (Offset >= ElementSize)
4899 return nullptr; // The offset points into alignment padding.
4900
4901 // See if any partition must be contained by the element.
4902 if (Offset > 0 || Size < ElementSize) {
4903 if ((Offset + Size) > ElementSize)
4904 return nullptr;
4905 return getTypePartition(DL, ElementTy, Offset, Size);
4906 }
4907 assert(Offset == 0);
4908
4909 if (Size == ElementSize)
4910 return stripAggregateTypeWrapping(DL, ElementTy);
4911
4912 StructType::element_iterator EI = STy->element_begin() + Index,
4913 EE = STy->element_end();
4914 if (EndOffset < SL->getSizeInBytes()) {
4915 unsigned EndIndex = SL->getElementContainingOffset(EndOffset);
4916 if (Index == EndIndex)
4917 return nullptr; // Within a single element and its padding.
4918
4919 // Don't try to form "natural" types if the elements don't line up with the
4920 // expected size.
4921 // FIXME: We could potentially recurse down through the last element in the
4922 // sub-struct to find a natural end point.
4923 if (SL->getElementOffset(EndIndex) != EndOffset)
4924 return nullptr;
4925
4926 assert(Index < EndIndex);
4927 EE = STy->element_begin() + EndIndex;
4928 }
4929
4930 // Try to build up a sub-structure.
4931 StructType *SubTy =
4932 StructType::get(STy->getContext(), ArrayRef(EI, EE), STy->isPacked());
4933 const StructLayout *SubSL = DL.getStructLayout(SubTy);
4934 if (Size != SubSL->getSizeInBytes())
4935 return nullptr; // The sub-struct doesn't have quite the size needed.
4936
4937 return SubTy;
4938}
4939
4940/// Pre-split loads and stores to simplify rewriting.
4941///
4942/// We want to break up the splittable load+store pairs as much as
4943/// possible. This is important to do as a preprocessing step, as once we
4944/// start rewriting the accesses to partitions of the alloca we lose the
4945/// necessary information to correctly split apart paired loads and stores
4946/// which both point into this alloca. The case to consider is something like
4947/// the following:
4948///
4949/// %a = alloca [12 x i8]
4950/// %gep1 = getelementptr i8, ptr %a, i32 0
4951/// %gep2 = getelementptr i8, ptr %a, i32 4
4952/// %gep3 = getelementptr i8, ptr %a, i32 8
4953/// store float 0.0, ptr %gep1
4954/// store float 1.0, ptr %gep2
4955/// %v = load i64, ptr %gep1
4956/// store i64 %v, ptr %gep2
4957/// %f1 = load float, ptr %gep2
4958/// %f2 = load float, ptr %gep3
4959///
4960/// Here we want to form 3 partitions of the alloca, each 4 bytes large, and
4961/// promote everything so we recover the 2 SSA values that should have been
4962/// there all along.
4963///
4964/// \returns true if any changes are made.
4965bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
4966 LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n");
4967
4968 // Track the loads and stores which are candidates for pre-splitting here, in
4969 // the order they first appear during the partition scan. These give stable
4970 // iteration order and a basis for tracking which loads and stores we
4971 // actually split.
4974
4975 // We need to accumulate the splits required of each load or store where we
4976 // can find them via a direct lookup. This is important to cross-check loads
4977 // and stores against each other. We also track the slice so that we can kill
4978 // all the slices that end up split.
4979 struct SplitOffsets {
4980 Slice *S;
4981 std::vector<uint64_t> Splits;
4982 };
4983 SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap;
4984
4985 // Track loads out of this alloca which cannot, for any reason, be pre-split.
4986 // This is important as we also cannot pre-split stores of those loads!
4987 // FIXME: This is all pretty gross. It means that we can be more aggressive
4988 // in pre-splitting when the load feeding the store happens to come from
4989 // a separate alloca. Put another way, the effectiveness of SROA would be
4990 // decreased by a frontend which just concatenated all of its local allocas
4991 // into one big flat alloca. But defeating such patterns is exactly the job
4992 // SROA is tasked with! Sadly, to not have this discrepancy we would have
4993 // change store pre-splitting to actually force pre-splitting of the load
4994 // that feeds it *and all stores*. That makes pre-splitting much harder, but
4995 // maybe it would make it more principled?
4996 SmallPtrSet<LoadInst *, 8> UnsplittableLoads;
4997
4998 LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
4999 for (auto &P : AS.partitions()) {
5000 for (Slice &S : P) {
5001 Instruction *I = cast<Instruction>(S.getUse()->getUser());
5002 if (!S.isSplittable() || S.endOffset() <= P.endOffset()) {
5003 // If this is a load we have to track that it can't participate in any
5004 // pre-splitting. If this is a store of a load we have to track that
5005 // that load also can't participate in any pre-splitting.
5006 if (auto *LI = dyn_cast<LoadInst>(I))
5007 UnsplittableLoads.insert(LI);
5008 else if (auto *SI = dyn_cast<StoreInst>(I))
5009 if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand()))
5010 UnsplittableLoads.insert(LI);
5011 continue;
5012 }
5013 assert(P.endOffset() > S.beginOffset() &&
5014 "Empty or backwards partition!");
5015
5016 // Determine if this is a pre-splittable slice.
5017 if (auto *LI = dyn_cast<LoadInst>(I)) {
5018 assert(!LI->isVolatile() && "Cannot split volatile loads!");
5019
5020 // The load must be used exclusively to store into other pointers for
5021 // us to be able to arbitrarily pre-split it. The stores must also be
5022 // simple to avoid changing semantics.
5023 auto IsLoadSimplyStored = [](LoadInst *LI) {
5024 for (User *LU : LI->users()) {
5025 auto *SI = dyn_cast<StoreInst>(LU);
5026 if (!SI || !SI->isSimple())
5027 return false;
5028 }
5029 return true;
5030 };
5031 if (!IsLoadSimplyStored(LI)) {
5032 UnsplittableLoads.insert(LI);
5033 continue;
5034 }
5035
5036 Loads.push_back(LI);
5037 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
5038 if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex()))
5039 // Skip stores *of* pointers. FIXME: This shouldn't even be possible!
5040 continue;
5041 auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand());
5042 if (!StoredLoad || !StoredLoad->isSimple())
5043 continue;
5044 assert(!SI->isVolatile() && "Cannot split volatile stores!");
5045
5046 Stores.push_back(SI);
5047 } else {
5048 // Other uses cannot be pre-split.
5049 continue;
5050 }
5051
5052 // Record the initial split.
5053 LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n");
5054 auto &Offsets = SplitOffsetsMap[I];
5055 assert(Offsets.Splits.empty() &&
5056 "Should not have splits the first time we see an instruction!");
5057 Offsets.S = &S;
5058 Offsets.Splits.push_back(P.endOffset() - S.beginOffset());
5059 }
5060
5061 // Now scan the already split slices, and add a split for any of them which
5062 // we're going to pre-split.
5063 for (Slice *S : P.splitSliceTails()) {
5064 auto SplitOffsetsMapI =
5065 SplitOffsetsMap.find(cast<Instruction>(S->getUse()->getUser()));
5066 if (SplitOffsetsMapI == SplitOffsetsMap.end())
5067 continue;
5068 auto &Offsets = SplitOffsetsMapI->second;
5069
5070 assert(Offsets.S == S && "Found a mismatched slice!");
5071 assert(!Offsets.Splits.empty() &&
5072 "Cannot have an empty set of splits on the second partition!");
5073 assert(Offsets.Splits.back() ==
5074 P.beginOffset() - Offsets.S->beginOffset() &&
5075 "Previous split does not end where this one begins!");
5076
5077 // Record each split. The last partition's end isn't needed as the size
5078 // of the slice dictates that.
5079 if (S->endOffset() > P.endOffset())
5080 Offsets.Splits.push_back(P.endOffset() - Offsets.S->beginOffset());
5081 }
5082 }
5083
5084 // We may have split loads where some of their stores are split stores. For
5085 // such loads and stores, we can only pre-split them if their splits exactly
5086 // match relative to their starting offset. We have to verify this prior to
5087 // any rewriting.
5088 llvm::erase_if(Stores, [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
5089 // Lookup the load we are storing in our map of split
5090 // offsets.
5091 auto *LI = cast<LoadInst>(SI->getValueOperand());
5092 // If it was completely unsplittable, then we're done,
5093 // and this store can't be pre-split.
5094 if (UnsplittableLoads.count(LI))
5095 return true;
5096
5097 auto LoadOffsetsI = SplitOffsetsMap.find(LI);
5098 if (LoadOffsetsI == SplitOffsetsMap.end())
5099 return false; // Unrelated loads are definitely safe.
5100 auto &LoadOffsets = LoadOffsetsI->second;
5101
5102 // Now lookup the store's offsets.
5103 auto &StoreOffsets = SplitOffsetsMap[SI];
5104
5105 // If the relative offsets of each split in the load and
5106 // store match exactly, then we can split them and we
5107 // don't need to remove them here.
5108 if (LoadOffsets.Splits == StoreOffsets.Splits)
5109 return false;
5110
5111 LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"
5112 << " " << *LI << "\n"
5113 << " " << *SI << "\n");
5114
5115 // We've found a store and load that we need to split
5116 // with mismatched relative splits. Just give up on them
5117 // and remove both instructions from our list of
5118 // candidates.
5119 UnsplittableLoads.insert(LI);
5120 return true;
5121 });
5122 // Now we have to go *back* through all the stores, because a later store may
5123 // have caused an earlier store's load to become unsplittable and if it is
5124 // unsplittable for the later store, then we can't rely on it being split in
5125 // the earlier store either.
5126 llvm::erase_if(Stores, [&UnsplittableLoads](StoreInst *SI) {
5127 auto *LI = cast<LoadInst>(SI->getValueOperand());
5128 return UnsplittableLoads.count(LI);
5129 });
5130 // Once we've established all the loads that can't be split for some reason,
5131 // filter any that made it into our list out.
5132 llvm::erase_if(Loads, [&UnsplittableLoads](LoadInst *LI) {
5133 return UnsplittableLoads.count(LI);
5134 });
5135
5136 // If no loads or stores are left, there is no pre-splitting to be done for
5137 // this alloca.
5138 if (Loads.empty() && Stores.empty())
5139 return false;
5140
5141 // From here on, we can't fail and will be building new accesses, so rig up
5142 // an IR builder.
5143 IRBuilderTy IRB(&AI);
5144
5145 // Collect the new slices which we will merge into the alloca slices.
5146 SmallVector<Slice, 4> NewSlices;
5147
5148 // Track any allocas we end up splitting loads and stores for so we iterate
5149 // on them.
5150 SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas;
5151
5152 // At this point, we have collected all of the loads and stores we can
5153 // pre-split, and the specific splits needed for them. We actually do the
5154 // splitting in a specific order in order to handle when one of the loads in
5155 // the value operand to one of the stores.
5156 //
5157 // First, we rewrite all of the split loads, and just accumulate each split
5158 // load in a parallel structure. We also build the slices for them and append
5159 // them to the alloca slices.
5160 SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap;
5161 std::vector<LoadInst *> SplitLoads;
5162 const DataLayout &DL = AI.getDataLayout();
5163 for (LoadInst *LI : Loads) {
5164 SplitLoads.clear();
5165
5166 auto &Offsets = SplitOffsetsMap[LI];
5167 unsigned SliceSize = Offsets.S->endOffset() - Offsets.S->beginOffset();
5168 assert(LI->getType()->getIntegerBitWidth() % 8 == 0 &&
5169 "Load must have type size equal to store size");
5170 assert(LI->getType()->getIntegerBitWidth() / 8 >= SliceSize &&
5171 "Load must be >= slice size");
5172
5173 uint64_t BaseOffset = Offsets.S->beginOffset();
5174 assert(BaseOffset + SliceSize > BaseOffset &&
5175 "Cannot represent alloca access size using 64-bit integers!");
5176
5178 IRB.SetInsertPoint(LI);
5179
5180 LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n");
5181
5182 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5183 int Idx = 0, Size = Offsets.Splits.size();
5184 for (;;) {
5185 auto *PartTy = Type::getIntNTy(LI->getContext(), PartSize * 8);
5186 auto AS = LI->getPointerAddressSpace();
5187 auto *PartPtrTy = LI->getPointerOperandType();
5188 LoadInst *PLoad = IRB.CreateAlignedLoad(
5189 PartTy,
5190 getAdjustedPtr(IRB, DL, BasePtr,
5191 APInt(DL.getIndexSizeInBits(AS), PartOffset),
5192 PartPtrTy, BasePtr->getName() + "."),
5193 getAdjustedAlignment(LI, PartOffset),
5194 /*IsVolatile*/ false, LI->getName());
5195 PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
5196 LLVMContext::MD_access_group});
5197
5198 // Append this load onto the list of split loads so we can find it later
5199 // to rewrite the stores.
5200 SplitLoads.push_back(PLoad);
5201
5202 // Now build a new slice for the alloca.
5203 NewSlices.push_back(
5204 Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5205 &PLoad->getOperandUse(PLoad->getPointerOperandIndex()),
5206 /*IsSplittable*/ false));
5207 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5208 << ", " << NewSlices.back().endOffset()
5209 << "): " << *PLoad << "\n");
5210
5211 // See if we've handled all the splits.
5212 if (Idx >= Size)
5213 break;
5214
5215 // Setup the next partition.
5216 PartOffset = Offsets.Splits[Idx];
5217 ++Idx;
5218 PartSize = (Idx < Size ? Offsets.Splits[Idx] : SliceSize) - PartOffset;
5219 }
5220
5221 // Now that we have the split loads, do the slow walk over all uses of the
5222 // load and rewrite them as split stores, or save the split loads to use
5223 // below if the store is going to be split there anyways.
5224 bool DeferredStores = false;
5225 for (User *LU : LI->users()) {
5226 StoreInst *SI = cast<StoreInst>(LU);
5227 if (!Stores.empty() && SplitOffsetsMap.count(SI)) {
5228 DeferredStores = true;
5229 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
5230 << "\n");
5231 continue;
5232 }
5233
5234 Value *StoreBasePtr = SI->getPointerOperand();
5235 IRB.SetInsertPoint(SI);
5236 AAMDNodes AATags = SI->getAAMetadata();
5237
5238 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
5239
5240 for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) {
5241 LoadInst *PLoad = SplitLoads[Idx];
5242 uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1];
5243 auto *PartPtrTy = SI->getPointerOperandType();
5244
5245 auto AS = SI->getPointerAddressSpace();
5246 StoreInst *PStore = IRB.CreateAlignedStore(
5247 PLoad,
5248 getAdjustedPtr(IRB, DL, StoreBasePtr,
5249 APInt(DL.getIndexSizeInBits(AS), PartOffset),
5250 PartPtrTy, StoreBasePtr->getName() + "."),
5251 getAdjustedAlignment(SI, PartOffset),
5252 /*IsVolatile*/ false);
5253 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access,
5254 LLVMContext::MD_access_group,
5255 LLVMContext::MD_DIAssignID});
5256
5257 if (AATags)
5258 PStore->setAAMetadata(
5259 AATags.adjustForAccess(PartOffset, PLoad->getType(), DL));
5260 LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
5261 }
5262
5263 // We want to immediately iterate on any allocas impacted by splitting
5264 // this store, and we have to track any promotable alloca (indicated by
5265 // a direct store) as needing to be resplit because it is no longer
5266 // promotable.
5267 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(StoreBasePtr)) {
5268 ResplitPromotableAllocas.insert(OtherAI);
5269 Worklist.insert(OtherAI);
5270 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5271 StoreBasePtr->stripInBoundsOffsets())) {
5272 Worklist.insert(OtherAI);
5273 }
5274
5275 // Mark the original store as dead.
5276 DeadInsts.push_back(SI);
5277 }
5278
5279 // Save the split loads if there are deferred stores among the users.
5280 if (DeferredStores)
5281 SplitLoadsMap.insert(std::make_pair(LI, std::move(SplitLoads)));
5282
5283 // Mark the original load as dead and kill the original slice.
5284 DeadInsts.push_back(LI);
5285 Offsets.S->kill();
5286 }
5287
5288 // Second, we rewrite all of the split stores. At this point, we know that
5289 // all loads from this alloca have been split already. For stores of such
5290 // loads, we can simply look up the pre-existing split loads. For stores of
5291 // other loads, we split those loads first and then write split stores of
5292 // them.
5293 for (StoreInst *SI : Stores) {
5294 auto *LI = cast<LoadInst>(SI->getValueOperand());
5295 IntegerType *Ty = cast<IntegerType>(LI->getType());
5296 assert(Ty->getBitWidth() % 8 == 0);
5297 uint64_t StoreSize = Ty->getBitWidth() / 8;
5298 assert(StoreSize > 0 && "Cannot have a zero-sized integer store!");
5299
5300 auto &Offsets = SplitOffsetsMap[SI];
5301 assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
5302 "Slice size should always match load size exactly!");
5303 uint64_t BaseOffset = Offsets.S->beginOffset();
5304 assert(BaseOffset + StoreSize > BaseOffset &&
5305 "Cannot represent alloca access size using 64-bit integers!");
5306
5307 Value *LoadBasePtr = LI->getPointerOperand();
5308 Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand());
5309
5310 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
5311
5312 // Check whether we have an already split load.
5313 auto SplitLoadsMapI = SplitLoadsMap.find(LI);
5314 std::vector<LoadInst *> *SplitLoads = nullptr;
5315 if (SplitLoadsMapI != SplitLoadsMap.end()) {
5316 SplitLoads = &SplitLoadsMapI->second;
5317 assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&
5318 "Too few split loads for the number of splits in the store!");
5319 } else {
5320 LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n");
5321 }
5322
5323 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5324 int Idx = 0, Size = Offsets.Splits.size();
5325 for (;;) {
5326 auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8);
5327 auto *LoadPartPtrTy = LI->getPointerOperandType();
5328 auto *StorePartPtrTy = SI->getPointerOperandType();
5329
5330 // Either lookup a split load or create one.
5331 LoadInst *PLoad;
5332 if (SplitLoads) {
5333 PLoad = (*SplitLoads)[Idx];
5334 } else {
5335 IRB.SetInsertPoint(LI);
5336 auto AS = LI->getPointerAddressSpace();
5337 PLoad = IRB.CreateAlignedLoad(
5338 PartTy,
5339 getAdjustedPtr(IRB, DL, LoadBasePtr,
5340 APInt(DL.getIndexSizeInBits(AS), PartOffset),
5341 LoadPartPtrTy, LoadBasePtr->getName() + "."),
5342 getAdjustedAlignment(LI, PartOffset),
5343 /*IsVolatile*/ false, LI->getName());
5344 PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
5345 LLVMContext::MD_access_group});
5346 }
5347
5348 // And store this partition.
5349 IRB.SetInsertPoint(SI);
5350 auto AS = SI->getPointerAddressSpace();
5351 StoreInst *PStore = IRB.CreateAlignedStore(
5352 PLoad,
5353 getAdjustedPtr(IRB, DL, StoreBasePtr,
5354 APInt(DL.getIndexSizeInBits(AS), PartOffset),
5355 StorePartPtrTy, StoreBasePtr->getName() + "."),
5356 getAdjustedAlignment(SI, PartOffset),
5357 /*IsVolatile*/ false);
5358 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access,
5359 LLVMContext::MD_access_group});
5360
5361 // Now build a new slice for the alloca.
5362 NewSlices.push_back(
5363 Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5364 &PStore->getOperandUse(PStore->getPointerOperandIndex()),
5365 /*IsSplittable*/ false));
5366 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5367 << ", " << NewSlices.back().endOffset()
5368 << "): " << *PStore << "\n");
5369 if (!SplitLoads) {
5370 LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n");
5371 }
5372
5373 // See if we've finished all the splits.
5374 if (Idx >= Size)
5375 break;
5376
5377 // Setup the next partition.
5378 PartOffset = Offsets.Splits[Idx];
5379 ++Idx;
5380 PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset;
5381 }
5382
5383 // We want to immediately iterate on any allocas impacted by splitting
5384 // this load, which is only relevant if it isn't a load of this alloca and
5385 // thus we didn't already split the loads above. We also have to keep track
5386 // of any promotable allocas we split loads on as they can no longer be
5387 // promoted.
5388 if (!SplitLoads) {
5389 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(LoadBasePtr)) {
5390 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5391 ResplitPromotableAllocas.insert(OtherAI);
5392 Worklist.insert(OtherAI);
5393 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5394 LoadBasePtr->stripInBoundsOffsets())) {
5395 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5396 Worklist.insert(OtherAI);
5397 }
5398 }
5399
5400 // Mark the original store as dead now that we've split it up and kill its
5401 // slice. Note that we leave the original load in place unless this store
5402 // was its only use. It may in turn be split up if it is an alloca load
5403 // for some other alloca, but it may be a normal load. This may introduce
5404 // redundant loads, but where those can be merged the rest of the optimizer
5405 // should handle the merging, and this uncovers SSA splits which is more
5406 // important. In practice, the original loads will almost always be fully
5407 // split and removed eventually, and the splits will be merged by any
5408 // trivial CSE, including instcombine.
5409 if (LI->hasOneUse()) {
5410 assert(*LI->user_begin() == SI && "Single use isn't this store!");
5411 DeadInsts.push_back(LI);
5412 }
5413 DeadInsts.push_back(SI);
5414 Offsets.S->kill();
5415 }
5416
5417 // Remove the killed slices that have ben pre-split.
5418 llvm::erase_if(AS, [](const Slice &S) { return S.isDead(); });
5419
5420 // Insert our new slices. This will sort and merge them into the sorted
5421 // sequence.
5422 AS.insert(NewSlices);
5423
5424 LLVM_DEBUG(dbgs() << " Pre-split slices:\n");
5425#ifndef NDEBUG
5426 for (auto I = AS.begin(), E = AS.end(); I != E; ++I)
5427 LLVM_DEBUG(AS.print(dbgs(), I, " "));
5428#endif
5429
5430 // Finally, don't try to promote any allocas that new require re-splitting.
5431 // They have already been added to the worklist above.
5432 PromotableAllocas.set_subtract(ResplitPromotableAllocas);
5433
5434 return true;
5435}
5436
5437/// Try to canonicalize a homogeneous struct partition to a vector type.
5438///
5439/// We can do this if all the elements of the struct are the same and the
5440/// corresponding vector has the same byte-level layout. This can sometimes
5441/// eliminate allocas because structs cannot get promoted to LLVM values, but
5442/// vectors can.
5443///
5444/// We only apply this transformation when all users of the partition are memory
5445/// intrinsics. Otherwise, if there is a load or store of some other type to the
5446/// partition, SROA would select that type.
5447///
5448/// Applying this transformation too early may hinder memcpyopt, which may
5449/// generate better code when eliminating allocas. For example, see
5450/// `struct-to-vector-fp-store-only-tail.ll`, which demonstrates that applying
5451/// this before memcpyopt can initialize previously uninitialized memory when
5452/// the alloca gets promoted to an SSA value. For another example, see
5453/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
5454/// this before memcpyopt can result in promoting an alloca so that we load a
5455/// temporary value instead of copying the temporary value into memory, whereas
5456/// memcpyopt eliminates the temporary altogether.
5457///
5458/// As such, we only apply this transformation after memcpyopt has run. We gate
5459/// this transformation by the "AggregateToVector" pass option.
5461 Partition &P,
5462 const DataLayout &DL) {
5463 unsigned NumElts = STy->getNumElements();
5464
5465 Type *EltTy = STy->getElementType(0);
5466 if (!llvm::all_equal(STy->elements()))
5467 return nullptr;
5468
5469 bool IsIntegralPointerTy =
5470 EltTy->isPointerTy() && !DL.isNonIntegralPointerType(EltTy);
5471 if (!EltTy->isIntegerTy() && !EltTy->isFloatingPointTy() &&
5472 !IsIntegralPointerTy)
5473 return nullptr;
5474
5475 // Ensure the struct is tightly packed so that the bit-layout is the same as
5476 // the corresponding vector. For example, this prevents a miscompile for
5477 // { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
5478 // tightly packed elements and trailing padding.
5479 if (DL.getTypeSizeInBits(EltTy) != DL.getTypeAllocSizeInBits(EltTy))
5480 return nullptr;
5481
5482 auto *VTy = FixedVectorType::get(EltTy, NumElts);
5483 TypeSize StructSize = DL.getStructLayout(STy)->getSizeInBytes();
5484 TypeSize VectorSize = DL.getTypeStoreSize(VTy);
5485 // After ruling out per-element padding, make sure a vector load/store
5486 // covers the same number of bytes as the struct layout.
5487 if (StructSize != VectorSize)
5488 return nullptr;
5489
5490 auto IsIgnorableOrMemIntrinsicSlice = [](const Slice &S) {
5491 if (S.isDead())
5492 return true;
5493 auto *U = S.getUse();
5494 if (!U)
5495 return true;
5496
5497 User *Usr = U->getUser();
5499 return true;
5500
5501 return isa<MemIntrinsic>(Usr);
5502 };
5503
5504 for (const Slice &S : P)
5505 if (!IsIgnorableOrMemIntrinsicSlice(S))
5506 return nullptr;
5507
5508 for (const Slice *S : P.splitSliceTails())
5509 if (!IsIgnorableOrMemIntrinsicSlice(*S))
5510 return nullptr;
5511
5512 return VTy;
5513}
5514
5515/// Select a partition type for an alloca partition.
5516///
5517/// Try to compute a friendly type for this partition of the alloca. This
5518/// won't always succeed, in which case we fall back to a legal integer type
5519/// or an i8 array of an appropriate size.
5520///
5521/// \returns A tuple with the following elements:
5522/// - PartitionType: The computed type for this partition.
5523/// - IsIntegerWideningViable: True if integer widening promotion is used.
5524/// - VectorType: The vector type if vector promotion is used, otherwise
5525/// nullptr.
5526static std::tuple<Type *, bool, VectorType *>
5528 LLVMContext &C, bool AggregateToVector) {
5529 auto LogSelection = [&](StringRef Path, Type *SelectedTy,
5530 VectorType *SelectedVecTy, bool SelectedIntWidening) {
5531 LLVM_DEBUG({
5532 dbgs() << "selectPartitionType path=" << Path
5533 << " func=" << AI.getFunction()->getName() << " alloca=";
5534 if (AI.hasName())
5535 dbgs() << AI.getName();
5536 else
5537 dbgs() << "<unnamed>";
5538 dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
5539 << ") size=" << P.size();
5540 if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
5541 dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
5542 if (SelectedTy)
5543 dbgs() << " chosen=" << *SelectedTy;
5544 if (SelectedVecTy)
5545 dbgs() << " vec=" << *SelectedVecTy;
5546 dbgs() << " intwiden=" << SelectedIntWidening << "\n";
5547 });
5548 };
5549 // First check if the partition is viable for vector promotion.
5550 //
5551 // We prefer vector promotion over integer widening promotion when:
5552 // - The vector element type is a floating-point type.
5553 // - All the loads/stores to the alloca are vector loads/stores to the
5554 // entire alloca or load/store a single element of the vector.
5555 //
5556 // Otherwise when there is an integer vector with mixed type loads/stores we
5557 // prefer integer widening promotion because it's more likely the user is
5558 // doing bitwise arithmetic and we generate better code.
5559 VectorType *VecTy =
5561 // If the vector element type is a floating-point type, we prefer vector
5562 // promotion. If the vector has one element, let the below code select
5563 // whether we promote with the vector or scalar.
5564 if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
5565 VecTy->getElementCount().getFixedValue() > 1) {
5566 LogSelection("direct-fp-vecty", VecTy, VecTy, false);
5567 return {VecTy, false, VecTy};
5568 }
5569
5570 // Check if there is a common type that all slices of the partition use that
5571 // spans the partition.
5572 auto [CommonUseTy, LargestIntTy] =
5573 findCommonType(P.begin(), P.end(), P.endOffset());
5574 if (CommonUseTy) {
5575 TypeSize CommonUseSize = DL.getTypeAllocSize(CommonUseTy);
5576 if (CommonUseSize.isFixed() && CommonUseSize.getFixedValue() >= P.size()) {
5577 // We prefer vector promotion here because if vector promotion is viable
5578 // and there is a common type used, then it implies the second listed
5579 // condition for preferring vector promotion is true.
5580 if (VecTy) {
5581 LogSelection("common-type-vecty", VecTy, VecTy, false);
5582 return {VecTy, false, VecTy};
5583 }
5584 bool IntWiden = isIntegerWideningViable(P, CommonUseTy, DL);
5585 LogSelection("common-type", CommonUseTy, nullptr, IntWiden);
5586 return {CommonUseTy, IntWiden, nullptr};
5587 }
5588 }
5589
5590 // Can we find an appropriate subtype in the original allocated
5591 // type?
5592 if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(),
5593 P.beginOffset(), P.size())) {
5594 // If the partition is an integer array that can be spanned by a legal
5595 // integer type, prefer to represent it as a legal integer type because
5596 // it's more likely to be promotable.
5597 if (TypePartitionTy->isArrayTy() &&
5598 TypePartitionTy->getArrayElementType()->isIntegerTy() &&
5599 DL.isLegalInteger(P.size() * 8))
5600 TypePartitionTy = Type::getIntNTy(C, P.size() * 8);
5601 // There was no common type used, so we prefer integer widening promotion.
5602 if (isIntegerWideningViable(P, TypePartitionTy, DL)) {
5603 LogSelection("type-partition-int-widen", TypePartitionTy, nullptr, true);
5604 return {TypePartitionTy, true, nullptr};
5605 }
5606 if (VecTy) {
5607 LogSelection("type-partition-vecty", VecTy, VecTy, false);
5608 return {VecTy, false, VecTy};
5609 }
5610 // If we couldn't promote with TypePartitionTy, try with the largest
5611 // integer type used.
5612 if (LargestIntTy &&
5613 DL.getTypeAllocSize(LargestIntTy).getFixedValue() >= P.size() &&
5614 isIntegerWideningViable(P, LargestIntTy, DL)) {
5615 LogSelection("largest-int-int-widen", LargestIntTy, nullptr, true);
5616 return {LargestIntTy, true, nullptr};
5617 }
5618
5619 // Try homogeneous struct to vector canonicalization when requested. Running
5620 // this too early can hide memcpy chains from MemCpyOpt.
5621 if (AggregateToVector) {
5622 if (auto *STy = dyn_cast<StructType>(TypePartitionTy)) {
5623 if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
5624 LogSelection("struct-fallback-vecty", VTy, nullptr, false);
5625 return {VTy, false, nullptr};
5626 }
5627 }
5628 }
5629
5630 // Fallback to TypePartitionTy and we probably won't promote.
5631 LogSelection("type-partition-fallback", TypePartitionTy, nullptr, false);
5632 return {TypePartitionTy, false, nullptr};
5633 }
5634
5635 // Select the largest integer type used if it spans the partition.
5636 if (LargestIntTy &&
5637 DL.getTypeAllocSize(LargestIntTy).getFixedValue() >= P.size()) {
5638 LogSelection("largest-int-fallback", LargestIntTy, nullptr, false);
5639 return {LargestIntTy, false, nullptr};
5640 }
5641
5642 // Select a legal integer type if it spans the partition.
5643 if (DL.isLegalInteger(P.size() * 8)) {
5644 Type *IntTy = Type::getIntNTy(C, P.size() * 8);
5645 LogSelection("legal-int-fallback", IntTy, nullptr, false);
5646 return {IntTy, false, nullptr};
5647 }
5648
5649 // Fallback to an i8 array.
5650 Type *ArrayTy = ArrayType::get(Type::getInt8Ty(C), P.size());
5651 LogSelection("byte-array-fallback", ArrayTy, nullptr, false);
5652 return {ArrayTy, false, nullptr};
5653}
5654
5655/// Rewrite an alloca partition's users.
5656///
5657/// This routine drives both of the rewriting goals of the SROA pass. It tries
5658/// to rewrite uses of an alloca partition to be conducive for SSA value
5659/// promotion. If the partition needs a new, more refined alloca, this will
5660/// build that new alloca, preserving as much type information as possible, and
5661/// rewrite the uses of the old alloca to point at the new one and have the
5662/// appropriate new offsets. It also evaluates how successful the rewrite was
5663/// at enabling promotion and if it was successful queues the alloca to be
5664/// promoted.
5665std::pair<AllocaInst *, uint64_t>
5666SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
5667 const DataLayout &DL = AI.getDataLayout();
5668 // Select the type for the new alloca that spans the partition.
5669 auto [PartitionTy, IsIntegerWideningViable, VecTy] =
5670 selectPartitionType(P, DL, AI, *C, AggregateToVector);
5671
5672 // Check for the case where we're going to rewrite to a new alloca of the
5673 // exact same type as the original, and with the same access offsets. In that
5674 // case, re-use the existing alloca, but still run through the rewriter to
5675 // perform phi and select speculation.
5676 // P.beginOffset() can be non-zero even with the same type in a case with
5677 // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll).
5678 AllocaInst *NewAI;
5679 if (PartitionTy == AI.getAllocatedType() && P.beginOffset() == 0) {
5680 NewAI = &AI;
5681 // FIXME: We should be able to bail at this point with "nothing changed".
5682 // FIXME: We might want to defer PHI speculation until after here.
5683 // FIXME: return nullptr;
5684 } else {
5685 // Make sure the alignment is compatible with P.beginOffset().
5686 const Align Alignment = commonAlignment(AI.getAlign(), P.beginOffset());
5687 NewAI =
5688 new AllocaInst(PartitionTy, AI.getAddressSpace(), nullptr, Alignment,
5689 AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()),
5690 AI.getIterator());
5691 tryEnforceAlignment(NewAI, DL.getPrefTypeAlign(PartitionTy), DL);
5692 // Copy the old AI debug location over to the new one.
5693 NewAI->setDebugLoc(AI.getDebugLoc());
5694 ++NumNewAllocas;
5695 }
5696
5697 LLVM_DEBUG(dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset()
5698 << "," << P.endOffset() << ") to: " << *NewAI << "\n");
5699
5700 // Track the high watermark on the worklist as it is only relevant for
5701 // promoted allocas. We will reset it to this point if the alloca is not in
5702 // fact scheduled for promotion.
5703 unsigned PPWOldSize = PostPromotionWorklist.size();
5704 unsigned NumUses = 0;
5705 SmallSetVector<PHINode *, 8> PHIUsers;
5706 SmallSetVector<SelectInst *, 8> SelectUsers;
5707
5708 AllocaSliceRewriter Rewriter(
5709 DL, AS, *this, AI, *NewAI, PartitionTy, P.beginOffset(), P.endOffset(),
5710 IsIntegerWideningViable, VecTy, PHIUsers, SelectUsers);
5711 bool Promotable = true;
5712 // Check whether we can have tree-structured merge.
5713 if (auto DeletedValues = Rewriter.rewriteTreeStructuredMerge(P)) {
5714 NumUses += DeletedValues->size() + 1;
5715 for (Value *V : *DeletedValues)
5716 DeadInsts.push_back(V);
5717 } else {
5718 for (Slice *S : P.splitSliceTails()) {
5719 Promotable &= Rewriter.visit(S);
5720 ++NumUses;
5721 }
5722 for (Slice &S : P) {
5723 Promotable &= Rewriter.visit(&S);
5724 ++NumUses;
5725 }
5726 }
5727
5728 NumAllocaPartitionUses += NumUses;
5729 MaxUsesPerAllocaPartition.updateMax(NumUses);
5730
5731 // Now that we've processed all the slices in the new partition, check if any
5732 // PHIs or Selects would block promotion.
5733 SmallVector<PHINode *, 8> NewSpeculatablePHIs;
5734 SmallVector<PHINode *, 2> NewPHIsWithStoreToRewrite;
5735 for (PHINode *PHI : PHIUsers) {
5736 if (isSafePHIToSpeculate(*PHI)) {
5737 NewSpeculatablePHIs.push_back(PHI);
5738 continue;
5739 }
5740 if (getPHIStoreToRewrite(*PHI, PreserveCFG, DTU->getDomTree())) {
5741 NewPHIsWithStoreToRewrite.push_back(PHI);
5742 continue;
5743 }
5744
5745 Promotable = false;
5746 SelectUsers.clear();
5747 break;
5748 }
5749
5751 NewSelectsToRewrite;
5752 NewSelectsToRewrite.reserve(SelectUsers.size());
5753 for (SelectInst *Sel : SelectUsers) {
5754 std::optional<RewriteableMemOps> Ops =
5755 isSafeSelectToSpeculate(*Sel, PreserveCFG);
5756 if (!Ops) {
5757 Promotable = false;
5758 break;
5759 }
5760 NewSelectsToRewrite.emplace_back(std::make_pair(Sel, *Ops));
5761 }
5762
5763 if (Promotable) {
5764 for (Use *U : AS.getDeadUsesIfPromotable()) {
5765 auto *OldInst = dyn_cast<Instruction>(U->get());
5766 Value::dropDroppableUse(*U);
5767 if (OldInst)
5768 if (isInstructionTriviallyDead(OldInst))
5769 DeadInsts.push_back(OldInst);
5770 }
5771 if (NewSpeculatablePHIs.empty() && NewPHIsWithStoreToRewrite.empty() &&
5772 SelectUsers.empty()) {
5773 // Promote the alloca.
5774 PromotableAllocas.insert(NewAI);
5775 } else {
5776 // If we have either PHIs or Selects to rewrite, add them to those
5777 // worklists and re-queue the new alloca so that we promote it on the next
5778 // iteration.
5779 SpeculatablePHIs.insert_range(NewSpeculatablePHIs);
5780 PHIsWithStoreToRewrite.insert_range(NewPHIsWithStoreToRewrite);
5781 SelectsToRewrite.reserve(SelectsToRewrite.size() +
5782 NewSelectsToRewrite.size());
5783 for (auto &&KV : llvm::make_range(
5784 std::make_move_iterator(NewSelectsToRewrite.begin()),
5785 std::make_move_iterator(NewSelectsToRewrite.end())))
5786 SelectsToRewrite.insert(std::move(KV));
5787 Worklist.insert(NewAI);
5788 }
5789 } else {
5790 // Drop any post-promotion work items if promotion didn't happen.
5791 while (PostPromotionWorklist.size() > PPWOldSize)
5792 PostPromotionWorklist.pop_back();
5793
5794 // We couldn't promote and we didn't create a new partition, nothing
5795 // happened.
5796 if (NewAI == &AI)
5797 return {nullptr, 0};
5798
5799 // If we can't promote the alloca, iterate on it to check for new
5800 // refinements exposed by splitting the current alloca. Don't iterate on an
5801 // alloca which didn't actually change and didn't get promoted.
5802 Worklist.insert(NewAI);
5803 }
5804
5805 return {NewAI, DL.getTypeSizeInBits(PartitionTy).getFixedValue()};
5806}
5807
5808// There isn't a shared interface to get the "address" parts out of a
5809// dbg.declare and dbg.assign, so provide some wrappers.
5812 return DVR->isKillAddress();
5813 return DVR->isKillLocation();
5814}
5815
5818 return DVR->getAddressExpression();
5819 return DVR->getExpression();
5820}
5821
5822/// Create or replace an existing fragment in a DIExpression with \p Frag.
5823/// If the expression already contains a DW_OP_LLVM_extract_bits_[sz]ext
5824/// operation, add \p BitExtractOffset to the offset part.
5825///
5826/// Returns the new expression, or nullptr if this fails (see details below).
5827///
5828/// This function is similar to DIExpression::createFragmentExpression except
5829/// for 3 important distinctions:
5830/// 1. The new fragment isn't relative to an existing fragment.
5831/// 2. It assumes the computed location is a memory location. This means we
5832/// don't need to perform checks that creating the fragment preserves the
5833/// expression semantics.
5834/// 3. Existing extract_bits are modified independently of fragment changes
5835/// using \p BitExtractOffset. A change to the fragment offset or size
5836/// may affect a bit extract. But a bit extract offset can change
5837/// independently of the fragment dimensions.
5838///
5839/// Returns the new expression, or nullptr if one couldn't be created.
5840/// Ideally this is only used to signal that a bit-extract has become
5841/// zero-sized (and thus the new debug record has no size and can be
5842/// dropped), however, it fails for other reasons too - see the FIXME below.
5843///
5844/// FIXME: To keep the change that introduces this function NFC it bails
5845/// in some situations unecessarily, e.g. when fragment and bit extract
5846/// sizes differ.
5849 int64_t BitExtractOffset) {
5851 bool HasFragment = false;
5852 bool HasBitExtract = false;
5853
5854 for (auto &Op : Expr->expr_ops()) {
5855 if (Op.getOp() == dwarf::DW_OP_LLVM_fragment) {
5856 HasFragment = true;
5857 continue;
5858 }
5859 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Op)) {
5860 HasBitExtract = true;
5861 int64_t ExtractOffsetInBits = Extract.getOffsetInBits();
5862 int64_t ExtractSizeInBits = Extract.getSizeInBits();
5863
5864 // DIExpression::createFragmentExpression doesn't know how to handle
5865 // a fragment that is smaller than the extract. Copy the behaviour
5866 // (bail) to avoid non-NFC changes.
5867 // FIXME: Don't do this.
5868 if (Frag.SizeInBits < uint64_t(ExtractSizeInBits))
5869 return nullptr;
5870
5871 assert(BitExtractOffset <= 0);
5872 int64_t AdjustedOffset = ExtractOffsetInBits + BitExtractOffset;
5873
5874 // DIExpression::createFragmentExpression doesn't know what to do
5875 // if the new extract starts "outside" the existing one. Copy the
5876 // behaviour (bail) to avoid non-NFC changes.
5877 // FIXME: Don't do this.
5878 if (AdjustedOffset < 0)
5879 return nullptr;
5880
5881 Ops.push_back(Op.getOp());
5882 Ops.push_back(std::max<int64_t>(0, AdjustedOffset));
5883 Ops.push_back(ExtractSizeInBits);
5884 continue;
5885 }
5886 Op.appendToVector(Ops);
5887 }
5888
5889 // Unsupported by createFragmentExpression, so don't support it here yet to
5890 // preserve NFC-ness.
5891 if (HasFragment && HasBitExtract)
5892 return nullptr;
5893
5894 if (!HasBitExtract) {
5896 Ops.push_back(Frag.OffsetInBits);
5897 Ops.push_back(Frag.SizeInBits);
5898 }
5899 return DIExpression::get(Expr->getContext(), Ops);
5900}
5901
5902/// Insert a new DbgRecord.
5903/// \p Orig Original to copy record type, debug loc and variable from, and
5904/// additionally value and value expression for dbg_assign records.
5905/// \p NewAddr Location's new base address.
5906/// \p NewAddrExpr New expression to apply to address.
5907/// \p BeforeInst Insert position.
5908/// \p NewFragment New fragment (absolute, non-relative).
5909/// \p BitExtractAdjustment Offset to apply to any extract_bits op.
5910static void
5912 DIExpression *NewAddrExpr, Instruction *BeforeInst,
5913 std::optional<DIExpression::FragmentInfo> NewFragment,
5914 int64_t BitExtractAdjustment) {
5915 (void)DIB;
5916
5917 // A dbg_assign puts fragment info in the value expression only. The address
5918 // expression has already been built: NewAddrExpr. A dbg_declare puts the
5919 // new fragment info into NewAddrExpr (as it only has one expression).
5920 DIExpression *NewFragmentExpr =
5921 Orig->isDbgAssign() ? Orig->getExpression() : NewAddrExpr;
5922 if (NewFragment)
5923 NewFragmentExpr = createOrReplaceFragment(NewFragmentExpr, *NewFragment,
5924 BitExtractAdjustment);
5925 if (!NewFragmentExpr)
5926 return;
5927
5928 if (Orig->isDbgDeclare()) {
5930 NewAddr, Orig->getVariable(), NewFragmentExpr, Orig->getDebugLoc());
5931 BeforeInst->getParent()->insertDbgRecordBefore(DVR,
5932 BeforeInst->getIterator());
5933 return;
5934 }
5935
5936 if (Orig->isDbgValue()) {
5938 NewAddr, Orig->getVariable(), NewFragmentExpr, Orig->getDebugLoc());
5939 // Drop debug information if the expression doesn't start with a
5940 // DW_OP_deref. This is because without a DW_OP_deref, the #dbg_value
5941 // describes the address of alloca rather than the value inside the alloca.
5942 if (!NewFragmentExpr->startsWithDeref())
5943 DVR->setKillAddress();
5944 BeforeInst->getParent()->insertDbgRecordBefore(DVR,
5945 BeforeInst->getIterator());
5946 return;
5947 }
5948
5949 // Apply a DIAssignID to the store if it doesn't already have it.
5950 if (!NewAddr->hasMetadata(LLVMContext::MD_DIAssignID)) {
5951 NewAddr->setMetadata(LLVMContext::MD_DIAssignID,
5953 }
5954
5956 NewAddr, Orig->getValue(), Orig->getVariable(), NewFragmentExpr, NewAddr,
5957 NewAddrExpr, Orig->getDebugLoc());
5958 LLVM_DEBUG(dbgs() << "Created new DVRAssign: " << *NewAssign << "\n");
5959 (void)NewAssign;
5960}
5961
5962/// Walks the slices of an alloca and form partitions based on them,
5963/// rewriting each of their uses.
5964bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
5965 if (AS.begin() == AS.end())
5966 return false;
5967
5968 unsigned NumPartitions = 0;
5969 bool Changed = false;
5970 const DataLayout &DL = AI.getModule()->getDataLayout();
5971
5972 // First try to pre-split loads and stores.
5973 Changed |= presplitLoadsAndStores(AI, AS);
5974
5975 // Now that we have identified any pre-splitting opportunities,
5976 // mark loads and stores unsplittable except for the following case.
5977 // We leave a slice splittable if all other slices are disjoint or fully
5978 // included in the slice, such as whole-alloca loads and stores.
5979 // If we fail to split these during pre-splitting, we want to force them
5980 // to be rewritten into a partition.
5981 bool IsSorted = true;
5982
5983 uint64_t AllocaSize = AI.getAllocationSize(DL)->getFixedValue();
5984 // We can split at the begin and end offsets of each slice, but only if those
5985 // offsets don't lie inside another slice. Because slices are ordered by
5986 // increasing begin offset, and then decreasing end offset, we can consider
5987 // the slices as being split up into sets with the same begin offset where we
5988 // can ignore every slice except the first (the begin offset will already be
5989 // handled as the begin offset of the set, and the end offset we know is not a
5990 // splittable offset as it's inside the first slice of the set).
5991 SparseBitVector<> SplittableOffset;
5992 uint64_t CurBegin = 0, CurEnd = 0;
5993 for (Slice &S : AS) {
5994 // Check if we have a new set of slices
5995 if (S.beginOffset() > CurBegin || S.endOffset() > CurEnd) {
5996 // If the start isn't inside the previous set it's splittable
5997 if (S.beginOffset() >= CurEnd) {
5998 SplittableOffset.set(S.beginOffset());
5999 }
6000 // If the previous end is inside this slice then remove it
6001 if (CurEnd > S.beginOffset() && CurEnd < S.endOffset()) {
6002 SplittableOffset.reset(CurEnd);
6003 }
6004 CurBegin = S.beginOffset();
6005 // If the end offset isn't inside the previous set it's splittable. We
6006 // also don't update the end offset in that case, as the next set may also
6007 // be inside the previous set.
6008 if (S.endOffset() > CurEnd) {
6009 CurEnd = S.endOffset();
6010 SplittableOffset.set(CurEnd);
6011 }
6012 }
6013 }
6014
6015 for (Slice &S : AS) {
6016 if (!S.isSplittable())
6017 continue;
6018
6019 if ((S.beginOffset() > AllocaSize ||
6020 SplittableOffset.test(S.beginOffset())) &&
6021 (S.endOffset() > AllocaSize || SplittableOffset.test(S.endOffset())))
6022 continue;
6023
6024 if (isa<LoadInst>(S.getUse()->getUser()) ||
6025 isa<StoreInst>(S.getUse()->getUser())) {
6026 S.makeUnsplittable();
6027 IsSorted = false;
6028 }
6029 }
6030
6031 if (!IsSorted)
6033
6034 /// Describes the allocas introduced by rewritePartition in order to migrate
6035 /// the debug info.
6036 struct Fragment {
6037 AllocaInst *Alloca;
6039 uint64_t Size;
6040 Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
6041 : Alloca(AI), Offset(O), Size(S) {}
6042 };
6043 SmallVector<Fragment, 4> Fragments;
6044
6045 // Rewrite each partition.
6046 for (auto &P : AS.partitions()) {
6047 auto [NewAI, ActiveBits] = rewritePartition(AI, AS, P);
6048 if (NewAI) {
6049 Changed = true;
6050 if (NewAI != &AI) {
6051 uint64_t SizeOfByte = 8;
6052 // Don't include any padding.
6053 uint64_t Size = std::min(ActiveBits, P.size() * SizeOfByte);
6054 Fragments.push_back(
6055 Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
6056 }
6057 }
6058 ++NumPartitions;
6059 }
6060
6061 NumAllocaPartitions += NumPartitions;
6062 MaxPartitionsPerAlloca.updateMax(NumPartitions);
6063
6064 // Migrate debug information from the old alloca to the new alloca(s)
6065 // and the individual partitions.
6066 auto MigrateOne = [&](DbgVariableRecord *DbgVariable) {
6067 // Can't overlap with undef memory.
6068 if (isKillAddress(DbgVariable))
6069 return;
6070
6071 const Value *DbgPtr = DbgVariable->getAddress();
6073 DbgVariable->getFragmentOrEntireVariable();
6074 // Get the address expression constant offset if one exists and the ops
6075 // that come after it.
6076 int64_t CurrentExprOffsetInBytes = 0;
6077 SmallVector<uint64_t> PostOffsetOps;
6078 if (!getAddressExpression(DbgVariable)
6079 ->extractLeadingOffset(CurrentExprOffsetInBytes, PostOffsetOps))
6080 return; // Couldn't interpret this DIExpression - drop the var.
6081
6082 // Offset defined by a DW_OP_LLVM_extract_bits_[sz]ext.
6083 int64_t ExtractOffsetInBits = 0;
6084 for (auto Op : getAddressExpression(DbgVariable)->expr_ops()) {
6085 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Op)) {
6086 ExtractOffsetInBits = Extract.getOffsetInBits();
6087 break;
6088 }
6089 }
6090
6091 DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false);
6092 for (auto Fragment : Fragments) {
6093 int64_t OffsetFromLocationInBits;
6094 std::optional<DIExpression::FragmentInfo> NewDbgFragment;
6095 // Find the variable fragment that the new alloca slice covers.
6096 // Drop debug info for this variable fragment if we can't compute an
6097 // intersect between it and the alloca slice.
6099 DL, &AI, Fragment.Offset, Fragment.Size, DbgPtr,
6100 CurrentExprOffsetInBytes * 8, ExtractOffsetInBits, VarFrag,
6101 NewDbgFragment, OffsetFromLocationInBits))
6102 continue; // Do not migrate this fragment to this slice.
6103
6104 // Zero sized fragment indicates there's no intersect between the variable
6105 // fragment and the alloca slice. Skip this slice for this variable
6106 // fragment.
6107 if (NewDbgFragment && !NewDbgFragment->SizeInBits)
6108 continue; // Do not migrate this fragment to this slice.
6109
6110 // No fragment indicates DbgVariable's variable or fragment exactly
6111 // overlaps the slice; copy its fragment (or nullopt if there isn't one).
6112 if (!NewDbgFragment)
6113 NewDbgFragment = DbgVariable->getFragment();
6114
6115 // Reduce the new expression offset by the bit-extract offset since
6116 // we'll be keeping that.
6117 int64_t OffestFromNewAllocaInBits =
6118 OffsetFromLocationInBits - ExtractOffsetInBits;
6119 // We need to adjust an existing bit extract if the offset expression
6120 // can't eat the slack (i.e., if the new offset would be negative).
6121 int64_t BitExtractOffset =
6122 std::min<int64_t>(0, OffestFromNewAllocaInBits);
6123 // The magnitude of a negative value indicates the number of bits into
6124 // the existing variable fragment that the memory region begins. The new
6125 // variable fragment already excludes those bits - the new DbgPtr offset
6126 // only needs to be applied if it's positive.
6127 OffestFromNewAllocaInBits =
6128 std::max(int64_t(0), OffestFromNewAllocaInBits);
6129
6130 // Rebuild the expression:
6131 // {Offset(OffestFromNewAllocaInBits), PostOffsetOps, NewDbgFragment}
6132 // Add NewDbgFragment later, because dbg.assigns don't want it in the
6133 // address expression but the value expression instead.
6134 DIExpression *NewExpr = DIExpression::get(AI.getContext(), PostOffsetOps);
6135 if (OffestFromNewAllocaInBits > 0) {
6136 int64_t OffsetInBytes = (OffestFromNewAllocaInBits + 7) / 8;
6137 NewExpr = DIExpression::prepend(NewExpr, /*flags=*/0, OffsetInBytes);
6138 }
6139
6140 // Remove any existing intrinsics on the new alloca describing
6141 // the variable fragment.
6142 auto RemoveOne = [DbgVariable](auto *OldDII) {
6143 auto SameVariableFragment = [](const auto *LHS, const auto *RHS) {
6144 return LHS->getVariable() == RHS->getVariable() &&
6145 LHS->getDebugLoc()->getInlinedAt() ==
6146 RHS->getDebugLoc()->getInlinedAt();
6147 };
6148 if (SameVariableFragment(OldDII, DbgVariable))
6149 OldDII->eraseFromParent();
6150 };
6151 for_each(findDVRDeclares(Fragment.Alloca), RemoveOne);
6152 for_each(findDVRValues(Fragment.Alloca), RemoveOne);
6153 insertNewDbgInst(DIB, DbgVariable, Fragment.Alloca, NewExpr, &AI,
6154 NewDbgFragment, BitExtractOffset);
6155 }
6156 };
6157
6158 // Migrate debug information from the old alloca to the new alloca(s)
6159 // and the individual partitions.
6160 for_each(findDVRDeclares(&AI), MigrateOne);
6161 for_each(findDVRValues(&AI), MigrateOne);
6162 for_each(at::getDVRAssignmentMarkers(&AI), MigrateOne);
6163
6164 return Changed;
6165}
6166
6167/// Clobber a use with poison, deleting the used value if it becomes dead.
6168void SROA::clobberUse(Use &U) {
6169 Value *OldV = U;
6170 // Replace the use with an poison value.
6171 U = PoisonValue::get(OldV->getType());
6172
6173 // Check for this making an instruction dead. We have to garbage collect
6174 // all the dead instructions to ensure the uses of any alloca end up being
6175 // minimal.
6176 if (Instruction *OldI = dyn_cast<Instruction>(OldV))
6177 if (isInstructionTriviallyDead(OldI)) {
6178 DeadInsts.push_back(OldI);
6179 }
6180}
6181
6182/// A basic LoadAndStorePromoter that does not remove store nodes.
6184public:
6186 Type *ZeroType)
6187 : LoadAndStorePromoter(Insts, S), ZeroType(ZeroType) {}
6188 bool shouldDelete(Instruction *I) const override {
6189 return !isa<StoreInst>(I) && !isa<AllocaInst>(I);
6190 }
6191
6193 return UndefValue::get(ZeroType);
6194 }
6195
6196private:
6197 Type *ZeroType;
6198};
6199
6200bool SROA::propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS) {
6201 // Look through each "partition", looking for slices with the same start/end
6202 // that do not overlap with any before them. The slices are sorted by
6203 // increasing beginOffset. We don't use AS.partitions(), as it will use a more
6204 // sophisticated algorithm that takes splittable slices into account.
6205 LLVM_DEBUG(dbgs() << "Attempting to propagate values on " << AI << "\n");
6206 bool AllSameAndValid = true;
6207 Type *PartitionType = nullptr;
6208 SmallVector<Instruction *> Insts;
6209 uint64_t BeginOffset = 0;
6210 uint64_t EndOffset = 0;
6211
6212 auto Flush = [&]() {
6213 if (AllSameAndValid && !Insts.empty()) {
6214 LLVM_DEBUG(dbgs() << "Propagate values on slice [" << BeginOffset << ", "
6215 << EndOffset << ")\n");
6217 SSAUpdater SSA(&NewPHIs);
6218 Insts.push_back(&AI);
6219 BasicLoadAndStorePromoter Promoter(Insts, SSA, PartitionType);
6220 Promoter.run(Insts);
6221 }
6222 AllSameAndValid = true;
6223 PartitionType = nullptr;
6224 Insts.clear();
6225 };
6226
6227 for (Slice &S : AS) {
6228 auto *User = cast<Instruction>(S.getUse()->getUser());
6229 if (isAssumeLikeIntrinsic(User)) {
6230 LLVM_DEBUG({
6231 dbgs() << "Ignoring slice: ";
6232 AS.print(dbgs(), &S);
6233 });
6234 continue;
6235 }
6236 if (S.beginOffset() >= EndOffset) {
6237 Flush();
6238 BeginOffset = S.beginOffset();
6239 EndOffset = S.endOffset();
6240 } else if (S.beginOffset() != BeginOffset || S.endOffset() != EndOffset) {
6241 if (AllSameAndValid) {
6242 LLVM_DEBUG({
6243 dbgs() << "Slice does not match range [" << BeginOffset << ", "
6244 << EndOffset << ")";
6245 AS.print(dbgs(), &S);
6246 });
6247 AllSameAndValid = false;
6248 }
6249 EndOffset = std::max(EndOffset, S.endOffset());
6250 continue;
6251 }
6252
6253 if (auto *LI = dyn_cast<LoadInst>(User)) {
6254 Type *UserTy = LI->getType();
6255 // LoadAndStorePromoter requires all the types to be the same.
6256 if (!LI->isSimple() || (PartitionType && UserTy != PartitionType))
6257 AllSameAndValid = false;
6258 PartitionType = UserTy;
6259 Insts.push_back(User);
6260 } else if (auto *SI = dyn_cast<StoreInst>(User)) {
6261 Type *UserTy = SI->getValueOperand()->getType();
6262 if (!SI->isSimple() || (PartitionType && UserTy != PartitionType))
6263 AllSameAndValid = false;
6264 PartitionType = UserTy;
6265 Insts.push_back(User);
6266 } else {
6267 AllSameAndValid = false;
6268 }
6269 }
6270
6271 Flush();
6272 return true;
6273}
6274
6275/// Analyze an alloca for SROA.
6276///
6277/// This analyzes the alloca to ensure we can reason about it, builds
6278/// the slices of the alloca, and then hands it off to be split and
6279/// rewritten as needed.
6280std::pair<bool /*Changed*/, bool /*CFGChanged*/>
6281SROA::runOnAlloca(AllocaInst &AI) {
6282 bool Changed = false;
6283 bool CFGChanged = false;
6284
6285 LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
6286 ++NumAllocasAnalyzed;
6287
6288 // Special case dead allocas, as they're trivial.
6289 if (AI.use_empty()) {
6290 AI.eraseFromParent();
6291 Changed = true;
6292 return {Changed, CFGChanged};
6293 }
6294 const DataLayout &DL = AI.getDataLayout();
6295
6296 // Skip alloca forms that this analysis can't handle.
6297 std::optional<TypeSize> Size = AI.getAllocationSize(DL);
6298 if (AI.isArrayAllocation() || !Size || Size->isScalable() || Size->isZero())
6299 return {Changed, CFGChanged};
6300
6301 // First, split any FCA loads and stores touching this alloca to promote
6302 // better splitting and promotion opportunities.
6303 IRBuilderTy IRB(&AI);
6304 AggLoadStoreRewriter AggRewriter(DL, IRB);
6305 Changed |= AggRewriter.rewrite(AI);
6306
6307 // Build the slices using a recursive instruction-visiting builder.
6308 AllocaSlices AS(DL, AI);
6309 LLVM_DEBUG(AS.print(dbgs()));
6310 if (AS.isEscaped())
6311 return {Changed, CFGChanged};
6312
6313 if (AS.isEscapedReadOnly()) {
6314 Changed |= propagateStoredValuesToLoads(AI, AS);
6315 return {Changed, CFGChanged};
6316 }
6317
6318 // Delete all the dead users of this alloca before splitting and rewriting it.
6319 for (Instruction *DeadUser : AS.getDeadUsers()) {
6320 // Free up everything used by this instruction.
6321 for (Use &DeadOp : DeadUser->operands())
6322 clobberUse(DeadOp);
6323
6324 // Now replace the uses of this instruction.
6325 DeadUser->replaceAllUsesWith(PoisonValue::get(DeadUser->getType()));
6326
6327 // And mark it for deletion.
6328 DeadInsts.push_back(DeadUser);
6329 Changed = true;
6330 }
6331 for (Use *DeadOp : AS.getDeadOperands()) {
6332 clobberUse(*DeadOp);
6333 Changed = true;
6334 }
6335
6336 // No slices to split. Leave the dead alloca for a later pass to clean up.
6337 if (AS.begin() == AS.end())
6338 return {Changed, CFGChanged};
6339
6340 Changed |= splitAlloca(AI, AS);
6341
6342 LLVM_DEBUG(dbgs() << " Speculating PHIs\n");
6343 while (!SpeculatablePHIs.empty())
6344 speculatePHINodeLoads(IRB, *SpeculatablePHIs.pop_back_val());
6345
6346 LLVM_DEBUG(dbgs() << " Rewriting stores through PHIs\n");
6347 auto RemainingPHIsWithStoreToRewrite = PHIsWithStoreToRewrite.takeVector();
6348 while (!RemainingPHIsWithStoreToRewrite.empty()) {
6349 PHINode *PN = RemainingPHIsWithStoreToRewrite.pop_back_val();
6350 auto *SI = cast<StoreInst>(PN->user_back());
6351 CFGChanged |= rewritePHINodeStore(*PN, *SI, *DTU, Worklist);
6352 }
6353
6354 LLVM_DEBUG(dbgs() << " Rewriting Selects\n");
6355 auto RemainingSelectsToRewrite = SelectsToRewrite.takeVector();
6356 while (!RemainingSelectsToRewrite.empty()) {
6357 const auto [K, V] = RemainingSelectsToRewrite.pop_back_val();
6358 CFGChanged |=
6359 rewriteSelectInstMemOps(*K, V, IRB, PreserveCFG ? nullptr : DTU);
6360 }
6361
6362 return {Changed, CFGChanged};
6363}
6364
6365/// Delete the dead instructions accumulated in this run.
6366///
6367/// Recursively deletes the dead instructions we've accumulated. This is done
6368/// at the very end to maximize locality of the recursive delete and to
6369/// minimize the problems of invalidated instruction pointers as such pointers
6370/// are used heavily in the intermediate stages of the algorithm.
6371///
6372/// We also record the alloca instructions deleted here so that they aren't
6373/// subsequently handed to mem2reg to promote.
6374bool SROA::deleteDeadInstructions(
6375 SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) {
6376 bool Changed = false;
6377 while (!DeadInsts.empty()) {
6378 Instruction *I = dyn_cast_or_null<Instruction>(DeadInsts.pop_back_val());
6379 if (!I)
6380 continue;
6381 LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
6382
6383 // If the instruction is an alloca, find the possible dbg.declare connected
6384 // to it, and remove it too. We must do this before calling RAUW or we will
6385 // not be able to find it.
6386 if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) {
6387 DeletedAllocas.insert(AI);
6388 for (DbgVariableRecord *OldDII : findDVRDeclares(AI))
6389 OldDII->eraseFromParent();
6390 }
6391
6393 I->replaceAllUsesWith(UndefValue::get(I->getType()));
6394
6395 for (Use &Operand : I->operands())
6396 if (Instruction *U = dyn_cast<Instruction>(Operand)) {
6397 // Zero out the operand and see if it becomes trivially dead.
6398 Operand = nullptr;
6400 DeadInsts.push_back(U);
6401 }
6402
6403 ++NumDeleted;
6404 I->eraseFromParent();
6405 Changed = true;
6406 }
6407 return Changed;
6408}
6409/// Promote the allocas, using the best available technique.
6410///
6411/// This attempts to promote whatever allocas have been identified as viable in
6412/// the PromotableAllocas list. If that list is empty, there is nothing to do.
6413/// This function returns whether any promotion occurred.
6414bool SROA::promoteAllocas() {
6415 if (PromotableAllocas.empty())
6416 return false;
6417
6418 if (SROASkipMem2Reg) {
6419 LLVM_DEBUG(dbgs() << "Not promoting allocas with mem2reg!\n");
6420 } else {
6421 LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
6422 NumPromoted += PromotableAllocas.size();
6423 PromoteMemToReg(PromotableAllocas.getArrayRef(), DTU->getDomTree(), AC);
6424 }
6425
6426 PromotableAllocas.clear();
6427 return true;
6428}
6429
6430std::pair<bool /*Changed*/, bool /*CFGChanged*/> SROA::runSROA(Function &F) {
6431 LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
6432
6433 const DataLayout &DL = F.getDataLayout();
6434 BasicBlock &EntryBB = F.getEntryBlock();
6435 for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(EntryBB.end());
6436 I != E; ++I) {
6437 if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) {
6438 std::optional<TypeSize> Size = AI->getAllocationSize(DL);
6439 if (Size && Size->isScalable() && isAllocaPromotable(AI))
6440 PromotableAllocas.insert(AI);
6441 else
6442 Worklist.insert(AI);
6443 }
6444 }
6445
6446 bool Changed = false;
6447 bool CFGChanged = false;
6448 // A set of deleted alloca instruction pointers which should be removed from
6449 // the list of promotable allocas.
6450 SmallPtrSet<AllocaInst *, 4> DeletedAllocas;
6451
6452 do {
6453 while (!Worklist.empty()) {
6454 auto [IterationChanged, IterationCFGChanged] =
6455 runOnAlloca(*Worklist.pop_back_val());
6456 Changed |= IterationChanged;
6457 CFGChanged |= IterationCFGChanged;
6458
6459 Changed |= deleteDeadInstructions(DeletedAllocas);
6460
6461 // Remove the deleted allocas from various lists so that we don't try to
6462 // continue processing them.
6463 if (!DeletedAllocas.empty()) {
6464 Worklist.set_subtract(DeletedAllocas);
6465 PostPromotionWorklist.set_subtract(DeletedAllocas);
6466 PromotableAllocas.set_subtract(DeletedAllocas);
6467 DeletedAllocas.clear();
6468 }
6469 }
6470
6471 Changed |= promoteAllocas();
6472
6473 Worklist = PostPromotionWorklist;
6474 PostPromotionWorklist.clear();
6475 } while (!Worklist.empty());
6476
6477 assert((!CFGChanged || Changed) && "Can not only modify the CFG.");
6478 assert((!CFGChanged || !PreserveCFG) &&
6479 "Should not have modified the CFG when told to preserve it.");
6480
6481 if (Changed && isAssignmentTrackingEnabled(*F.getParent())) {
6482 for (auto &BB : F) {
6484 }
6485 }
6486
6487 return {Changed, CFGChanged};
6488}
6489
6493 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6494 auto [Changed, CFGChanged] =
6495 SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6496 if (!Changed)
6497 return PreservedAnalyses::all();
6499 if (!CFGChanged)
6502 return PA;
6503}
6504
6506 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6507 static_cast<PassInfoMixin<SROAPass> *>(this)->printPipeline(
6508 OS, MapClassName2PassName);
6509 OS << '<'
6510 << (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
6511 : "modify-cfg");
6512 if (Options.AggregateToVector)
6513 OS << ";aggregate-to-vector";
6514 OS << '>';
6515}
6516
6517SROAPass::SROAPass(SROAOptions Options) : Options(Options) {}
6518
6519namespace {
6520
6521/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
6522class SROALegacyPass : public FunctionPass {
6524
6525public:
6526 static char ID;
6527
6529 : FunctionPass(ID), Options(Options) {
6531 }
6532
6533 bool runOnFunction(Function &F) override {
6534 if (skipFunction(F))
6535 return false;
6536
6537 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6538 AssumptionCache &AC =
6539 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6540 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6541 auto [Changed, _] = SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6542 return Changed;
6543 }
6544
6545 void getAnalysisUsage(AnalysisUsage &AU) const override {
6546 AU.addRequired<AssumptionCacheTracker>();
6547 AU.addRequired<DominatorTreeWrapperPass>();
6548 AU.addPreserved<GlobalsAAWrapperPass>();
6549 AU.addPreserved<DominatorTreeWrapperPass>();
6550 }
6551
6552 StringRef getPassName() const override { return "SROA"; }
6553};
6554
6555} // end anonymous namespace
6556
6557char SROALegacyPass::ID = 0;
6558
6559FunctionPass *llvm::createSROAPass(bool PreserveCFG, bool AggregateToVector) {
6560 return new SROALegacyPass(SROAOptions(PreserveCFG ? SROAOptions::PreserveCFG
6562 AggregateToVector));
6563}
6564
6565INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
6566 "Scalar Replacement Of Aggregates", false, false)
6569INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
unsigned uint64_t
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition Compiler.h:678
This file contains the declarations for the subclasses of Constant, which represent the different fla...
DXIL Forward Handle Accesses
DXIL Resource Access
This file defines the DenseMap class.
static bool runOnFunction(Function &F, bool PostInlining)
Flatten the CFG
#define DEBUG_TYPE
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
#define _
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
This defines the Use class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static LVOptions Options
Definition LVOptions.cpp:25
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
print mir2vec MIR2Vec Vocabulary Printer Pass
Definition MIR2Vec.cpp:622
This file implements a map that provides insertion order iteration.
static std::optional< AllocFnsTy > getAllocationSize(const CallBase *CB, const TargetLibraryInfo *TLI)
static std::optional< uint64_t > getSizeInBytes(std::optional< uint64_t > SizeInBits)
Memory SSA
Definition MemorySSA.cpp:73
This file contains the declarations for metadata subclasses.
#define T
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define P(N)
if(PassOpts->AAPipeline)
PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file defines the PointerIntPair class.
This file provides a collection of visitors which walk the (instruction) uses of a pointer.
const SmallVectorImpl< MachineOperand > & Cond
Remove Loads Into Fake Uses
bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static void visit(BasicBlock &Start, std::function< bool(BasicBlock *)> op)
static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, uint64_t OldAllocaOffsetInBits, uint64_t SliceSizeInBits, Instruction *OldInst, Instruction *Inst, Value *Dest, Value *Value, const DataLayout &DL)
Find linked dbg.assign and generate a new one with the correct FragmentInfo.
Definition SROA.cpp:348
static VectorType * isVectorPromotionViable(Partition &P, const DataLayout &DL, unsigned VScale)
Test whether the given alloca partitioning and range of slices can be promoted to a vector.
Definition SROA.cpp:2374
static Align getAdjustedAlignment(Instruction *I, uint64_t Offset)
Compute the adjusted alignment for a load or store from an offset.
Definition SROA.cpp:2047
static VectorType * checkVectorTypesForPromotion(Partition &P, const DataLayout &DL, SmallVectorImpl< VectorType * > &CandidateTys, bool HaveCommonEltTy, Type *CommonEltTy, bool HaveVecPtrTy, bool HaveCommonVecPtrTy, VectorType *CommonVecPtrTy, unsigned VScale)
Test whether any vector type in CandidateTys is viable for promotion.
Definition SROA.cpp:2225
static std::pair< Type *, IntegerType * > findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E, uint64_t EndOffset)
Walk the range of a partitioning looking for a common type to cover this sequence of slices.
Definition SROA.cpp:1514
static Type * stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty)
Strip aggregate type wrapping.
Definition SROA.cpp:4794
static FragCalcResult calculateFragment(DILocalVariable *Variable, uint64_t NewStorageSliceOffsetInBits, uint64_t NewStorageSliceSizeInBits, std::optional< DIExpression::FragmentInfo > StorageFragment, std::optional< DIExpression::FragmentInfo > CurrentFragment, DIExpression::FragmentInfo &Target)
Definition SROA.cpp:283
static DIExpression * createOrReplaceFragment(const DIExpression *Expr, DIExpression::FragmentInfo Frag, int64_t BitExtractOffset)
Create or replace an existing fragment in a DIExpression with Frag.
Definition SROA.cpp:5847
static Value * insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old, Value *V, uint64_t Offset, const Twine &Name)
Definition SROA.cpp:2617
static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S, VectorType *Ty, uint64_t ElementSize, const DataLayout &DL, unsigned VScale)
Test whether the given slice use can be promoted to a vector.
Definition SROA.cpp:2145
static Value * getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr, APInt Offset, Type *PointerTy, const Twine &NamePrefix)
Compute an adjusted pointer from Ptr by Offset bytes where the resulting pointer has PointerTy.
Definition SROA.cpp:2036
static bool isIntegerWideningViableForSlice(const Slice &S, uint64_t AllocBeginOffset, Type *AllocaTy, const DataLayout &DL, bool &WholeAllocaOp)
Test whether a slice of an alloca is valid for integer widening.
Definition SROA.cpp:2456
static bool rewritePHINodeStore(PHINode &PN, StoreInst &SI, DomTreeUpdater &DTU, SmallSetVector< AllocaInst *, 16 > &Worklist)
Move a store through a pointer PHI onto each of the PHI's incoming edges.
Definition SROA.cpp:1767
static Value * extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex, unsigned EndIndex, const Twine &Name)
Definition SROA.cpp:2650
static Value * foldPHINodeOrSelectInst(Instruction &I)
A helper that folds a PHI node or a select.
Definition SROA.cpp:1010
static bool rewriteSelectInstMemOps(SelectInst &SI, const RewriteableMemOps &Ops, IRBuilderTy &IRB, DomTreeUpdater *DTU)
Definition SROA.cpp:2002
static void rewriteMemOpOfSelect(SelectInst &SI, T &I, SelectHandSpeculativity Spec, DomTreeUpdater &DTU)
Definition SROA.cpp:1935
static Value * foldSelectInst(SelectInst &SI)
Definition SROA.cpp:997
bool isKillAddress(const DbgVariableRecord *DVR)
Definition SROA.cpp:5810
static Value * insertVector(IRBuilderTy &IRB, Value *Old, Value *V, unsigned BeginIndex, const Twine &Name)
Definition SROA.cpp:2671
static bool isIntegerWideningViable(Partition &P, Type *AllocaTy, const DataLayout &DL)
Test whether the given alloca partition's integer operations can be widened to promotable ones.
Definition SROA.cpp:2551
static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN)
Definition SROA.cpp:1712
static VectorType * createAndCheckVectorTypesForPromotion(SetVector< Type * > &OtherTys, ArrayRef< VectorType * > CandidateTysCopy, function_ref< void(Type *)> CheckCandidateType, Partition &P, const DataLayout &DL, SmallVectorImpl< VectorType * > &CandidateTys, bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy, bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy, unsigned VScale)
Definition SROA.cpp:2330
static DebugVariable getAggregateVariable(DbgVariableRecord *DVR)
Definition SROA.cpp:329
static std::tuple< Type *, bool, VectorType * > selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI, LLVMContext &C, bool AggregateToVector)
Select a partition type for an alloca partition.
Definition SROA.cpp:5527
static bool isSafePHIToSpeculate(PHINode &PN)
PHI instructions that use an alloca and are subsequently loaded can be rewritten to load both input p...
Definition SROA.cpp:1583
static FixedVectorType * tryCanonicalizeStructToVector(StructType *STy, Partition &P, const DataLayout &DL)
Try to canonicalize a homogeneous struct partition to a vector type.
Definition SROA.cpp:5460
static Value * extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V, IntegerType *Ty, uint64_t Offset, const Twine &Name)
Definition SROA.cpp:2592
static void insertNewDbgInst(DIBuilder &DIB, DbgVariableRecord *Orig, AllocaInst *NewAddr, DIExpression *NewAddrExpr, Instruction *BeforeInst, std::optional< DIExpression::FragmentInfo > NewFragment, int64_t BitExtractAdjustment)
Insert a new DbgRecord.
Definition SROA.cpp:5911
static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI, IRBuilderTy &IRB)
Definition SROA.cpp:1897
static Value * mergeTwoVectors(Value *V0, Value *V1, const DataLayout &DL, Type *NewAIEltTy, IRBuilder<> &Builder)
This function takes two vector values and combines them into a single vector by concatenating their e...
Definition SROA.cpp:2742
const DIExpression * getAddressExpression(const DbgVariableRecord *DVR)
Definition SROA.cpp:5816
static Type * getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset, uint64_t Size)
Try to find a partition of the aggregate type passed in for a given offset and size.
Definition SROA.cpp:4832
static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy, unsigned VScale=0)
Test whether we can convert a value from the old to the new type.
Definition SROA.cpp:2057
static SelectHandSpeculativity isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG)
Definition SROA.cpp:1838
static StoreInst * getPHIStoreToRewrite(PHINode &PN, bool PreserveCFG, DominatorTree &DT)
Check whether a single store through PN can be moved onto each incoming edge.
Definition SROA.cpp:1659
static Type * findCommonTypeThroughPHIOrSelect(Instruction &I)
Find a common load/store type used through a pointer PHI or select.
Definition SROA.cpp:1490
This file provides the interface for LLVM's Scalar Replacement of Aggregates pass.
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the SparseBitVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:119
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
Virtual Register Rewriter
Value * RHS
Value * LHS
Builder for the alloca slices.
Definition SROA.cpp:1022
SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
Definition SROA.cpp:1038
An iterator over partitions of the alloca's slices.
Definition SROA.cpp:810
bool operator==(const partition_iterator &RHS) const
Definition SROA.cpp:957
partition_iterator & operator++()
Definition SROA.cpp:977
bool shouldDelete(Instruction *I) const override
Return false if a sub-class wants to keep one of the loads/stores after the SSA construction.
Definition SROA.cpp:6188
BasicLoadAndStorePromoter(ArrayRef< const Instruction * > Insts, SSAUpdater &S, Type *ZeroType)
Definition SROA.cpp:6185
Value * getValueToUseForAlloca(Instruction *I) const override
Return the value to use for the point in the code that the alloca is positioned.
Definition SROA.cpp:6192
Class for arbitrary precision integers.
Definition APInt.h:78
an instruction to allocate memory on the stack
LLVM_ABI bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
PointerType * getType() const
Overload to return most specific pointer type.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
iterator end() const
Definition ArrayRef.h:130
size_t size() const
Get the array size.
Definition ArrayRef.h:141
iterator begin() const
Definition ArrayRef.h:129
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator end()
Definition BasicBlock.h:459
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:446
LLVM_ABI InstListType::const_iterator getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
LLVM_ABI bool canSplitPredecessors() const
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
LLVM_ABI CaptureInfo getCaptureInfo(unsigned OpNo) const
Return which pointer components this operand may capture.
bool onlyReadsMemory(unsigned OpNo) const
bool isDataOperand(const Use *U) const
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static DIAssignID * getDistinct(LLVMContext &Context)
LLVM_ABI DbgRecord * insertDbgAssign(Instruction *LinkedInstr, Value *Val, DILocalVariable *SrcVar, DIExpression *ValExpr, Value *Addr, DIExpression *AddrExpr, const DILocation *DL)
Insert a new dbg_assign record.
DWARF expression.
iterator_range< expr_op_iterator > expr_ops() const
DbgVariableFragmentInfo FragmentInfo
LLVM_ABI bool startsWithDeref() const
Return whether the first element a DW_OP_deref.
static LLVM_ABI bool calculateFragmentIntersect(const DataLayout &DL, const Value *SliceStart, uint64_t SliceOffsetInBits, uint64_t SliceSizeInBits, const Value *DbgPtr, int64_t DbgPtrOffsetInBits, int64_t DbgExtractOffsetInBits, DIExpression::FragmentInfo VarFrag, std::optional< DIExpression::FragmentInfo > &Result, int64_t &OffsetFromLocationInBits)
Computes a fragment, bit-extract operation if needed, and new constant offset to describe a part of a...
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
LLVM_ABI void moveBefore(DbgRecord *MoveBefore)
DebugLoc getDebugLoc() const
void setDebugLoc(DebugLoc Loc)
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LLVM_ABI void setKillAddress()
Kill the address component.
LLVM_ABI bool isKillLocation() const
LLVM_ABI bool isKillAddress() const
Check whether this kills the address component.
LLVM_ABI void replaceVariableLocationOp(Value *OldValue, Value *NewValue, bool AllowEmpty=false)
Value * getValue(unsigned OpIdx=0) const
static LLVM_ABI DbgVariableRecord * createLinkedDVRAssign(Instruction *LinkedInstr, Value *Val, DILocalVariable *Variable, DIExpression *Expression, Value *Address, DIExpression *AddressExpression, const DILocation *DI)
LLVM_ABI void setAssignId(DIAssignID *New)
DIExpression * getExpression() const
static LLVM_ABI DbgVariableRecord * createDVRDeclare(Value *Address, DILocalVariable *DV, DIExpression *Expr, const DILocation *DI)
static LLVM_ABI DbgVariableRecord * createDbgVariableRecord(Value *Location, DILocalVariable *DV, DIExpression *Expr, const DILocation *DI)
DILocalVariable * getVariable() const
DIExpression * getAddressExpression() const
LLVM_ABI DILocation * getInlinedAt() const
Definition DebugLoc.cpp:58
Identifies a unique instance of a variable.
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
Definition DenseMap.h:285
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:258
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition DenseMap.h:254
iterator end()
Definition DenseMap.h:176
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:319
Analysis pass which computes a DominatorTree.
Definition Dominators.h:241
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:277
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Class to represent fixed width SIMD vectors.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:314
unsigned getVScaleValue() const
Return the value for vscale based on the vscale_range attribute or 0 when unknown.
const BasicBlock & getEntryBlock() const
Definition Function.h:794
LLVM_ABI bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset, function_ref< bool(Value &, APInt &)> ExternalAnalysis=nullptr) const
Accumulate the constant address offset of this GEP if possible.
Definition Operator.cpp:130
iterator_range< op_iterator > indices()
Type * getSourceElementType() const
LLVM_ABI GEPNoWrapFlags getNoWrapFlags() const
Get the nowrap flags for the GEP instruction.
This provides the default implementation of the IRBuilder 'InsertHelper' method that is called whenev...
Definition IRBuilder.h:61
virtual void InsertHelper(Instruction *I, const Twine &Name, BasicBlock::iterator InsertPt) const
Definition IRBuilder.h:65
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2903
Base class for instruction visitors.
Definition InstVisitor.h:78
LLVM_ABI unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
LLVM_ABI bool isAtomic() const LLVM_READONLY
Return true if this instruction has an AtomicOrdering of unordered or higher.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
LLVM_ABI bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
iterator_range< user_iterator > users()
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
user_iterator user_begin()
LLVM_ABI AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
@ MAX_INT_BITS
Maximum number of bits that can be specified.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
LLVM_ABI LoadAndStorePromoter(ArrayRef< const Instruction * > Insts, SSAUpdater &S, StringRef Name=StringRef())
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
void setAlignment(Align Align)
Value * getPointerOperand()
bool isVolatile() const
Return true if this is a load from a volatile memory location.
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this load instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Type * getPointerOperandType() const
static unsigned getPointerOperandIndex()
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
bool isSimple() const
Align getAlign() const
Return the alignment of the access that is being performed.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1579
LLVMContext & getContext() const
Definition Metadata.h:1245
This is the common base class for memset/memcpy/memmove.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
PointerIntPair - This class implements a pair of a pointer and small integer.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:132
PtrUseVisitor(const DataLayout &DL)
LLVM_ABI SROAPass(SROAOptions Options)
If PreserveCFG is set, then the pass is not allowed to modify CFG in any way, even if it would update...
Definition SROA.cpp:6517
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Run the pass over the function.
Definition SROA.cpp:6490
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition SROA.cpp:6505
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition SSAUpdater.h:39
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
void clear()
Completely clear the SetVector.
Definition SetVector.h:273
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:345
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
typename SuperClass::const_iterator const_iterator
typename SuperClass::iterator iterator
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
void set(unsigned Idx)
bool test(unsigned Idx) const
void reset(unsigned Idx)
An instruction for storing to memory.
void setAlignment(Align Align)
Value * getValueOperand()
static unsigned getPointerOperandIndex()
Value * getPointerOperand()
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this store instruction.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
static constexpr size_t npos
Definition StringRef.h:58
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
size_t rfind(char C, size_t From=npos) const
Search for the last character C in the string.
Definition StringRef.h:365
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition StringRef.h:290
LLVM_ABI size_t find_first_not_of(char C, size_t From=0) const
Find the first character in the string that is not C or npos if not found.
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition DataLayout.h:743
TypeSize getSizeInBytes() const
Definition DataLayout.h:752
LLVM_ABI unsigned getElementContainingOffset(uint64_t FixedOffset) const
Given a valid byte offset into the structure, returns the structure index that contains it.
TypeSize getElementOffset(unsigned Idx) const
Definition DataLayout.h:774
TypeSize getSizeInBits() const
Definition DataLayout.h:754
Class to represent struct types.
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:467
element_iterator element_end() const
ArrayRef< Type * > elements() const
element_iterator element_begin() const
bool isPacked() const
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
Type::subtype_iterator element_iterator
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:339
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI unsigned getIntegerBitWidth() const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
Definition Type.h:306
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:297
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:271
bool isTargetExtTy() const
Return true if this is a target extension type.
Definition Type.h:205
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:303
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_range operands()
Definition User.h:267
op_iterator op_begin()
Definition User.h:259
const Use & getOperandUse(unsigned i) const
Definition User.h:220
Value * getOperand(unsigned i) const
Definition User.h:207
op_iterator op_end()
Definition User.h:261
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
LLVM_ABI const Value * stripInBoundsOffsets(function_ref< void(const Value *)> Func=[](const Value *) {}) const
Strip off pointer casts and inbounds GEPs.
Definition Value.cpp:828
LLVM_ABI void dropDroppableUsesIn(User &Usr)
Remove every use of this value in User that can safely be removed.
Definition Value.cpp:215
LLVM_ABI const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr, bool LookThroughIntToPtr=false) const
Accumulate the constant offset this value has compared to a base pointer.
bool use_empty() const
Definition Value.h:348
iterator_range< use_iterator > uses()
Definition Value.h:382
bool hasName() const
Definition Value.h:263
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
static VectorType * getWithSizeAndScalar(VectorType *SizeTy, Type *EltTy)
This static method attempts to construct a VectorType with the same size-in-bits as SizeTy but with a...
static LLVM_ABI bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:168
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:165
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...
Definition iterator.h:80
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
Changed
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:81
Offsets
Offsets in bytes from the start of the input buffer.
SmallVector< DbgVariableRecord * > getDVRAssignmentMarkers(const Instruction *Inst)
Return a range of dbg_assign records for which Inst performs the assignment they encode.
Definition DebugInfo.h:212
LLVM_ABI void deleteAssignmentMarkers(const Instruction *Inst)
Delete the llvm.dbg.assign intrinsics linked to Inst.
initializer< Ty > init(const Ty &Val)
@ DW_OP_LLVM_fragment
Only used in LLVM metadata.
Definition Dwarf.h:144
@ User
could "use" a pointer
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
bool empty() const
Definition BasicBlock.h:101
iterator end() const
Definition BasicBlock.h:89
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
LLVM_ABI iterator begin() const
unsigned getNumElements(Type *Ty)
Definition SLPUtils.cpp:86
This is an optimization pass for GlobalISel generic memory operations.
static cl::opt< bool > SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(false), cl::Hidden)
Disable running mem2reg during SROA in order to test or debug SROA.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
bool operator<(int64_t V1, const APSInt &V2)
Definition APSInt.h:360
void stable_sort(R &&Range)
Definition STLExtras.h:2132
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1748
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
LLVM_ABI void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
LLVM_ABI bool isAssumeLikeIntrinsic(const Instruction *I)
Return true if it is an intrinsic that cannot be speculated but also cannot trap.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
auto successors(const MachineBasicBlock *BB)
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool operator!=(uint64_t V1, const APInt &V2)
Definition APInt.h:2139
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
LLVM_ABI std::optional< RegOrConstant > getVectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition Utils.cpp:1447
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
void * PointerTy
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
auto unique(Range &&R, Predicate P)
Definition STLExtras.h:2150
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
LLVM_ABI bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition STLExtras.h:2216
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition Local.cpp:402
bool capturesFullProvenance(CaptureComponents CC)
Definition ModRef.h:396
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
LLVM_ABI void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void initializeSROALegacyPassPass(PassRegistry &)
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
LLVM_ABI TinyPtrVector< DbgVariableRecord * > findDVRValues(Value *V)
As above, for DVRValues.
Definition DebugInfo.cpp:82
LLVM_ABI void llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, unsigned line=0)
This function calls abort(), and prints the optional message to stderr.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
constexpr int PoisonMaskElem
iterator_range(Container &&) -> iterator_range< llvm::detail::IterOfRange< Container > >
LLVM_ABI BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
DWARFExpression::Operation Op
LLVM_ABI Align tryEnforceAlignment(Value *V, Align PrefAlign, const DataLayout &DL)
If the specified pointer points to an object that we control, try to modify the object's alignment to...
Definition Local.cpp:1509
LLVM_ABI FunctionPass * createSROAPass(bool PreserveCFG=true, bool AggregateToVector=false)
Definition SROA.cpp:6559
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
LLVM_ABI TinyPtrVector< DbgVariableRecord * > findDVRDeclares(Value *V)
Finds dbg.declare records declaring local variables as living in the memory that 'V' points to.
Definition DebugInfo.cpp:48
LLVM_ABI bool isSafeToLoadUnconditionally(Value *V, Align Alignment, const APInt &Size, const SimplifyQuery &SQ)
Return true if we know that executing a load from this value cannot trap.
Definition Loads.cpp:456
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:201
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
#define NDEBUG
Definition regutils.h:48
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition Metadata.h:774
AAMDNodes shift(size_t Offset) const
Create a new AAMDNode that describes this AAMDNode after applying a constant offset to the start of t...
Definition Metadata.h:833
LLVM_ABI AAMDNodes adjustForAccess(unsigned AccessSize)
Create a new AAMDNode for accessing AccessSize bytes of this AAMDNode.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Describes an element of a Bitfield.
Definition Bitfields.h:176
static Bitfield::Type get(StorageType Packed)
Unpacks the field from the Packed value.
Definition Bitfields.h:207
static void set(StorageType &Packed, typename Bitfield::Type Value)
Sets the typed value in the provided Packed value.
Definition Bitfields.h:223