LLVM 24.0.0git
SROA.cpp
Go to the documentation of this file.
1//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This transformation implements the well known scalar replacement of
10/// aggregates transformation. It tries to identify promotable elements of an
11/// aggregate alloca, and promote them to registers. It will also try to
12/// convert uses of an element (or set of elements) of an alloca into a vector
13/// or bitfield-style integer scalar if appropriate.
14///
15/// It works to do this with minimal slicing of the alloca so that regions
16/// which are merely transferred in and out of external memory remain unchanged
17/// and are not decomposed to scalar code.
18///
19/// Because this also performs alloca promotion, it can be thought of as also
20/// serving the purpose of SSA formation. The algorithm iterates on the
21/// function until all opportunities for promotion have been realized.
22///
23//===----------------------------------------------------------------------===//
24
26#include "llvm/ADT/APInt.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/MapVector.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/SetVector.h"
36#include "llvm/ADT/Statistic.h"
37#include "llvm/ADT/StringRef.h"
38#include "llvm/ADT/Twine.h"
39#include "llvm/ADT/iterator.h"
44#include "llvm/Analysis/Loads.h"
48#include "llvm/IR/BasicBlock.h"
49#include "llvm/IR/Constant.h"
51#include "llvm/IR/Constants.h"
52#include "llvm/IR/DIBuilder.h"
53#include "llvm/IR/DataLayout.h"
54#include "llvm/IR/DebugInfo.h"
57#include "llvm/IR/Dominators.h"
58#include "llvm/IR/Function.h"
59#include "llvm/IR/GlobalAlias.h"
60#include "llvm/IR/IRBuilder.h"
61#include "llvm/IR/InstVisitor.h"
62#include "llvm/IR/Instruction.h"
65#include "llvm/IR/LLVMContext.h"
66#include "llvm/IR/Metadata.h"
67#include "llvm/IR/Module.h"
68#include "llvm/IR/Operator.h"
69#include "llvm/IR/PassManager.h"
70#include "llvm/IR/Type.h"
71#include "llvm/IR/Use.h"
72#include "llvm/IR/User.h"
73#include "llvm/IR/Value.h"
74#include "llvm/IR/ValueHandle.h"
76#include "llvm/Pass.h"
80#include "llvm/Support/Debug.h"
88#include <algorithm>
89#include <cassert>
90#include <cstddef>
91#include <cstdint>
92#include <cstring>
93#include <iterator>
94#include <string>
95#include <tuple>
96#include <utility>
97#include <variant>
98#include <vector>
99
100using namespace llvm;
101
102#define DEBUG_TYPE "sroa"
103
104STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
105STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed");
106STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca");
107STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten");
108STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition");
109STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced");
110STATISTIC(NumPromoted, "Number of allocas promoted to SSA values");
111STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
112STATISTIC(NumLoadsPredicated,
113 "Number of loads rewritten into predicated loads to allow promotion");
115 NumStoresPredicated,
116 "Number of stores rewritten into predicated stores to allow promotion");
117STATISTIC(NumDeleted, "Number of instructions deleted");
118STATISTIC(NumVectorized, "Number of vectorized aggregates");
119
120namespace llvm {
121/// Disable running mem2reg during SROA in order to test or debug SROA.
122static cl::opt<bool> SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(false),
123 cl::Hidden);
124} // namespace llvm
125
126namespace {
127
128class AllocaSliceRewriter;
129class AllocaSlices;
130class Partition;
131class Slice;
132
133class SelectHandSpeculativity {
134 unsigned char Storage = 0; // None are speculatable by default.
135 using TrueVal = Bitfield::Element<bool, 0, 1>; // Low 0'th bit.
136 using FalseVal = Bitfield::Element<bool, 1, 1>; // Low 1'th bit.
137public:
138 SelectHandSpeculativity() = default;
139 SelectHandSpeculativity &setAsSpeculatable(bool isTrueVal);
140 bool isSpeculatable(bool isTrueVal) const;
141 bool areAllSpeculatable() const;
142 bool areAnySpeculatable() const;
143 bool areNoneSpeculatable() const;
144 // For interop as int half of PointerIntPair.
145 explicit operator intptr_t() const { return static_cast<intptr_t>(Storage); }
146 explicit SelectHandSpeculativity(intptr_t Storage_) : Storage(Storage_) {}
147};
148static_assert(sizeof(SelectHandSpeculativity) == sizeof(unsigned char));
149
150using PossiblySpeculatableLoad =
152using UnspeculatableStore = StoreInst *;
153using RewriteableMemOp =
154 std::variant<PossiblySpeculatableLoad, UnspeculatableStore>;
155using RewriteableMemOps = SmallVector<RewriteableMemOp, 2>;
156using InstructionSliceMap =
158
159/// An optimization pass providing Scalar Replacement of Aggregates.
160///
161/// This pass takes allocations which can be completely analyzed (that is, they
162/// don't escape) and tries to turn them into scalar SSA values. There are
163/// a few steps to this process.
164///
165/// 1) It takes allocations of aggregates and analyzes the ways in which they
166/// are used to try to split them into smaller allocations, ideally of
167/// a single scalar data type. It will split up memcpy and memset accesses
168/// as necessary and try to isolate individual scalar accesses.
169/// 2) It will transform accesses into forms which are suitable for SSA value
170/// promotion. This can be replacing a memset with a scalar store of an
171/// integer value, or it can involve speculating operations on a PHI or
172/// select to be a PHI or select of the results.
173/// 3) Finally, this will try to detect a pattern of accesses which map cleanly
174/// onto insert and extract operations on a vector value, and convert them to
175/// this form. By doing so, it will enable promotion of vector aggregates to
176/// SSA vector values.
177class SROA {
178 LLVMContext *const C;
179 DomTreeUpdater *const DTU;
180 AssumptionCache *const AC;
181 const bool PreserveCFG;
182 const bool AggregateToVector;
183
184 /// Worklist of alloca instructions to simplify.
185 ///
186 /// Each alloca in the function is added to this. Each new alloca formed gets
187 /// added to it as well to recursively simplify unless that alloca can be
188 /// directly promoted. Finally, each time we rewrite a use of an alloca other
189 /// the one being actively rewritten, we add it back onto the list if not
190 /// already present to ensure it is re-visited.
191 SmallSetVector<AllocaInst *, 16> Worklist;
192
193 /// A collection of instructions to delete.
194 /// We try to batch deletions to simplify code and make things a bit more
195 /// efficient. We also make sure there is no dangling pointers.
196 SmallVector<WeakVH, 8> DeadInsts;
197
198 /// Post-promotion worklist.
199 ///
200 /// Sometimes we discover an alloca which has a high probability of becoming
201 /// viable for SROA after a round of promotion takes place. In those cases,
202 /// the alloca is enqueued here for re-processing.
203 ///
204 /// Note that we have to be very careful to clear allocas out of this list in
205 /// the event they are deleted.
206 SmallSetVector<AllocaInst *, 16> PostPromotionWorklist;
207
208 /// A collection of alloca instructions we can directly promote.
209 SetVector<AllocaInst *, SmallVector<AllocaInst *>,
210 SmallPtrSet<AllocaInst *, 16>, 16>
211 PromotableAllocas;
212
213 /// A worklist of PHIs to speculate prior to promoting allocas.
214 ///
215 /// All of these PHIs have been checked for the safety of speculation and by
216 /// being speculated will allow promoting allocas currently in the promotable
217 /// queue.
218 SmallSetVector<PHINode *, 8> SpeculatablePHIs;
219
220 /// A worklist of PHIs whose stores should be predicated onto incoming edges.
221 ///
222 /// All of these PHIs have been checked so that rewriting their stores will
223 /// allow the corresponding allocas to be promoted. Edges which require
224 /// splitting are only present when CFG modification is allowed.
225 SmallSetVector<PHINode *, 8> PHIsWithStoreToRewrite;
226
227 /// A worklist of select instructions to rewrite prior to promoting
228 /// allocas.
229 SmallMapVector<SelectInst *, RewriteableMemOps, 8> SelectsToRewrite;
230
231 /// Select instructions that use an alloca and are subsequently loaded can be
232 /// rewritten to load both input pointers and then select between the result,
233 /// allowing the load of the alloca to be promoted.
234 /// From this:
235 /// %P2 = select i1 %cond, ptr %Alloca, ptr %Other
236 /// %V = load <type>, ptr %P2
237 /// to:
238 /// %V1 = load <type>, ptr %Alloca -> will be mem2reg'd
239 /// %V2 = load <type>, ptr %Other
240 /// %V = select i1 %cond, <type> %V1, <type> %V2
241 ///
242 /// We can do this to a select if its only uses are loads
243 /// and if either the operand to the select can be loaded unconditionally,
244 /// or if we are allowed to perform CFG modifications.
245 static std::optional<RewriteableMemOps>
246 isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG);
247
248public:
249 SROA(LLVMContext *C, DomTreeUpdater *DTU, AssumptionCache *AC,
250 SROAOptions Options)
251 : C(C), DTU(DTU), AC(AC),
252 PreserveCFG(Options.CFG == SROAOptions::PreserveCFG),
253 AggregateToVector(Options.AggregateToVector) {}
254
255 /// Main run method used by both the SROAPass and by the legacy pass.
256 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runSROA(Function &F);
257
258private:
259 friend class AllocaSliceRewriter;
260
261 bool presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS);
262 std::pair<AllocaInst *, uint64_t>
263 rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P);
264 bool splitAlloca(AllocaInst &AI, AllocaSlices &AS);
265 bool propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS);
266 std::pair<bool /*Changed*/, bool /*CFGChanged*/> runOnAlloca(AllocaInst &AI);
267 void clobberUse(Use &U);
268 bool deleteDeadInstructions(SmallPtrSetImpl<AllocaInst *> &DeletedAllocas);
269 bool promoteAllocas();
270};
271
272} // end anonymous namespace
273
274/// Calculate the fragment of a variable to use when slicing a store
275/// based on the slice dimensions, existing fragment, and base storage
276/// fragment.
277/// Results:
278/// UseFrag - Use Target as the new fragment.
279/// UseNoFrag - The new slice already covers the whole variable.
280/// Skip - The new alloca slice doesn't include this variable.
281/// FIXME: Can we use calculateFragmentIntersect instead?
282namespace {
283enum FragCalcResult { UseFrag, UseNoFrag, Skip };
284}
285static FragCalcResult
287 uint64_t NewStorageSliceOffsetInBits,
288 uint64_t NewStorageSliceSizeInBits,
289 std::optional<DIExpression::FragmentInfo> StorageFragment,
290 std::optional<DIExpression::FragmentInfo> CurrentFragment,
292 // If the base storage describes part of the variable apply the offset and
293 // the size constraint.
294 if (StorageFragment) {
295 Target.SizeInBits =
296 std::min(NewStorageSliceSizeInBits, StorageFragment->SizeInBits);
297 Target.OffsetInBits =
298 NewStorageSliceOffsetInBits + StorageFragment->OffsetInBits;
299 } else {
300 Target.SizeInBits = NewStorageSliceSizeInBits;
301 Target.OffsetInBits = NewStorageSliceOffsetInBits;
302 }
303
304 // If this slice extracts the entirety of an independent variable from a
305 // larger alloca, do not produce a fragment expression, as the variable is
306 // not fragmented.
307 if (!CurrentFragment) {
308 if (auto Size = Variable->getSizeInBits()) {
309 // Treat the current fragment as covering the whole variable.
310 CurrentFragment = DIExpression::FragmentInfo(*Size, 0);
311 if (Target == CurrentFragment)
312 return UseNoFrag;
313 }
314 }
315
316 // No additional work to do if there isn't a fragment already, or there is
317 // but it already exactly describes the new assignment.
318 if (!CurrentFragment || *CurrentFragment == Target)
319 return UseFrag;
320
321 // Reject the target fragment if it doesn't fit wholly within the current
322 // fragment. TODO: We could instead chop up the target to fit in the case of
323 // a partial overlap.
324 if (Target.startInBits() < CurrentFragment->startInBits() ||
325 Target.endInBits() > CurrentFragment->endInBits())
326 return Skip;
327
328 // Target fits within the current fragment, return it.
329 return UseFrag;
330}
331
333 return DebugVariable(DVR->getVariable(), std::nullopt,
334 DVR->getDebugLoc().getInlinedAt());
335}
336
337/// Find linked dbg.assign and generate a new one with the correct
338/// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the
339/// value component is copied from the old dbg.assign to the new.
340/// \param OldAlloca Alloca for the variable before splitting.
341/// \param IsSplit True if the store (not necessarily alloca)
342/// is being split.
343/// \param OldAllocaOffsetInBits Offset of the slice taken from OldAlloca.
344/// \param SliceSizeInBits New number of bits being written to.
345/// \param OldInst Instruction that is being split.
346/// \param Inst New instruction performing this part of the
347/// split store.
348/// \param Dest Store destination.
349/// \param Value Stored value.
350/// \param DL Datalayout.
351static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
352 uint64_t OldAllocaOffsetInBits,
353 uint64_t SliceSizeInBits, Instruction *OldInst,
354 Instruction *Inst, Value *Dest, Value *Value,
355 const DataLayout &DL) {
356 // If we want allocas to be migrated using this helper then we need to ensure
357 // that the BaseFragments map code still works. A simple solution would be
358 // to choose to always clone alloca dbg_assigns (rather than sometimes
359 // "stealing" them).
360 assert(!isa<AllocaInst>(Inst) && "Unexpected alloca");
361
362 auto DVRAssignMarkerRange = at::getDVRAssignmentMarkers(OldInst);
363 // Nothing to do if OldInst has no linked dbg.assign intrinsics.
364 if (DVRAssignMarkerRange.empty())
365 return;
366
367 LLVM_DEBUG(dbgs() << " migrateDebugInfo\n");
368 LLVM_DEBUG(dbgs() << " OldAlloca: " << *OldAlloca << "\n");
369 LLVM_DEBUG(dbgs() << " IsSplit: " << IsSplit << "\n");
370 LLVM_DEBUG(dbgs() << " OldAllocaOffsetInBits: " << OldAllocaOffsetInBits
371 << "\n");
372 LLVM_DEBUG(dbgs() << " SliceSizeInBits: " << SliceSizeInBits << "\n");
373 LLVM_DEBUG(dbgs() << " OldInst: " << *OldInst << "\n");
374 LLVM_DEBUG(dbgs() << " Inst: " << *Inst << "\n");
375 LLVM_DEBUG(dbgs() << " Dest: " << *Dest << "\n");
376 if (Value)
377 LLVM_DEBUG(dbgs() << " Value: " << *Value << "\n");
378
379 /// Map of aggregate variables to their fragment associated with OldAlloca.
381 BaseFragments;
382 for (auto *DVR : at::getDVRAssignmentMarkers(OldAlloca))
383 BaseFragments[getAggregateVariable(DVR)] =
384 DVR->getExpression()->getFragmentInfo();
385
386 // The new inst needs a DIAssignID unique metadata tag (if OldInst has
387 // one). It shouldn't already have one: assert this assumption.
388 assert(!Inst->getMetadata(LLVMContext::MD_DIAssignID));
389 DIAssignID *NewID = nullptr;
390 auto &Ctx = Inst->getContext();
391 DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false);
392 assert(OldAlloca->isStaticAlloca());
393
394 auto MigrateDbgAssign = [&](DbgVariableRecord *DbgAssign) {
395 LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign
396 << "\n");
397 auto *Expr = DbgAssign->getExpression();
398 bool SetKillLocation = false;
399
400 if (IsSplit) {
401 std::optional<DIExpression::FragmentInfo> BaseFragment;
402 {
403 auto R = BaseFragments.find(getAggregateVariable(DbgAssign));
404 if (R == BaseFragments.end())
405 return;
406 BaseFragment = R->second;
407 }
408 std::optional<DIExpression::FragmentInfo> CurrentFragment =
409 Expr->getFragmentInfo();
410 DIExpression::FragmentInfo NewFragment;
411 FragCalcResult Result = calculateFragment(
412 DbgAssign->getVariable(), OldAllocaOffsetInBits, SliceSizeInBits,
413 BaseFragment, CurrentFragment, NewFragment);
414
415 if (Result == Skip)
416 return;
417 if (Result == UseFrag && !(NewFragment == CurrentFragment)) {
418 if (CurrentFragment) {
419 // Rewrite NewFragment to be relative to the existing one (this is
420 // what createFragmentExpression wants). CalculateFragment has
421 // already resolved the size for us. FIXME: Should it return the
422 // relative fragment too?
423 NewFragment.OffsetInBits -= CurrentFragment->OffsetInBits;
424 }
425 // Add the new fragment info to the existing expression if possible.
427 Expr, NewFragment.OffsetInBits, NewFragment.SizeInBits)) {
428 Expr = *E;
429 } else {
430 // Otherwise, add the new fragment info to an empty expression and
431 // discard the value component of this dbg.assign as the value cannot
432 // be computed with the new fragment.
434 DIExpression::get(Expr->getContext(), {}),
435 NewFragment.OffsetInBits, NewFragment.SizeInBits);
436 SetKillLocation = true;
437 }
438 }
439 }
440
441 // If we haven't created a DIAssignID ID do that now and attach it to Inst.
442 if (!NewID) {
443 NewID = DIAssignID::getDistinct(Ctx);
444 Inst->setMetadata(LLVMContext::MD_DIAssignID, NewID);
445 }
446
447 DbgVariableRecord *NewAssign;
448 if (IsSplit) {
449 ::Value *NewValue = Value ? Value : DbgAssign->getValue();
451 Inst, NewValue, DbgAssign->getVariable(), Expr, Dest,
452 DIExpression::get(Expr->getContext(), {}), DbgAssign->getDebugLoc()));
453 } else {
454 // The store is not split, simply steal the existing dbg_assign.
455 NewAssign = DbgAssign;
456 NewAssign->setAssignId(NewID); // FIXME: Can we avoid generating new IDs?
457 NewAssign->setAddress(Dest);
458 if (Value)
459 NewAssign->replaceVariableLocationOp(0u, Value);
460 assert(Expr == NewAssign->getExpression());
461 }
462
463 // If we've updated the value but the original dbg.assign has an arglist
464 // then kill it now - we can't use the requested new value.
465 // We can't replace the DIArgList with the new value as it'd leave
466 // the DIExpression in an invalid state (DW_OP_LLVM_arg operands without
467 // an arglist). And we can't keep the DIArgList in case the linked store
468 // is being split - in which case the DIArgList + expression may no longer
469 // be computing the correct value.
470 // This should be a very rare situation as it requires the value being
471 // stored to differ from the dbg.assign (i.e., the value has been
472 // represented differently in the debug intrinsic for some reason).
473 SetKillLocation |=
474 Value && (DbgAssign->hasArgList() ||
475 !DbgAssign->getExpression()->isSingleLocationExpression());
476 if (SetKillLocation)
477 NewAssign->setKillLocation();
478
479 // We could use more precision here at the cost of some additional (code)
480 // complexity - if the original dbg.assign was adjacent to its store, we
481 // could position this new dbg.assign adjacent to its store rather than the
482 // old dbg.assgn. That would result in interleaved dbg.assigns rather than
483 // what we get now:
484 // split store !1
485 // split store !2
486 // dbg.assign !1
487 // dbg.assign !2
488 // This (current behaviour) results results in debug assignments being
489 // noted as slightly offset (in code) from the store. In practice this
490 // should have little effect on the debugging experience due to the fact
491 // that all the split stores should get the same line number.
492 if (NewAssign != DbgAssign) {
493 NewAssign->moveBefore(DbgAssign->getIterator());
494 NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
495 }
496 LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
497 };
498
499 for_each(DVRAssignMarkerRange, MigrateDbgAssign);
500}
501
502namespace {
503
504/// A custom IRBuilder inserter which prefixes all names, but only in
505/// Assert builds.
506class IRBuilderPrefixedInserter final : public IRBuilderDefaultInserter {
507 std::string Prefix;
508
509 Twine getNameWithPrefix(const Twine &Name) const {
510 return Name.isTriviallyEmpty() ? Name : Prefix + Name;
511 }
512
513public:
514 void SetNamePrefix(const Twine &P) { Prefix = P.str(); }
515
516 void InsertHelper(Instruction *I, const Twine &Name,
517 BasicBlock::iterator InsertPt) const override {
518 IRBuilderDefaultInserter::InsertHelper(I, getNameWithPrefix(Name),
519 InsertPt);
520 }
521};
522
523/// Provide a type for IRBuilder that drops names in release builds.
525
526/// A used slice of an alloca.
527///
528/// This structure represents a slice of an alloca used by some instruction. It
529/// stores both the begin and end offsets of this use, a pointer to the use
530/// itself, and a flag indicating whether we can classify the use as splittable
531/// or not when forming partitions of the alloca.
532class Slice {
533 /// The beginning offset of the range.
534 uint64_t BeginOffset = 0;
535
536 /// The ending offset, not included in the range.
537 uint64_t EndOffset = 0;
538
539 /// Storage for both the use of this slice and whether it can be
540 /// split.
541 PointerIntPair<Use *, 1, bool> UseAndIsSplittable;
542
543public:
544 Slice() = default;
545
546 Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable)
547 : BeginOffset(BeginOffset), EndOffset(EndOffset),
548 UseAndIsSplittable(U, IsSplittable) {}
549
550 uint64_t beginOffset() const { return BeginOffset; }
551 uint64_t endOffset() const { return EndOffset; }
552
553 bool isSplittable() const { return UseAndIsSplittable.getInt(); }
554 void makeUnsplittable() { UseAndIsSplittable.setInt(false); }
555
556 Use *getUse() const { return UseAndIsSplittable.getPointer(); }
557
558 bool isDead() const { return getUse() == nullptr; }
559 void kill() { UseAndIsSplittable.setPointer(nullptr); }
560
561 /// Support for ordering ranges.
562 ///
563 /// This provides an ordering over ranges such that start offsets are
564 /// always increasing, and within equal start offsets, the end offsets are
565 /// decreasing. Thus the spanning range comes first in a cluster with the
566 /// same start position.
567 bool operator<(const Slice &RHS) const {
568 if (beginOffset() < RHS.beginOffset())
569 return true;
570 if (beginOffset() > RHS.beginOffset())
571 return false;
572 if (isSplittable() != RHS.isSplittable())
573 return !isSplittable();
574 if (endOffset() > RHS.endOffset())
575 return true;
576 return false;
577 }
578
579 /// Support comparison with a single offset to allow binary searches.
580 [[maybe_unused]] friend bool operator<(const Slice &LHS, uint64_t RHSOffset) {
581 return LHS.beginOffset() < RHSOffset;
582 }
583 [[maybe_unused]] friend bool operator<(uint64_t LHSOffset, const Slice &RHS) {
584 return LHSOffset < RHS.beginOffset();
585 }
586
587 bool operator==(const Slice &RHS) const {
588 return isSplittable() == RHS.isSplittable() &&
589 beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset();
590 }
591 bool operator!=(const Slice &RHS) const { return !operator==(RHS); }
592};
593
594/// Representation of the alloca slices.
595///
596/// This class represents the slices of an alloca which are formed by its
597/// various uses. If a pointer escapes, we can't fully build a representation
598/// for the slices used and we reflect that in this structure. The uses are
599/// stored, sorted by increasing beginning offset and with unsplittable slices
600/// starting at a particular offset before splittable slices.
601class AllocaSlices {
602public:
603 /// Construct the slices of a particular alloca.
604 AllocaSlices(const DataLayout &DL, AllocaInst &AI);
605
606 /// Test whether a pointer to the allocation escapes our analysis.
607 ///
608 /// If this is true, the slices are never fully built and should be
609 /// ignored.
610 bool isEscaped() const { return PointerEscapingInstr; }
611 bool isEscapedReadOnly() const { return PointerEscapingInstrReadOnly; }
612
613 /// Support for iterating over the slices.
614 /// @{
615 using iterator = SmallVectorImpl<Slice>::iterator;
616 using range = iterator_range<iterator>;
617
618 iterator begin() { return Slices.begin(); }
619 iterator end() { return Slices.end(); }
620
621 using const_iterator = SmallVectorImpl<Slice>::const_iterator;
622 using const_range = iterator_range<const_iterator>;
623
624 const_iterator begin() const { return Slices.begin(); }
625 const_iterator end() const { return Slices.end(); }
626 /// @}
627
628 /// Erase a range of slices.
629 void erase(iterator Start, iterator Stop) { Slices.erase(Start, Stop); }
630
631 /// Insert new slices for this alloca.
632 ///
633 /// This moves the slices into the alloca's slices collection, and re-sorts
634 /// everything so that the usual ordering properties of the alloca's slices
635 /// hold.
636 void insert(ArrayRef<Slice> NewSlices) {
637 int OldSize = Slices.size();
638 Slices.append(NewSlices.begin(), NewSlices.end());
639 auto SliceI = Slices.begin() + OldSize;
640 std::stable_sort(SliceI, Slices.end());
641 std::inplace_merge(Slices.begin(), SliceI, Slices.end());
642 }
643
644 // Forward declare the iterator and range accessor for walking the
645 // partitions.
646 class partition_iterator;
648
649 /// Access the dead users for this alloca.
650 ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; }
651
652 /// Access Uses that should be dropped if the alloca is promotable.
653 ArrayRef<Use *> getDeadUsesIfPromotable() const {
654 return DeadUseIfPromotable;
655 }
656
657 /// Access the dead operands referring to this alloca.
658 ///
659 /// These are operands which have cannot actually be used to refer to the
660 /// alloca as they are outside its range and the user doesn't correct for
661 /// that. These mostly consist of PHI node inputs and the like which we just
662 /// need to replace with undef.
663 ArrayRef<Use *> getDeadOperands() const { return DeadOperands; }
664
665#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
666 void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const;
667 void printSlice(raw_ostream &OS, const_iterator I,
668 StringRef Indent = " ") const;
669 void printUse(raw_ostream &OS, const_iterator I,
670 StringRef Indent = " ") const;
671 void print(raw_ostream &OS) const;
672 void dump(const_iterator I) const;
673 void dump() const;
674#endif
675
676private:
677 template <typename DerivedT, typename RetT = void> class BuilderBase;
678 class SliceBuilder;
679
680 friend class AllocaSlices::SliceBuilder;
681
682#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
683 /// Handle to alloca instruction to simplify method interfaces.
684 AllocaInst &AI;
685#endif
686
687 /// The instruction responsible for this alloca not having a known set
688 /// of slices.
689 ///
690 /// When an instruction (potentially) escapes the pointer to the alloca, we
691 /// store a pointer to that here and abort trying to form slices of the
692 /// alloca. This will be null if the alloca slices are analyzed successfully.
693 Instruction *PointerEscapingInstr;
694 Instruction *PointerEscapingInstrReadOnly;
695
696 /// The slices of the alloca.
697 ///
698 /// We store a vector of the slices formed by uses of the alloca here. This
699 /// vector is sorted by increasing begin offset, and then the unsplittable
700 /// slices before the splittable ones. See the Slice inner class for more
701 /// details.
703
704 /// Instructions which will become dead if we rewrite the alloca.
705 ///
706 /// Note that these are not separated by slice. This is because we expect an
707 /// alloca to be completely rewritten or not rewritten at all. If rewritten,
708 /// all these instructions can simply be removed and replaced with poison as
709 /// they come from outside of the allocated space.
710 SmallVector<Instruction *, 8> DeadUsers;
711
712 /// Uses which will become dead if can promote the alloca.
713 SmallVector<Use *, 8> DeadUseIfPromotable;
714
715 /// Operands which will become dead if we rewrite the alloca.
716 ///
717 /// These are operands that in their particular use can be replaced with
718 /// poison when we rewrite the alloca. These show up in out-of-bounds inputs
719 /// to PHI nodes and the like. They aren't entirely dead (there might be
720 /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
721 /// want to swap this particular input for poison to simplify the use lists of
722 /// the alloca.
723 SmallVector<Use *, 8> DeadOperands;
724};
725
726/// A partition of the slices.
727///
728/// An ephemeral representation for a range of slices which can be viewed as
729/// a partition of the alloca. This range represents a span of the alloca's
730/// memory which cannot be split, and provides access to all of the slices
731/// overlapping some part of the partition.
732///
733/// Objects of this type are produced by traversing the alloca's slices, but
734/// are only ephemeral and not persistent.
735class Partition {
736private:
737 friend class AllocaSlices;
738 friend class AllocaSlices::partition_iterator;
739
740 using iterator = AllocaSlices::iterator;
741
742 /// The beginning and ending offsets of the alloca for this
743 /// partition.
744 uint64_t BeginOffset = 0, EndOffset = 0;
745
746 /// The start and end iterators of this partition.
747 iterator SI, SJ;
748
749 /// A collection of split slice tails overlapping the partition.
750 SmallVector<Slice *, 4> SplitTails;
751
752 /// Raw constructor builds an empty partition starting and ending at
753 /// the given iterator.
754 Partition(iterator SI) : SI(SI), SJ(SI) {}
755
756public:
757 /// The start offset of this partition.
758 ///
759 /// All of the contained slices start at or after this offset.
760 uint64_t beginOffset() const { return BeginOffset; }
761
762 /// The end offset of this partition.
763 ///
764 /// All of the contained slices end at or before this offset.
765 uint64_t endOffset() const { return EndOffset; }
766
767 /// The size of the partition.
768 ///
769 /// Note that this can never be zero.
770 uint64_t size() const {
771 assert(BeginOffset < EndOffset && "Partitions must span some bytes!");
772 return EndOffset - BeginOffset;
773 }
774
775 /// Test whether this partition contains no slices, and merely spans
776 /// a region occupied by split slices.
777 bool empty() const { return SI == SJ; }
778
779 /// \name Iterate slices that start within the partition.
780 /// These may be splittable or unsplittable. They have a begin offset >= the
781 /// partition begin offset.
782 /// @{
783 // FIXME: We should probably define a "concat_iterator" helper and use that
784 // to stitch together pointee_iterators over the split tails and the
785 // contiguous iterators of the partition. That would give a much nicer
786 // interface here. We could then additionally expose filtered iterators for
787 // split, unsplit, and unsplittable splices based on the usage patterns.
788 iterator begin() const { return SI; }
789 iterator end() const { return SJ; }
790 /// @}
791
792 /// Get the sequence of split slice tails.
793 ///
794 /// These tails are of slices which start before this partition but are
795 /// split and overlap into the partition. We accumulate these while forming
796 /// partitions.
797 ArrayRef<Slice *> splitSliceTails() const { return SplitTails; }
798};
799
800} // end anonymous namespace
801
802/// An iterator over partitions of the alloca's slices.
803///
804/// This iterator implements the core algorithm for partitioning the alloca's
805/// slices. It is a forward iterator as we don't support backtracking for
806/// efficiency reasons, and re-use a single storage area to maintain the
807/// current set of split slices.
808///
809/// It is templated on the slice iterator type to use so that it can operate
810/// with either const or non-const slice iterators.
812 : public iterator_facade_base<partition_iterator, std::forward_iterator_tag,
813 Partition> {
814 friend class AllocaSlices;
815
816 /// Most of the state for walking the partitions is held in a class
817 /// with a nice interface for examining them.
818 Partition P;
819
820 /// We need to keep the end of the slices to know when to stop.
821 AllocaSlices::iterator SE;
822
823 /// We also need to keep track of the maximum split end offset seen.
824 /// FIXME: Do we really?
825 uint64_t MaxSplitSliceEndOffset = 0;
826
827 /// Sets the partition to be empty at given iterator, and sets the
828 /// end iterator.
829 partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE)
830 : P(SI), SE(SE) {
831 // If not already at the end, advance our state to form the initial
832 // partition.
833 if (SI != SE)
834 advance();
835 }
836
837 /// Advance the iterator to the next partition.
838 ///
839 /// Requires that the iterator not be at the end of the slices.
840 void advance() {
841 assert((P.SI != SE || !P.SplitTails.empty()) &&
842 "Cannot advance past the end of the slices!");
843
844 // Clear out any split uses which have ended.
845 if (!P.SplitTails.empty()) {
846 if (P.EndOffset >= MaxSplitSliceEndOffset) {
847 // If we've finished all splits, this is easy.
848 P.SplitTails.clear();
849 MaxSplitSliceEndOffset = 0;
850 } else {
851 // Remove the uses which have ended in the prior partition. This
852 // cannot change the max split slice end because we just checked that
853 // the prior partition ended prior to that max.
854 llvm::erase_if(P.SplitTails,
855 [&](Slice *S) { return S->endOffset() <= P.EndOffset; });
856 assert(llvm::any_of(P.SplitTails,
857 [&](Slice *S) {
858 return S->endOffset() == MaxSplitSliceEndOffset;
859 }) &&
860 "Could not find the current max split slice offset!");
861 assert(llvm::all_of(P.SplitTails,
862 [&](Slice *S) {
863 return S->endOffset() <= MaxSplitSliceEndOffset;
864 }) &&
865 "Max split slice end offset is not actually the max!");
866 }
867 }
868
869 // If P.SI is already at the end, then we've cleared the split tail and
870 // now have an end iterator.
871 if (P.SI == SE) {
872 assert(P.SplitTails.empty() && "Failed to clear the split slices!");
873 return;
874 }
875
876 // If we had a non-empty partition previously, set up the state for
877 // subsequent partitions.
878 if (P.SI != P.SJ) {
879 // Accumulate all the splittable slices which started in the old
880 // partition into the split list.
881 for (Slice &S : P)
882 if (S.isSplittable() && S.endOffset() > P.EndOffset) {
883 P.SplitTails.push_back(&S);
884 MaxSplitSliceEndOffset =
885 std::max(S.endOffset(), MaxSplitSliceEndOffset);
886 }
887
888 // Start from the end of the previous partition.
889 P.SI = P.SJ;
890
891 // If P.SI is now at the end, we at most have a tail of split slices.
892 if (P.SI == SE) {
893 P.BeginOffset = P.EndOffset;
894 P.EndOffset = MaxSplitSliceEndOffset;
895 return;
896 }
897
898 // If the we have split slices and the next slice is after a gap and is
899 // not splittable immediately form an empty partition for the split
900 // slices up until the next slice begins.
901 if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset &&
902 !P.SI->isSplittable()) {
903 P.BeginOffset = P.EndOffset;
904 P.EndOffset = P.SI->beginOffset();
905 return;
906 }
907 }
908
909 // OK, we need to consume new slices. Set the end offset based on the
910 // current slice, and step SJ past it. The beginning offset of the
911 // partition is the beginning offset of the next slice unless we have
912 // pre-existing split slices that are continuing, in which case we begin
913 // at the prior end offset.
914 P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset;
915 P.EndOffset = P.SI->endOffset();
916 ++P.SJ;
917
918 // There are two strategies to form a partition based on whether the
919 // partition starts with an unsplittable slice or a splittable slice.
920 if (!P.SI->isSplittable()) {
921 // When we're forming an unsplittable region, it must always start at
922 // the first slice and will extend through its end.
923 assert(P.BeginOffset == P.SI->beginOffset());
924
925 // Form a partition including all of the overlapping slices with this
926 // unsplittable slice.
927 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
928 if (!P.SJ->isSplittable())
929 P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset());
930 ++P.SJ;
931 }
932
933 // We have a partition across a set of overlapping unsplittable
934 // partitions.
935 return;
936 }
937
938 // If we're starting with a splittable slice, then we need to form
939 // a synthetic partition spanning it and any other overlapping splittable
940 // splices.
941 assert(P.SI->isSplittable() && "Forming a splittable partition!");
942
943 // Collect all of the overlapping splittable slices.
944 while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset &&
945 P.SJ->isSplittable()) {
946 P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset());
947 ++P.SJ;
948 }
949
950 // Back upiP.EndOffset if we ended the span early when encountering an
951 // unsplittable slice. This synthesizes the early end offset of
952 // a partition spanning only splittable slices.
953 if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) {
954 assert(!P.SJ->isSplittable());
955 P.EndOffset = P.SJ->beginOffset();
956 }
957 }
958
959public:
960 bool operator==(const partition_iterator &RHS) const {
961 assert(SE == RHS.SE &&
962 "End iterators don't match between compared partition iterators!");
963
964 // The observed positions of partitions is marked by the P.SI iterator and
965 // the emptiness of the split slices. The latter is only relevant when
966 // P.SI == SE, as the end iterator will additionally have an empty split
967 // slices list, but the prior may have the same P.SI and a tail of split
968 // slices.
969 if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) {
970 assert(P.SJ == RHS.P.SJ &&
971 "Same set of slices formed two different sized partitions!");
972 assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&
973 "Same slice position with differently sized non-empty split "
974 "slice tails!");
975 return true;
976 }
977 return false;
978 }
979
980 partition_iterator &operator++() {
981 advance();
982 return *this;
983 }
984
985 Partition &operator*() { return P; }
986};
987
988/// A forward range over the partitions of the alloca's slices.
989///
990/// This accesses an iterator range over the partitions of the alloca's
991/// slices. It computes these partitions on the fly based on the overlapping
992/// offsets of the slices and the ability to split them. It will visit "empty"
993/// partitions to cover regions of the alloca only accessed via split
994/// slices.
995iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() {
996 return make_range(partition_iterator(begin(), end()),
997 partition_iterator(end(), end()));
998}
999
1001 // If the condition being selected on is a constant or the same value is
1002 // being selected between, fold the select. Yes this does (rarely) happen
1003 // early on.
1004 if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition()))
1005 return SI.getOperand(1 + CI->isZero());
1006 if (SI.getOperand(1) == SI.getOperand(2))
1007 return SI.getOperand(1);
1008
1009 return nullptr;
1010}
1011
1012/// A helper that folds a PHI node or a select.
1014 if (PHINode *PN = dyn_cast<PHINode>(&I)) {
1015 // If PN merges together the same value, return that value.
1016 return PN->hasConstantValue();
1017 }
1019}
1020
1021/// Builder for the alloca slices.
1022///
1023/// This class builds a set of alloca slices by recursively visiting the uses
1024/// of an alloca and making a slice for each load and store at each offset.
1025class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> {
1026 friend class PtrUseVisitor<SliceBuilder>;
1027 friend class InstVisitor<SliceBuilder>;
1028
1029 using Base = PtrUseVisitor<SliceBuilder>;
1030
1031 const uint64_t AllocSize;
1032 AllocaSlices &AS;
1033
1034 SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap;
1036
1037 /// Set to de-duplicate dead instructions found in the use walk.
1038 SmallPtrSet<Instruction *, 4> VisitedDeadInsts;
1039
1040public:
1041 SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
1043 AllocSize(AI.getAllocationSize(DL)->getFixedValue()), AS(AS) {}
1044
1045private:
1046 void markAsDead(Instruction &I) {
1047 if (VisitedDeadInsts.insert(&I).second)
1048 AS.DeadUsers.push_back(&I);
1049 }
1050
1051 void insertUse(Instruction &I, const APInt &Offset, uint64_t Size,
1052 bool IsSplittable = false) {
1053 // Completely skip uses which have a zero size or start either before or
1054 // past the end of the allocation.
1055 if (Size == 0 || Offset.uge(AllocSize)) {
1056 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"
1057 << Offset
1058 << " which has zero size or starts outside of the "
1059 << AllocSize << " byte alloca:\n"
1060 << " alloca: " << AS.AI << "\n"
1061 << " use: " << I << "\n");
1062 return markAsDead(I);
1063 }
1064
1065 uint64_t BeginOffset = Offset.getZExtValue();
1066 uint64_t EndOffset = BeginOffset + Size;
1067
1068 // Clamp the end offset to the end of the allocation. Note that this is
1069 // formulated to handle even the case where "BeginOffset + Size" overflows.
1070 // This may appear superficially to be something we could ignore entirely,
1071 // but that is not so! There may be widened loads or PHI-node uses where
1072 // some instructions are dead but not others. We can't completely ignore
1073 // them, and so have to record at least the information here.
1074 assert(AllocSize >= BeginOffset); // Established above.
1075 if (Size > AllocSize - BeginOffset) {
1076 LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"
1077 << Offset << " to remain within the " << AllocSize
1078 << " byte alloca:\n"
1079 << " alloca: " << AS.AI << "\n"
1080 << " use: " << I << "\n");
1081 EndOffset = AllocSize;
1082 }
1083
1084 AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable));
1085 }
1086
1087 void visitBitCastInst(BitCastInst &BC) {
1088 if (BC.use_empty())
1089 return markAsDead(BC);
1090
1091 return Base::visitBitCastInst(BC);
1092 }
1093
1094 void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
1095 if (ASC.use_empty())
1096 return markAsDead(ASC);
1097
1098 return Base::visitAddrSpaceCastInst(ASC);
1099 }
1100
1101 void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
1102 if (GEPI.use_empty())
1103 return markAsDead(GEPI);
1104
1105 return Base::visitGetElementPtrInst(GEPI);
1106 }
1107
1108 void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset,
1109 uint64_t Size, bool IsVolatile) {
1110 // We allow splitting of non-volatile loads and stores where the type is an
1111 // integer type. These may be used to implement 'memcpy' or other "transfer
1112 // of bits" patterns.
1113 bool IsSplittable =
1114 Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty);
1115
1116 insertUse(I, Offset, Size, IsSplittable);
1117 }
1118
1119 void visitLoadInst(LoadInst &LI) {
1120 assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&
1121 "All simple FCA loads should have been pre-split");
1122
1123 // If there is a load with an unknown offset, we can still perform store
1124 // to load forwarding for other known-offset loads.
1125 if (!IsOffsetKnown)
1126 return PI.setEscapedReadOnly(&LI);
1127
1128 TypeSize Size = DL.getTypeStoreSize(LI.getType());
1129 if (Size.isScalable()) {
1130 unsigned VScale = LI.getFunction()->getVScaleValue();
1131 if (!VScale)
1132 return PI.setAborted(&LI);
1133
1134 Size = TypeSize::getFixed(Size.getKnownMinValue() * VScale);
1135 }
1136
1137 return handleLoadOrStore(LI.getType(), LI, Offset, Size.getFixedValue(),
1138 LI.isVolatile());
1139 }
1140
1141 void visitStoreInst(StoreInst &SI) {
1142 Value *ValOp = SI.getValueOperand();
1143 if (ValOp == *U)
1144 return PI.setEscapedAndAborted(&SI);
1145 if (!IsOffsetKnown)
1146 return PI.setAborted(&SI);
1147
1148 TypeSize StoreSize = DL.getTypeStoreSize(ValOp->getType());
1149 if (StoreSize.isScalable()) {
1150 unsigned VScale = SI.getFunction()->getVScaleValue();
1151 if (!VScale)
1152 return PI.setAborted(&SI);
1153
1154 StoreSize = TypeSize::getFixed(StoreSize.getKnownMinValue() * VScale);
1155 }
1156
1157 uint64_t Size = StoreSize.getFixedValue();
1158
1159 // If this memory access can be shown to *statically* extend outside the
1160 // bounds of the allocation, it's behavior is undefined, so simply
1161 // ignore it. Note that this is more strict than the generic clamping
1162 // behavior of insertUse. We also try to handle cases which might run the
1163 // risk of overflow.
1164 // FIXME: We should instead consider the pointer to have escaped if this
1165 // function is being instrumented for addressing bugs or race conditions.
1166 if (Size > AllocSize || Offset.ugt(AllocSize - Size)) {
1167 LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"
1168 << Offset << " which extends past the end of the "
1169 << AllocSize << " byte alloca:\n"
1170 << " alloca: " << AS.AI << "\n"
1171 << " use: " << SI << "\n");
1172 return markAsDead(SI);
1173 }
1174
1175 assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&
1176 "All simple FCA stores should have been pre-split");
1177 handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile());
1178 }
1179
1180 void visitMemSetInst(MemSetInst &II) {
1181 assert(II.getRawDest() == *U && "Pointer use is not the destination?");
1182 ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
1183 if ((Length && Length->getValue() == 0) ||
1184 (IsOffsetKnown && Offset.uge(AllocSize)))
1185 // Zero-length mem transfer intrinsics can be ignored entirely.
1186 return markAsDead(II);
1187
1188 if (!IsOffsetKnown)
1189 return PI.setAborted(&II);
1190
1191 insertUse(II, Offset,
1192 Length ? Length->getLimitedValue()
1193 : AllocSize - Offset.getLimitedValue(),
1194 (bool)Length);
1195 }
1196
1197 void visitMemTransferInst(MemTransferInst &II) {
1198 ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
1199 if (Length && Length->getValue() == 0)
1200 // Zero-length mem transfer intrinsics can be ignored entirely.
1201 return markAsDead(II);
1202
1203 // Because we can visit these intrinsics twice, also check to see if the
1204 // first time marked this instruction as dead. If so, skip it.
1205 if (VisitedDeadInsts.count(&II))
1206 return;
1207
1208 if (!IsOffsetKnown)
1209 return PI.setAborted(&II);
1210
1211 // This side of the transfer is completely out-of-bounds, and so we can
1212 // nuke the entire transfer. However, we also need to nuke the other side
1213 // if already added to our partitions.
1214 // FIXME: Yet another place we really should bypass this when
1215 // instrumenting for ASan.
1216 if (Offset.uge(AllocSize)) {
1217 auto MTPI = MemTransferSliceMap.find(&II);
1218 if (MTPI != MemTransferSliceMap.end())
1219 AS.Slices[MTPI->second].kill();
1220 return markAsDead(II);
1221 }
1222
1223 uint64_t RawOffset = Offset.getLimitedValue();
1224 uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset;
1225
1226 // Check for the special case where the same exact value is used for both
1227 // source and dest.
1228 if (*U == II.getRawDest() && *U == II.getRawSource()) {
1229 // For non-volatile transfers this is a no-op.
1230 if (!II.isVolatile())
1231 return markAsDead(II);
1232
1233 return insertUse(II, Offset, Size, /*IsSplittable=*/false);
1234 }
1235
1236 // If we have seen both source and destination for a mem transfer, then
1237 // they both point to the same alloca.
1238 bool Inserted;
1239 SmallDenseMap<Instruction *, unsigned>::iterator MTPI;
1240 std::tie(MTPI, Inserted) =
1241 MemTransferSliceMap.insert(std::make_pair(&II, AS.Slices.size()));
1242 unsigned PrevIdx = MTPI->second;
1243 if (!Inserted) {
1244 Slice &PrevP = AS.Slices[PrevIdx];
1245
1246 // Check if the begin offsets match and this is a non-volatile transfer.
1247 // In that case, we can completely elide the transfer.
1248 if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) {
1249 PrevP.kill();
1250 return markAsDead(II);
1251 }
1252
1253 // Otherwise we have an offset transfer within the same alloca. We can't
1254 // split those.
1255 PrevP.makeUnsplittable();
1256 }
1257
1258 // Insert the use now that we've fixed up the splittable nature.
1259 insertUse(II, Offset, Size, /*IsSplittable=*/Inserted && Length);
1260
1261 // Check that we ended up with a valid index in the map.
1262 assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&
1263 "Map index doesn't point back to a slice with this user.");
1264 }
1265
1266 // Disable SRoA for any intrinsics except for lifetime invariants.
1267 // FIXME: What about debug intrinsics? This matches old behavior, but
1268 // doesn't make sense.
1269 void visitIntrinsicInst(IntrinsicInst &II) {
1270 if (II.isDroppable()) {
1271 AS.DeadUseIfPromotable.push_back(U);
1272 return;
1273 }
1274
1275 if (!IsOffsetKnown)
1276 return PI.setAborted(&II);
1277
1278 if (II.isLifetimeStartOrEnd()) {
1279 insertUse(II, Offset, AllocSize, true);
1280 return;
1281 }
1282
1283 Base::visitIntrinsicInst(II);
1284 }
1285
1286 Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) {
1287 // We consider any PHI or select that results in a direct load or store of
1288 // the same offset to be a viable use for slicing purposes. These uses
1289 // are considered unsplittable and the size is the maximum loaded or stored
1290 // size.
1291 SmallPtrSet<Instruction *, 4> Visited;
1293 Visited.insert(Root);
1294 Uses.push_back(std::make_pair(cast<Instruction>(*U), Root));
1295 const DataLayout &DL = Root->getDataLayout();
1296 // If there are no loads or stores, the access is dead. We mark that as
1297 // a size zero access.
1298 Size = 0;
1299 do {
1300 Instruction *I, *UsedI;
1301 std::tie(UsedI, I) = Uses.pop_back_val();
1302
1303 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
1304 TypeSize LoadSize = DL.getTypeStoreSize(LI->getType());
1305 if (LoadSize.isScalable()) {
1306 PI.setAborted(LI);
1307 return nullptr;
1308 }
1309 Size = std::max(Size, LoadSize.getFixedValue());
1310 continue;
1311 }
1312 if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
1313 Value *Op = SI->getOperand(0);
1314 if (Op == UsedI)
1315 return SI;
1316 TypeSize StoreSize = DL.getTypeStoreSize(Op->getType());
1317 if (StoreSize.isScalable()) {
1318 PI.setAborted(SI);
1319 return nullptr;
1320 }
1321 Size = std::max(Size, StoreSize.getFixedValue());
1322 continue;
1323 }
1324
1325 if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) {
1326 if (!GEP->hasAllZeroIndices())
1327 return GEP;
1328 } else if (!isa<BitCastInst>(I) && !isa<PHINode>(I) &&
1330 return I;
1331 }
1332
1333 for (User *U : I->users())
1334 if (Visited.insert(cast<Instruction>(U)).second)
1335 Uses.push_back(std::make_pair(I, cast<Instruction>(U)));
1336 } while (!Uses.empty());
1337
1338 return nullptr;
1339 }
1340
1341 void visitPHINodeOrSelectInst(Instruction &I) {
1343 if (I.use_empty())
1344 return markAsDead(I);
1345
1346 // If this is a PHI node before a catchswitch, we cannot insert any non-PHI
1347 // instructions in this BB, which may be required during rewriting. Bail out
1348 // on these cases.
1349 if (isa<PHINode>(I) && !I.getParent()->hasInsertionPt())
1350 return PI.setAborted(&I);
1351
1352 // TODO: We could use simplifyInstruction here to fold PHINodes and
1353 // SelectInsts. However, doing so requires to change the current
1354 // dead-operand-tracking mechanism. For instance, suppose neither loading
1355 // from %U nor %other traps. Then "load (select undef, %U, %other)" does not
1356 // trap either. However, if we simply replace %U with undef using the
1357 // current dead-operand-tracking mechanism, "load (select undef, undef,
1358 // %other)" may trap because the select may return the first operand
1359 // "undef".
1360 if (Value *Result = foldPHINodeOrSelectInst(I)) {
1361 if (Result == *U)
1362 // If the result of the constant fold will be the pointer, recurse
1363 // through the PHI/select as if we had RAUW'ed it.
1364 enqueueUsers(I);
1365 else
1366 // Otherwise the operand to the PHI/select is dead, and we can replace
1367 // it with poison.
1368 AS.DeadOperands.push_back(U);
1369
1370 return;
1371 }
1372
1373 if (!IsOffsetKnown)
1374 return PI.setAborted(&I);
1375
1376 // See if we already have computed info on this node.
1377 uint64_t &Size = PHIOrSelectSizes[&I];
1378 if (!Size) {
1379 // This is a new PHI/Select, check for an unsafe use of it.
1380 if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(&I, Size))
1381 return PI.setAborted(UnsafeI);
1382 }
1383
1384 // For PHI and select operands outside the alloca, we can't nuke the entire
1385 // phi or select -- the other side might still be relevant, so we special
1386 // case them here and use a separate structure to track the operands
1387 // themselves which should be replaced with poison.
1388 // FIXME: This should instead be escaped in the event we're instrumenting
1389 // for address sanitization.
1390 if (Offset.uge(AllocSize)) {
1391 AS.DeadOperands.push_back(U);
1392 return;
1393 }
1394
1395 insertUse(I, Offset, Size);
1396 }
1397
1398 void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(PN); }
1399
1400 void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); }
1401
1402 /// Disable SROA entirely if there are unhandled users of the alloca.
1403 void visitInstruction(Instruction &I) { PI.setAborted(&I); }
1404
1405 void visitCallBase(CallBase &CB) {
1406 // If the call operand is read-only and only does a read-only or address
1407 // capture, then we mark it as EscapedReadOnly.
1408 if (CB.isDataOperand(U) &&
1409 !capturesFullProvenance(CB.getCaptureInfo(U->getOperandNo())) &&
1410 CB.onlyReadsMemory(U->getOperandNo())) {
1411 PI.setEscapedReadOnly(&CB);
1412 return;
1413 }
1414
1415 Base::visitCallBase(CB);
1416 }
1417};
1418
1419AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
1420 :
1421#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1422 AI(AI),
1423#endif
1424 PointerEscapingInstr(nullptr), PointerEscapingInstrReadOnly(nullptr) {
1425 SliceBuilder PB(DL, AI, *this);
1426 SliceBuilder::PtrInfo PtrI = PB.visitPtr(AI);
1427 if (PtrI.isEscaped() || PtrI.isAborted()) {
1428 // FIXME: We should sink the escape vs. abort info into the caller nicely,
1429 // possibly by just storing the PtrInfo in the AllocaSlices.
1430 PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst()
1431 : PtrI.getAbortingInst();
1432 assert(PointerEscapingInstr && "Did not track a bad instruction");
1433 return;
1434 }
1435 PointerEscapingInstrReadOnly = PtrI.getEscapedReadOnlyInst();
1436
1437 llvm::erase_if(Slices, [](const Slice &S) { return S.isDead(); });
1438
1439 // Sort the uses. This arranges for the offsets to be in ascending order,
1440 // and the sizes to be in descending order.
1441 llvm::stable_sort(Slices);
1442}
1443
1444#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1445
1446void AllocaSlices::print(raw_ostream &OS, const_iterator I,
1447 StringRef Indent) const {
1448 printSlice(OS, I, Indent);
1449 OS << "\n";
1450 printUse(OS, I, Indent);
1451}
1452
1453void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I,
1454 StringRef Indent) const {
1455 OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")"
1456 << " slice #" << (I - begin())
1457 << (I->isSplittable() ? " (splittable)" : "");
1458}
1459
1460void AllocaSlices::printUse(raw_ostream &OS, const_iterator I,
1461 StringRef Indent) const {
1462 OS << Indent << " used by: " << *I->getUse()->getUser() << "\n";
1463}
1464
1465void AllocaSlices::print(raw_ostream &OS) const {
1466 if (PointerEscapingInstr) {
1467 OS << "Can't analyze slices for alloca: " << AI << "\n"
1468 << " A pointer to this alloca escaped by:\n"
1469 << " " << *PointerEscapingInstr << "\n";
1470 return;
1471 }
1472
1473 if (PointerEscapingInstrReadOnly)
1474 OS << "Escapes into ReadOnly: " << *PointerEscapingInstrReadOnly << "\n";
1475
1476 OS << "Slices of alloca: " << AI << "\n";
1477 for (const_iterator I = begin(), E = end(); I != E; ++I)
1478 print(OS, I);
1479}
1480
1481LLVM_DUMP_METHOD void AllocaSlices::dump(const_iterator I) const {
1482 print(dbgs(), I);
1483}
1484LLVM_DUMP_METHOD void AllocaSlices::dump() const { print(dbgs()); }
1485
1486#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1487
1488/// Find a common load/store type used through a pointer PHI or select.
1489///
1490/// Look through a PHI or select to see if all of its users are loads or stores
1491/// of one common type. Whether those accesses can be speculated does not affect
1492/// the type they use and is checked separately when attempting promotion.
1494 assert((isa<PHINode, SelectInst>(I)) && "expected a PHI or select");
1495 Type *Ty = nullptr;
1496
1497 for (User *U : I.users()) {
1498 Type *UserTy = nullptr;
1499 if (auto *LI = dyn_cast<LoadInst>(U))
1500 UserTy = LI->getType();
1501 else if (auto *Store = dyn_cast<StoreInst>(U))
1502 // Slice building rejects stores of the PHI-or-select-derived pointer, so
1503 // it must be the store's pointer operand here.
1504 UserTy = Store->getValueOperand()->getType();
1505
1506 if (!UserTy || (Ty && Ty != UserTy))
1507 return nullptr;
1508 Ty = UserTy;
1509 }
1510
1511 return Ty;
1512}
1513
1514/// Walk the range of a partitioning looking for a common type to cover this
1515/// sequence of slices.
1516static std::pair<Type *, IntegerType *>
1517findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E,
1518 uint64_t EndOffset) {
1519 Type *Ty = nullptr;
1520 bool TyIsCommon = true;
1521 IntegerType *ITy = nullptr;
1522
1523 // Note that we need to look at *every* alloca slice's Use to ensure we
1524 // always get consistent results regardless of the order of slices.
1525 for (AllocaSlices::const_iterator I = B; I != E; ++I) {
1526 Use *U = I->getUse();
1527 if (isa<IntrinsicInst>(*U->getUser()))
1528 continue;
1529 if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset)
1530 continue;
1531
1532 Type *UserTy = nullptr;
1533 if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
1534 UserTy = LI->getType();
1535 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
1536 UserTy = SI->getValueOperand()->getType();
1537 } else if (isa<PHINode, SelectInst>(U->getUser())) {
1538 UserTy =
1540 }
1541
1542 if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(UserTy)) {
1543 // If the type is larger than the partition, skip it. We only encounter
1544 // this for split integer operations where we want to use the type of the
1545 // entity causing the split. Also skip if the type is not a byte width
1546 // multiple.
1547 if (UserITy->getBitWidth() % 8 != 0 ||
1548 UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset()))
1549 continue;
1550
1551 // Track the largest bitwidth integer type used in this way in case there
1552 // is no common type.
1553 if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth())
1554 ITy = UserITy;
1555 }
1556
1557 // To avoid depending on the order of slices, Ty and TyIsCommon must not
1558 // depend on types skipped above.
1559 if (!UserTy || (Ty && Ty != UserTy))
1560 TyIsCommon = false; // Give up on anything but an iN type.
1561 else
1562 Ty = UserTy;
1563 }
1564
1565 return {TyIsCommon ? Ty : nullptr, ITy};
1566}
1567
1568/// PHI instructions that use an alloca and are subsequently loaded can be
1569/// rewritten to load both input pointers in the pred blocks and then PHI the
1570/// results, allowing the load of the alloca to be promoted.
1571/// From this:
1572/// %P2 = phi [i32* %Alloca, i32* %Other]
1573/// %V = load i32* %P2
1574/// to:
1575/// %V1 = load i32* %Alloca -> will be mem2reg'd
1576/// ...
1577/// %V2 = load i32* %Other
1578/// ...
1579/// %V = phi [i32 %V1, i32 %V2]
1580///
1581/// We can do this to a PHI if its only uses are loads and if any loads moved
1582/// across other outgoing edges can be executed unconditionally.
1583///
1584/// FIXME: This should be hoisted into a generic utility, likely in
1585/// Transforms/Util/Local.h
1587 const DataLayout &DL = PN.getDataLayout();
1588
1589 // For now, we can only do this promotion if the load is in the same block
1590 // as the PHI, and if there are no stores between the phi and load.
1591 // TODO: Allow recursive phi users.
1592 BasicBlock *BB = PN.getParent();
1593 Align MaxAlign;
1594 uint64_t APWidth = DL.getIndexTypeSizeInBits(PN.getType());
1595 Type *LoadType = nullptr;
1596 for (User *U : PN.users()) {
1598 if (!LI || !LI->isSimple())
1599 return false;
1600
1601 // For now we only allow loads in the same block as the PHI. This is
1602 // a common case that happens when instcombine merges two loads through
1603 // a PHI.
1604 if (LI->getParent() != BB)
1605 return false;
1606
1607 if (LoadType) {
1608 if (LoadType != LI->getType())
1609 return false;
1610 } else {
1611 LoadType = LI->getType();
1612 }
1613
1614 // Ensure that there are no instructions between the PHI and the load that
1615 // could store.
1616 for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI)
1617 if (BBI->mayWriteToMemory())
1618 return false;
1619
1620 MaxAlign = std::max(MaxAlign, LI->getAlign());
1621 }
1622
1623 if (!LoadType)
1624 return false;
1625
1626 APInt LoadSize =
1627 APInt(APWidth, DL.getTypeStoreSize(LoadType).getFixedValue());
1628
1629 // We can only transform this if it is safe to push the loads into the
1630 // predecessor blocks. The only thing to watch out for is that we can't put
1631 // a possibly trapping load in the predecessor if it is a critical edge.
1632 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1634 Value *InVal = PN.getIncomingValue(Idx);
1635
1636 // If the value is produced by the terminator of the predecessor (an
1637 // invoke) or it has side-effects, there is no valid place to put a load
1638 // in the predecessor.
1639 if (TI == InVal || TI->mayHaveSideEffects())
1640 return false;
1641
1642 // If the predecessor has a single successor, then the edge isn't
1643 // critical.
1644 if (TI->getNumSuccessors() == 1)
1645 continue;
1646
1647 // If this pointer is always safe to load, or if we can prove that there
1648 // is already a load in the block, then we can move the load to the pred
1649 // block.
1650 if (isSafeToLoadUnconditionally(InVal, MaxAlign, LoadSize,
1651 SimplifyQuery(DL, TI)))
1652 continue;
1653
1654 return false;
1655 }
1656
1657 return true;
1658}
1659
1660/// Check whether a single store through PN can be moved onto each incoming
1661/// edge.
1662static StoreInst *getPHIStoreToRewrite(PHINode &PN, bool PreserveCFG,
1663 DominatorTree &DT) {
1664 // TODO: Support multiple stores and mixed load/store users.
1665 // TODO: Look through other instructions, such as other phis or addrspacecasts
1666 if (!PN.hasOneUse())
1667 return nullptr;
1668
1669 auto *SI = dyn_cast<StoreInst>(PN.user_back());
1670 if (!SI || SI->getPointerOperand() != &PN)
1671 return nullptr;
1672
1673 if (SI->isVolatile())
1674 return nullptr;
1675
1676 BasicBlock *BB = PN.getParent();
1677 // TODO: Allow a harmless prefix between the PHIs and the store.
1678 if (&*BB->getFirstNonPHIOrDbg() != SI)
1679 return nullptr;
1680
1681 Value *StoredValue = SI->getValueOperand();
1683 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1684 BasicBlock *Pred = PN.getIncomingBlock(Idx);
1685
1686 // Only one store is needed for duplicate edges from the same predecessor.
1687 if (!SeenPreds.insert(Pred).second)
1688 continue;
1689
1690 // TODO: Support other terminators.
1691 Instruction *TI = Pred->getTerminator();
1693 return nullptr;
1694
1695 if (Pred == BB)
1696 return nullptr;
1697
1698 // TODO: If StoredValue is another PHI in BB, use its corresponding
1699 // incoming value instead of requiring it to dominate every predecessor.
1700 if (!DT.dominates(StoredValue, TI))
1701 return nullptr;
1702
1703 if (TI->getNumSuccessors() == 1)
1704 continue;
1705
1706 // If the predecessor has more than one successor, then we will need to
1707 // split it so that we can insert the store only on the path to this BB.
1708 if (PreserveCFG || !BB->canSplitPredecessors())
1709 return nullptr;
1710 }
1711
1712 return SI;
1713}
1714
1715static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN) {
1716 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
1717
1718 LoadInst *SomeLoad = cast<LoadInst>(PN.user_back());
1719 Type *LoadTy = SomeLoad->getType();
1720 IRB.SetInsertPoint(&PN);
1721 PHINode *NewPN = IRB.CreatePHI(LoadTy, PN.getNumIncomingValues(),
1722 PN.getName() + ".sroa.speculated");
1723
1724 // Get the AA tags and alignment to use from one of the loads. It does not
1725 // matter which one we get and if any differ.
1726 AAMDNodes AATags = SomeLoad->getAAMetadata();
1727 Align Alignment = SomeLoad->getAlign();
1728
1729 // Rewrite all loads of the PN to use the new PHI.
1730 while (!PN.use_empty()) {
1731 LoadInst *LI = cast<LoadInst>(PN.user_back());
1732 LI->replaceAllUsesWith(NewPN);
1733 LI->eraseFromParent();
1734 }
1735
1736 // Inject loads into all of the pred blocks.
1737 DenseMap<BasicBlock *, Value *> InjectedLoads;
1738 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1739 BasicBlock *Pred = PN.getIncomingBlock(Idx);
1740 Value *InVal = PN.getIncomingValue(Idx);
1741
1742 // A PHI node is allowed to have multiple (duplicated) entries for the same
1743 // basic block, as long as the value is the same. So if we already injected
1744 // a load in the predecessor, then we should reuse the same load for all
1745 // duplicated entries.
1746 if (Value *V = InjectedLoads.lookup(Pred)) {
1747 NewPN->addIncoming(V, Pred);
1748 continue;
1749 }
1750
1751 Instruction *TI = Pred->getTerminator();
1752 IRB.SetInsertPoint(TI);
1753
1754 LoadInst *Load = IRB.CreateAlignedLoad(
1755 LoadTy, InVal, Alignment,
1756 (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
1757 ++NumLoadsSpeculated;
1758 if (AATags)
1759 Load->setAAMetadata(AATags);
1760 NewPN->addIncoming(Load, Pred);
1761 InjectedLoads[Pred] = Load;
1762 }
1763
1764 LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n");
1765 PN.eraseFromParent();
1766}
1767
1768/// Move a store through a pointer PHI onto each of the PHI's incoming edges.
1769/// Returns whether this required modifying the CFG.
1772 LLVM_DEBUG(dbgs() << " original: " << PN << "\n"
1773 << " " << SI << "\n");
1774
1775 // Splitting one edge rewrites all PHIs in the destination block. Snapshot
1776 // the original predecessor/value pairs before making any CFG changes.
1779 for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
1780 BasicBlock *Pred = PN.getIncomingBlock(Idx);
1781 if (!SeenPreds.insert(Pred).second)
1782 continue;
1783 Value *InVal = PN.getIncomingValue(Idx);
1784 IncomingValues.emplace_back(Pred, InVal);
1785
1786 // Revisit every alloca exposed by removing the pointer PHI.
1787 if (auto *AI = dyn_cast<AllocaInst>(getUnderlyingObject(InVal)))
1788 Worklist.insert(AI);
1789 }
1790
1791 bool CFGChanged = false;
1792 BasicBlock *BB = PN.getParent();
1793 for (auto [Pred, InVal] : IncomingValues) {
1794 BasicBlock *StoreBB = Pred;
1795 if (Pred->getTerminator()->getNumSuccessors() != 1) {
1796 StoreBB = SplitBlockPredecessors(BB, {Pred}, ".sroa.store", &DTU);
1797 assert(StoreBB && "store edge was not checked for splitting");
1798 CFGChanged = true;
1799 }
1800
1801 auto *NewStore = cast<StoreInst>(SI.clone());
1802 NewStore->setOperand(StoreInst::getPointerOperandIndex(), InVal);
1803 NewStore->insertBefore(StoreBB->getTerminator()->getIterator());
1804 ++NumStoresPredicated;
1805 LLVM_DEBUG(dbgs() << " to: " << *NewStore << "\n");
1806 }
1807
1808 SI.eraseFromParent();
1809 PN.eraseFromParent();
1810 return CFGChanged;
1811}
1812
1813SelectHandSpeculativity &
1814SelectHandSpeculativity::setAsSpeculatable(bool isTrueVal) {
1815 if (isTrueVal)
1817 else
1819 return *this;
1820}
1821
1822bool SelectHandSpeculativity::isSpeculatable(bool isTrueVal) const {
1823 return isTrueVal ? Bitfield::get<SelectHandSpeculativity::TrueVal>(Storage)
1824 : Bitfield::get<SelectHandSpeculativity::FalseVal>(Storage);
1825}
1826
1827bool SelectHandSpeculativity::areAllSpeculatable() const {
1828 return isSpeculatable(/*isTrueVal=*/true) &&
1829 isSpeculatable(/*isTrueVal=*/false);
1830}
1831
1832bool SelectHandSpeculativity::areAnySpeculatable() const {
1833 return isSpeculatable(/*isTrueVal=*/true) ||
1834 isSpeculatable(/*isTrueVal=*/false);
1835}
1836bool SelectHandSpeculativity::areNoneSpeculatable() const {
1837 return !areAnySpeculatable();
1838}
1839
1840static SelectHandSpeculativity
1842 assert(LI.isSimple() && "Only for simple loads");
1843 SelectHandSpeculativity Spec;
1844
1845 const DataLayout &DL = SI.getDataLayout();
1846 for (Value *Value : {SI.getTrueValue(), SI.getFalseValue()})
1848 SimplifyQuery(DL, &LI)))
1849 Spec.setAsSpeculatable(/*isTrueVal=*/Value == SI.getTrueValue());
1850 else if (PreserveCFG)
1851 return Spec;
1852
1853 return Spec;
1854}
1855
1856std::optional<RewriteableMemOps>
1857SROA::isSafeSelectToSpeculate(SelectInst &SI, bool PreserveCFG) {
1858 RewriteableMemOps Ops;
1859
1860 for (User *U : SI.users()) {
1861 if (auto *Store = dyn_cast<StoreInst>(U)) {
1862 // Note that atomic stores can be transformed; atomic semantics do not
1863 // have any meaning for a local alloca. Stores are not speculatable,
1864 // however, so if we can't turn it into a predicated store, we are done.
1865 if (Store->isVolatile() || PreserveCFG)
1866 return {}; // Give up on this `select`.
1867 Ops.emplace_back(Store);
1868 continue;
1869 }
1870
1871 auto *LI = dyn_cast<LoadInst>(U);
1872
1873 // Note that atomic loads can be transformed;
1874 // atomic semantics do not have any meaning for a local alloca.
1875 if (!LI || LI->isVolatile())
1876 return {}; // Give up on this `select`.
1877
1878 PossiblySpeculatableLoad Load(LI);
1879 if (!LI->isSimple()) {
1880 // If the `load` is not simple, we can't speculatively execute it,
1881 // but we could handle this via a CFG modification. But can we?
1882 if (PreserveCFG)
1883 return {}; // Give up on this `select`.
1884 Ops.emplace_back(Load);
1885 continue;
1886 }
1887
1888 SelectHandSpeculativity Spec =
1889 isSafeLoadOfSelectToSpeculate(*LI, SI, PreserveCFG);
1890 if (PreserveCFG && !Spec.areAllSpeculatable())
1891 return {}; // Give up on this `select`.
1892
1893 Load.setInt(Spec);
1894 Ops.emplace_back(Load);
1895 }
1896
1897 return Ops;
1898}
1899
1901 IRBuilderTy &IRB) {
1902 LLVM_DEBUG(dbgs() << " original load: " << SI << "\n");
1903
1904 Value *TV = SI.getTrueValue();
1905 Value *FV = SI.getFalseValue();
1906 // Replace the given load of the select with a select of two loads.
1907
1908 assert(LI.isSimple() && "We only speculate simple loads");
1909
1910 IRB.SetInsertPoint(&LI);
1911
1912 LoadInst *TL =
1913 IRB.CreateAlignedLoad(LI.getType(), TV, LI.getAlign(),
1914 LI.getName() + ".sroa.speculate.load.true");
1915 LoadInst *FL =
1916 IRB.CreateAlignedLoad(LI.getType(), FV, LI.getAlign(),
1917 LI.getName() + ".sroa.speculate.load.false");
1918 NumLoadsSpeculated += 2;
1919
1920 // Transfer alignment and AA info if present.
1921 TL->setAlignment(LI.getAlign());
1922 FL->setAlignment(LI.getAlign());
1923
1924 AAMDNodes Tags = LI.getAAMetadata();
1925 if (Tags) {
1926 TL->setAAMetadata(Tags);
1927 FL->setAAMetadata(Tags);
1928 }
1929
1930 Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL,
1931 LI.getName() + ".sroa.speculated", &SI);
1932
1933 LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n");
1934 LI.replaceAllUsesWith(V);
1935}
1936
1937template <typename T>
1939 SelectHandSpeculativity Spec,
1940 DomTreeUpdater &DTU) {
1941 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) && "Only for load and store!");
1942 LLVM_DEBUG(dbgs() << " original mem op: " << I << "\n");
1943 BasicBlock *Head = I.getParent();
1944 Instruction *ThenTerm = nullptr;
1945 Instruction *ElseTerm = nullptr;
1946 if (Spec.areNoneSpeculatable())
1947 SplitBlockAndInsertIfThenElse(SI.getCondition(), &I, &ThenTerm, &ElseTerm,
1948 SI.getMetadata(LLVMContext::MD_prof), &DTU);
1949 else {
1950 SplitBlockAndInsertIfThen(SI.getCondition(), &I, /*Unreachable=*/false,
1951 SI.getMetadata(LLVMContext::MD_prof), &DTU,
1952 /*LI=*/nullptr, /*ThenBlock=*/nullptr);
1953 if (Spec.isSpeculatable(/*isTrueVal=*/true))
1954 cast<CondBrInst>(Head->getTerminator())->swapSuccessors();
1955 }
1956 auto *HeadBI = cast<CondBrInst>(Head->getTerminator());
1957 Spec = {}; // Do not use `Spec` beyond this point.
1958 BasicBlock *Tail = I.getParent();
1959 Tail->setName(Head->getName() + ".cont");
1960 PHINode *PN;
1961 if (isa<LoadInst>(I))
1962 PN = PHINode::Create(I.getType(), 2, "", I.getIterator());
1963 for (BasicBlock *SuccBB : successors(Head)) {
1964 bool IsThen = SuccBB == HeadBI->getSuccessor(0);
1965 int SuccIdx = IsThen ? 0 : 1;
1966 auto *NewMemOpBB = SuccBB == Tail ? Head : SuccBB;
1967 auto &CondMemOp = cast<T>(*I.clone());
1968 if (NewMemOpBB != Head) {
1969 NewMemOpBB->setName(Head->getName() + (IsThen ? ".then" : ".else"));
1970 if (isa<LoadInst>(I))
1971 ++NumLoadsPredicated;
1972 else
1973 ++NumStoresPredicated;
1974 } else {
1975 CondMemOp.dropUBImplyingAttrsAndMetadata();
1976 ++NumLoadsSpeculated;
1977 }
1978 CondMemOp.insertBefore(NewMemOpBB->getTerminator()->getIterator());
1979 Value *Ptr = SI.getOperand(1 + SuccIdx);
1980 CondMemOp.setOperand(I.getPointerOperandIndex(), Ptr);
1981 if (isa<LoadInst>(I)) {
1982 CondMemOp.setName(I.getName() + (IsThen ? ".then" : ".else") + ".val");
1983 PN->addIncoming(&CondMemOp, NewMemOpBB);
1984 } else
1985 LLVM_DEBUG(dbgs() << " to: " << CondMemOp << "\n");
1986 }
1987 if (isa<LoadInst>(I)) {
1988 PN->takeName(&I);
1989 LLVM_DEBUG(dbgs() << " to: " << *PN << "\n");
1990 I.replaceAllUsesWith(PN);
1991 }
1992}
1993
1995 SelectHandSpeculativity Spec,
1996 DomTreeUpdater &DTU) {
1997 if (auto *LI = dyn_cast<LoadInst>(&I))
1998 rewriteMemOpOfSelect(SelInst, *LI, Spec, DTU);
1999 else if (auto *SI = dyn_cast<StoreInst>(&I))
2000 rewriteMemOpOfSelect(SelInst, *SI, Spec, DTU);
2001 else
2002 llvm_unreachable_internal("Only for load and store.");
2003}
2004
2006 const RewriteableMemOps &Ops,
2007 IRBuilderTy &IRB, DomTreeUpdater *DTU) {
2008 bool CFGChanged = false;
2009 LLVM_DEBUG(dbgs() << " original select: " << SI << "\n");
2010
2011 for (const RewriteableMemOp &Op : Ops) {
2012 SelectHandSpeculativity Spec;
2013 Instruction *I;
2014 if (auto *const *US = std::get_if<UnspeculatableStore>(&Op)) {
2015 I = *US;
2016 } else {
2017 auto PSL = std::get<PossiblySpeculatableLoad>(Op);
2018 I = PSL.getPointer();
2019 Spec = PSL.getInt();
2020 }
2021 if (Spec.areAllSpeculatable()) {
2023 } else {
2024 assert(DTU && "Should not get here when not allowed to modify the CFG!");
2025 rewriteMemOpOfSelect(SI, *I, Spec, *DTU);
2026 CFGChanged = true;
2027 }
2028 I->eraseFromParent();
2029 }
2030
2031 for (User *U : make_early_inc_range(SI.users()))
2032 cast<BitCastInst>(U)->eraseFromParent();
2033 SI.eraseFromParent();
2034 return CFGChanged;
2035}
2036
2037/// Compute an adjusted pointer from Ptr by Offset bytes where the
2038/// resulting pointer has PointerTy.
2039static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr,
2041 const Twine &NamePrefix) {
2042 if (Offset != 0)
2043 Ptr = IRB.CreateInBoundsPtrAdd(Ptr, IRB.getInt(Offset),
2044 NamePrefix + "sroa_idx");
2045 return IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, PointerTy,
2046 NamePrefix + "sroa_cast");
2047}
2048
2049/// Compute the adjusted alignment for a load or store from an offset.
2053
2054/// Test whether we can convert a value from the old to the new type.
2055///
2056/// This predicate should be used to guard calls to convertValue in order to
2057/// ensure that we only try to convert viable values. The strategy is that we
2058/// will peel off single element struct and array wrappings to get to an
2059/// underlying value, and convert that value.
2060static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy,
2061 unsigned VScale = 0) {
2062 if (OldTy == NewTy)
2063 return true;
2064
2065 // For integer types, we can't handle any bit-width differences. This would
2066 // break both vector conversions with extension and introduce endianness
2067 // issues when in conjunction with loads and stores.
2068 if (isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) {
2070 cast<IntegerType>(NewTy)->getBitWidth() &&
2071 "We can't have the same bitwidth for different int types");
2072 return false;
2073 }
2074
2075 TypeSize NewSize = DL.getTypeSizeInBits(NewTy);
2076 TypeSize OldSize = DL.getTypeSizeInBits(OldTy);
2077
2078 if ((isa<ScalableVectorType>(NewTy) && isa<FixedVectorType>(OldTy)) ||
2079 (isa<ScalableVectorType>(OldTy) && isa<FixedVectorType>(NewTy))) {
2080 // Conversion is only possible when the size of scalable vectors is known.
2081 if (!VScale)
2082 return false;
2083
2084 // For ptr-to-int and int-to-ptr casts, the pointer side is resolved within
2085 // a single domain (either fixed or scalable). Any additional conversion
2086 // between fixed and scalable types is handled through integer types.
2087 auto OldVTy = OldTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(OldTy) : OldTy;
2088 auto NewVTy = NewTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(NewTy) : NewTy;
2089
2090 if (isa<ScalableVectorType>(NewTy)) {
2092 return false;
2093
2094 NewSize = TypeSize::getFixed(NewSize.getKnownMinValue() * VScale);
2095 } else {
2097 return false;
2098
2099 OldSize = TypeSize::getFixed(OldSize.getKnownMinValue() * VScale);
2100 }
2101 }
2102
2103 if (NewSize != OldSize)
2104 return false;
2105 if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
2106 return false;
2107
2108 // We can convert pointers to integers and vice-versa. Same for vectors
2109 // of pointers and integers.
2110 OldTy = OldTy->getScalarType();
2111 NewTy = NewTy->getScalarType();
2112 if (NewTy->isPointerTy() || OldTy->isPointerTy()) {
2113 if (NewTy->isPointerTy() && OldTy->isPointerTy()) {
2114 unsigned OldAS = OldTy->getPointerAddressSpace();
2115 unsigned NewAS = NewTy->getPointerAddressSpace();
2116 // Convert pointers if they are pointers from the same address space or
2117 // different integral (not non-integral) address spaces with the same
2118 // pointer size.
2119 return OldAS == NewAS ||
2120 (!DL.isNonIntegralAddressSpace(OldAS) &&
2121 !DL.isNonIntegralAddressSpace(NewAS) &&
2122 DL.getPointerSize(OldAS) == DL.getPointerSize(NewAS));
2123 }
2124
2125 // We can convert integers to integral pointers, but not to non-integral
2126 // pointers.
2127 if (OldTy->isIntegerTy())
2128 return !DL.isNonIntegralPointerType(NewTy);
2129
2130 // We can convert integral pointers to integers, but non-integral pointers
2131 // need to remain pointers.
2132 if (!DL.isNonIntegralPointerType(OldTy))
2133 return NewTy->isIntegerTy();
2134
2135 return false;
2136 }
2137
2138 if (OldTy->isTargetExtTy() || NewTy->isTargetExtTy())
2139 return false;
2140
2141 return true;
2142}
2143
2144/// Test whether the given slice use can be promoted to a vector.
2145///
2146/// This function is called to test each entry in a partition which is slated
2147/// for a single slice.
2148static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
2149 VectorType *Ty,
2150 uint64_t ElementSize,
2151 const DataLayout &DL,
2152 unsigned VScale) {
2153 // First validate the slice offsets.
2154 uint64_t BeginOffset =
2155 std::max(S.beginOffset(), P.beginOffset()) - P.beginOffset();
2156 uint64_t BeginIndex = BeginOffset / ElementSize;
2157 if (BeginIndex * ElementSize != BeginOffset ||
2158 BeginIndex >= cast<FixedVectorType>(Ty)->getNumElements())
2159 return false;
2160 uint64_t EndOffset = std::min(S.endOffset(), P.endOffset()) - P.beginOffset();
2161 uint64_t EndIndex = EndOffset / ElementSize;
2162 if (EndIndex * ElementSize != EndOffset ||
2163 EndIndex > cast<FixedVectorType>(Ty)->getNumElements())
2164 return false;
2165
2166 assert(EndIndex > BeginIndex && "Empty vector!");
2167 uint64_t NumElements = EndIndex - BeginIndex;
2168 Type *SliceTy = (NumElements == 1)
2169 ? Ty->getElementType()
2170 : FixedVectorType::get(Ty->getElementType(), NumElements);
2171
2172 Type *SplitIntTy =
2173 Type::getIntNTy(Ty->getContext(), NumElements * ElementSize * 8);
2174
2175 Use *U = S.getUse();
2176
2177 if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) {
2178 if (MI->isVolatile())
2179 return false;
2180 if (!S.isSplittable())
2181 return false; // Skip any unsplittable intrinsics.
2182 if (isa<MemSetInst>(MI)) {
2183 Type *SplatTy = Type::getIntNTy(Ty->getContext(), ElementSize * 8);
2184 if (!canConvertValue(DL, SplatTy, Ty->getElementType(), VScale))
2185 return false;
2186 }
2187 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
2188 if (!II->isLifetimeStartOrEnd() && !II->isDroppable())
2189 return false;
2190 } else if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
2191 if (LI->isVolatile())
2192 return false;
2193 Type *LTy = LI->getType();
2194 // Disable vector promotion when there are loads or stores of an FCA.
2195 if (LTy->isStructTy())
2196 return false;
2197 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2198 assert(LTy->isIntegerTy());
2199 LTy = SplitIntTy;
2200 }
2201 if (!canConvertValue(DL, SliceTy, LTy, VScale))
2202 return false;
2203 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
2204 if (SI->isVolatile())
2205 return false;
2206 Type *STy = SI->getValueOperand()->getType();
2207 // Disable vector promotion when there are loads or stores of an FCA.
2208 if (STy->isStructTy())
2209 return false;
2210 if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) {
2211 assert(STy->isIntegerTy());
2212 STy = SplitIntTy;
2213 }
2214 if (!canConvertValue(DL, STy, SliceTy, VScale))
2215 return false;
2216 } else {
2217 return false;
2218 }
2219
2220 return true;
2221}
2222
2223/// Test whether any vector type in \p CandidateTys is viable for promotion.
2224///
2225/// This implements the necessary checking for \c isVectorPromotionViable over
2226/// all slices of the alloca for the given VectorType.
2227static VectorType *
2229 SmallVectorImpl<VectorType *> &CandidateTys,
2230 bool HaveCommonEltTy, Type *CommonEltTy,
2231 bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
2232 VectorType *CommonVecPtrTy, unsigned VScale) {
2233 // If we didn't find a vector type, nothing to do here.
2234 if (CandidateTys.empty())
2235 return nullptr;
2236
2237 // Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
2238 // then we should choose it, not some other alternative.
2239 // But, we can't perform a no-op pointer address space change via bitcast,
2240 // so if we didn't have a common pointer element type, bail.
2241 if (HaveVecPtrTy && !HaveCommonVecPtrTy)
2242 return nullptr;
2243
2244 // Try to pick the "best" element type out of the choices.
2245 if (!HaveCommonEltTy && HaveVecPtrTy) {
2246 // If there was a pointer element type, there's really only one choice.
2247 CandidateTys.clear();
2248 CandidateTys.push_back(CommonVecPtrTy);
2249 } else if (!HaveCommonEltTy && !HaveVecPtrTy) {
2250 // Integer-ify vector types.
2251 for (VectorType *&VTy : CandidateTys) {
2252 if (!VTy->getElementType()->isIntegerTy())
2253 VTy = cast<VectorType>(VTy->getWithNewType(IntegerType::getIntNTy(
2254 VTy->getContext(), VTy->getScalarSizeInBits())));
2255 }
2256
2257 // Rank the remaining candidate vector types. This is easy because we know
2258 // they're all integer vectors. We sort by ascending number of elements.
2259 auto RankVectorTypesComp = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2260 (void)DL;
2261 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2262 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2263 "Cannot have vector types of different sizes!");
2264 assert(RHSTy->getElementType()->isIntegerTy() &&
2265 "All non-integer types eliminated!");
2266 assert(LHSTy->getElementType()->isIntegerTy() &&
2267 "All non-integer types eliminated!");
2268 return cast<FixedVectorType>(RHSTy)->getNumElements() <
2269 cast<FixedVectorType>(LHSTy)->getNumElements();
2270 };
2271 auto RankVectorTypesEq = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
2272 (void)DL;
2273 assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
2274 DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
2275 "Cannot have vector types of different sizes!");
2276 assert(RHSTy->getElementType()->isIntegerTy() &&
2277 "All non-integer types eliminated!");
2278 assert(LHSTy->getElementType()->isIntegerTy() &&
2279 "All non-integer types eliminated!");
2280 return cast<FixedVectorType>(RHSTy)->getNumElements() ==
2281 cast<FixedVectorType>(LHSTy)->getNumElements();
2282 };
2283 llvm::sort(CandidateTys, RankVectorTypesComp);
2284 CandidateTys.erase(llvm::unique(CandidateTys, RankVectorTypesEq),
2285 CandidateTys.end());
2286 } else {
2287// The only way to have the same element type in every vector type is to
2288// have the same vector type. Check that and remove all but one.
2289#ifndef NDEBUG
2290 for (VectorType *VTy : CandidateTys) {
2291 assert(VTy->getElementType() == CommonEltTy &&
2292 "Unaccounted for element type!");
2293 assert(VTy == CandidateTys[0] &&
2294 "Different vector types with the same element type!");
2295 }
2296#endif
2297 CandidateTys.resize(1);
2298 }
2299
2300 // FIXME: hack. Do we have a named constant for this?
2301 // SDAG SDNode can't have more than 65535 operands.
2302 llvm::erase_if(CandidateTys, [](VectorType *VTy) {
2303 return cast<FixedVectorType>(VTy)->getNumElements() >
2304 std::numeric_limits<unsigned short>::max();
2305 });
2306
2307 // Find a vector type viable for promotion by iterating over all slices.
2308 auto *VTy = llvm::find_if(CandidateTys, [&](VectorType *VTy) -> bool {
2309 uint64_t ElementSize =
2310 DL.getTypeSizeInBits(VTy->getElementType()).getFixedValue();
2311
2312 // While the definition of LLVM vectors is bitpacked, we don't support sizes
2313 // that aren't byte sized.
2314 if (ElementSize % 8)
2315 return false;
2316 assert((DL.getTypeSizeInBits(VTy).getFixedValue() % 8) == 0 &&
2317 "vector size not a multiple of element size?");
2318 ElementSize /= 8;
2319
2320 for (const Slice &S : P)
2321 if (!isVectorPromotionViableForSlice(P, S, VTy, ElementSize, DL, VScale))
2322 return false;
2323
2324 for (const Slice *S : P.splitSliceTails())
2325 if (!isVectorPromotionViableForSlice(P, *S, VTy, ElementSize, DL, VScale))
2326 return false;
2327
2328 return true;
2329 });
2330 return VTy != CandidateTys.end() ? *VTy : nullptr;
2331}
2332
2334 SetVector<Type *> &OtherTys, ArrayRef<VectorType *> CandidateTysCopy,
2335 function_ref<void(Type *)> CheckCandidateType, Partition &P,
2336 const DataLayout &DL, SmallVectorImpl<VectorType *> &CandidateTys,
2337 bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy,
2338 bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy, unsigned VScale) {
2339 [[maybe_unused]] VectorType *OriginalElt =
2340 CandidateTysCopy.size() ? CandidateTysCopy[0] : nullptr;
2341 // Consider additional vector types where the element type size is a
2342 // multiple of load/store element size.
2343 for (Type *Ty : OtherTys) {
2345 continue;
2346 unsigned TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
2347 // Make a copy of CandidateTys and iterate through it, because we
2348 // might append to CandidateTys in the loop.
2349 for (VectorType *const VTy : CandidateTysCopy) {
2350 // The elements in the copy should remain invariant throughout the loop
2351 assert(CandidateTysCopy[0] == OriginalElt && "Different Element");
2352 unsigned VectorSize = DL.getTypeSizeInBits(VTy).getFixedValue();
2353 unsigned ElementSize =
2354 DL.getTypeSizeInBits(VTy->getElementType()).getFixedValue();
2355 if (TypeSize != VectorSize && TypeSize != ElementSize &&
2356 VectorSize % TypeSize == 0) {
2357 VectorType *NewVTy = VectorType::get(Ty, VectorSize / TypeSize, false);
2358 CheckCandidateType(NewVTy);
2359 }
2360 }
2361 }
2362
2364 P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2365 HaveCommonVecPtrTy, CommonVecPtrTy, VScale);
2366}
2367
2368/// Test whether the given alloca partitioning and range of slices can be
2369/// promoted to a vector.
2370///
2371/// This is a quick test to check whether we can rewrite a particular alloca
2372/// partition (and its newly formed alloca) into a vector alloca with only
2373/// whole-vector loads and stores such that it could be promoted to a vector
2374/// SSA value. We only can ensure this for a limited set of operations, and we
2375/// don't want to do the rewrites unless we are confident that the result will
2376/// be promotable, so we have an early test here.
2378 unsigned VScale) {
2379 // Collect the candidate types for vector-based promotion. Also track whether
2380 // we have different element types.
2381 SmallVector<VectorType *, 4> CandidateTys;
2382 SetVector<Type *> LoadStoreTys;
2383 SetVector<Type *> DeferredTys;
2384 Type *CommonEltTy = nullptr;
2385 VectorType *CommonVecPtrTy = nullptr;
2386 bool HaveVecPtrTy = false;
2387 bool HaveCommonEltTy = true;
2388 bool HaveCommonVecPtrTy = true;
2389 auto CheckCandidateType = [&](Type *Ty) {
2390 if (auto *VTy = dyn_cast<FixedVectorType>(Ty)) {
2391 // Return if bitcast to vectors is different for total size in bits.
2392 if (!CandidateTys.empty()) {
2393 VectorType *V = CandidateTys[0];
2394 if (DL.getTypeSizeInBits(VTy).getFixedValue() !=
2395 DL.getTypeSizeInBits(V).getFixedValue()) {
2396 CandidateTys.clear();
2397 return;
2398 }
2399 }
2400 CandidateTys.push_back(VTy);
2401 Type *EltTy = VTy->getElementType();
2402
2403 if (!CommonEltTy)
2404 CommonEltTy = EltTy;
2405 else if (CommonEltTy != EltTy)
2406 HaveCommonEltTy = false;
2407
2408 if (EltTy->isPointerTy()) {
2409 HaveVecPtrTy = true;
2410 if (!CommonVecPtrTy)
2411 CommonVecPtrTy = VTy;
2412 else if (CommonVecPtrTy != VTy)
2413 HaveCommonVecPtrTy = false;
2414 }
2415 }
2416 };
2417
2418 // Put load and store types into a set for de-duplication.
2419 for (const Slice &S : P) {
2420 Type *Ty;
2421 if (auto *LI = dyn_cast<LoadInst>(S.getUse()->getUser()))
2422 Ty = LI->getType();
2423 else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser()))
2424 Ty = SI->getValueOperand()->getType();
2425 else
2426 continue;
2427
2428 auto CandTy = Ty->getScalarType();
2429 if (CandTy->isPointerTy() && (S.beginOffset() != P.beginOffset() ||
2430 S.endOffset() != P.endOffset())) {
2431 DeferredTys.insert(Ty);
2432 continue;
2433 }
2434
2435 LoadStoreTys.insert(Ty);
2436 // Consider any loads or stores that are the exact size of the slice.
2437 if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset())
2438 CheckCandidateType(Ty);
2439 }
2440
2441 SmallVector<VectorType *, 4> CandidateTysCopy = CandidateTys;
2443 LoadStoreTys, CandidateTysCopy, CheckCandidateType, P, DL,
2444 CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
2445 HaveCommonVecPtrTy, CommonVecPtrTy, VScale))
2446 return VTy;
2447
2448 CandidateTys.clear();
2450 DeferredTys, CandidateTysCopy, CheckCandidateType, P, DL, CandidateTys,
2451 HaveCommonEltTy, CommonEltTy, HaveVecPtrTy, HaveCommonVecPtrTy,
2452 CommonVecPtrTy, VScale);
2453}
2454
2455/// Test whether a slice of an alloca is valid for integer widening.
2456///
2457/// This implements the necessary checking for the \c isIntegerWideningViable
2458/// test below on a single slice of the alloca.
2459static bool isIntegerWideningViableForSlice(const Slice &S,
2460 uint64_t AllocBeginOffset,
2461 Type *AllocaTy,
2462 const DataLayout &DL,
2463 bool &WholeAllocaOp) {
2464 uint64_t Size = DL.getTypeStoreSize(AllocaTy).getFixedValue();
2465
2466 uint64_t RelBegin = S.beginOffset() - AllocBeginOffset;
2467 uint64_t RelEnd = S.endOffset() - AllocBeginOffset;
2468
2469 Use *U = S.getUse();
2470
2471 // Lifetime intrinsics operate over the whole alloca whose sizes are usually
2472 // larger than other load/store slices (RelEnd > Size). But lifetime are
2473 // always promotable and should not impact other slices' promotability of the
2474 // partition.
2475 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
2476 if (II->isLifetimeStartOrEnd() || II->isDroppable())
2477 return true;
2478 }
2479
2480 // We can't reasonably handle cases where the load or store extends past
2481 // the end of the alloca's type and into its padding.
2482 if (RelEnd > Size)
2483 return false;
2484
2485 if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) {
2486 if (LI->isVolatile())
2487 return false;
2488 // We can't handle loads that extend past the allocated memory.
2489 TypeSize LoadSize = DL.getTypeStoreSize(LI->getType());
2490 if (!LoadSize.isFixed() || LoadSize.getFixedValue() > Size)
2491 return false;
2492 // So far, AllocaSliceRewriter does not support widening split slice tails
2493 // in rewriteIntegerLoad.
2494 if (S.beginOffset() < AllocBeginOffset)
2495 return false;
2496 // Note that we don't count vector loads or stores as whole-alloca
2497 // operations which enable integer widening because we would prefer to use
2498 // vector widening instead.
2499 if (!isa<VectorType>(LI->getType()) && RelBegin == 0 && RelEnd == Size)
2500 WholeAllocaOp = true;
2501 if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) {
2502 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedValue())
2503 return false;
2504 } else if (RelBegin != 0 || RelEnd != Size ||
2505 !canConvertValue(DL, AllocaTy, LI->getType())) {
2506 // Non-integer loads need to be convertible from the alloca type so that
2507 // they are promotable.
2508 return false;
2509 }
2510 } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) {
2511 Type *ValueTy = SI->getValueOperand()->getType();
2512 if (SI->isVolatile())
2513 return false;
2514 // We can't handle stores that extend past the allocated memory.
2515 TypeSize StoreSize = DL.getTypeStoreSize(ValueTy);
2516 if (!StoreSize.isFixed() || StoreSize.getFixedValue() > Size)
2517 return false;
2518 // So far, AllocaSliceRewriter does not support widening split slice tails
2519 // in rewriteIntegerStore.
2520 if (S.beginOffset() < AllocBeginOffset)
2521 return false;
2522 // Note that we don't count vector loads or stores as whole-alloca
2523 // operations which enable integer widening because we would prefer to use
2524 // vector widening instead.
2525 if (!isa<VectorType>(ValueTy) && RelBegin == 0 && RelEnd == Size)
2526 WholeAllocaOp = true;
2527 if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) {
2528 if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy).getFixedValue())
2529 return false;
2530 } else if (RelBegin != 0 || RelEnd != Size ||
2531 !canConvertValue(DL, ValueTy, AllocaTy)) {
2532 // Non-integer stores need to be convertible to the alloca type so that
2533 // they are promotable.
2534 return false;
2535 }
2536 } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) {
2537 if (MI->isVolatile() || !isa<Constant>(MI->getLength()))
2538 return false;
2539 if (!S.isSplittable())
2540 return false; // Skip any unsplittable intrinsics.
2541 } else {
2542 return false;
2543 }
2544
2545 return true;
2546}
2547
2548/// Test whether the given alloca partition's integer operations can be
2549/// widened to promotable ones.
2550///
2551/// This is a quick test to check whether we can rewrite the integer loads and
2552/// stores to a particular alloca into wider loads and stores and be able to
2553/// promote the resulting alloca.
2554static bool isIntegerWideningViable(Partition &P, Type *AllocaTy,
2555 const DataLayout &DL) {
2556 uint64_t SizeInBits = DL.getTypeSizeInBits(AllocaTy).getFixedValue();
2557 // Don't create integer types larger than the maximum bitwidth.
2558 if (SizeInBits > IntegerType::MAX_INT_BITS)
2559 return false;
2560
2561 // Don't try to handle allocas with bit-padding.
2562 if (SizeInBits != DL.getTypeStoreSizeInBits(AllocaTy).getFixedValue())
2563 return false;
2564
2565 // We need to ensure that an integer type with the appropriate bitwidth can
2566 // be converted to the alloca type, whatever that is. We don't want to force
2567 // the alloca itself to have an integer type if there is a more suitable one.
2568 Type *IntTy = Type::getIntNTy(AllocaTy->getContext(), SizeInBits);
2569 if (!canConvertValue(DL, AllocaTy, IntTy) ||
2570 !canConvertValue(DL, IntTy, AllocaTy))
2571 return false;
2572
2573 // While examining uses, we ensure that the alloca has a covering load or
2574 // store. We don't want to widen the integer operations only to fail to
2575 // promote due to some other unsplittable entry (which we may make splittable
2576 // later). However, if there are only splittable uses, go ahead and assume
2577 // that we cover the alloca.
2578 // FIXME: We shouldn't consider split slices that happen to start in the
2579 // partition here...
2580 bool WholeAllocaOp = P.empty() && DL.isLegalInteger(SizeInBits);
2581
2582 for (const Slice &S : P)
2583 if (!isIntegerWideningViableForSlice(S, P.beginOffset(), AllocaTy, DL,
2584 WholeAllocaOp))
2585 return false;
2586
2587 for (const Slice *S : P.splitSliceTails())
2588 if (!isIntegerWideningViableForSlice(*S, P.beginOffset(), AllocaTy, DL,
2589 WholeAllocaOp))
2590 return false;
2591
2592 return WholeAllocaOp;
2593}
2594
2595static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V,
2597 const Twine &Name) {
2598 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2599 IntegerType *IntTy = cast<IntegerType>(V->getType());
2600 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2601 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2602 "Element extends past full value");
2603 uint64_t ShAmt = 8 * Offset;
2604 if (DL.isBigEndian())
2605 ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedValue() -
2606 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2607 if (ShAmt) {
2608 V = IRB.CreateLShr(V, ShAmt, Name + ".shift");
2609 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2610 }
2611 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2612 "Cannot extract to a larger integer!");
2613 if (Ty != IntTy) {
2614 V = IRB.CreateTrunc(V, Ty, Name + ".trunc");
2615 LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n");
2616 }
2617 return V;
2618}
2619
2620static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old,
2621 Value *V, uint64_t Offset, const Twine &Name) {
2622 IntegerType *IntTy = cast<IntegerType>(Old->getType());
2623 IntegerType *Ty = cast<IntegerType>(V->getType());
2624 assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
2625 "Cannot insert a larger integer!");
2626 LLVM_DEBUG(dbgs() << " start: " << *V << "\n");
2627 if (Ty != IntTy) {
2628 V = IRB.CreateZExt(V, IntTy, Name + ".ext");
2629 LLVM_DEBUG(dbgs() << " extended: " << *V << "\n");
2630 }
2631 assert(DL.getTypeStoreSize(Ty).getFixedValue() + Offset <=
2632 DL.getTypeStoreSize(IntTy).getFixedValue() &&
2633 "Element store outside of alloca store");
2634 uint64_t ShAmt = 8 * Offset;
2635 if (DL.isBigEndian())
2636 ShAmt = 8 * (DL.getTypeStoreSize(IntTy).getFixedValue() -
2637 DL.getTypeStoreSize(Ty).getFixedValue() - Offset);
2638 if (ShAmt) {
2639 V = IRB.CreateShl(V, ShAmt, Name + ".shift");
2640 LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n");
2641 }
2642
2643 if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) {
2644 APInt Mask = ~Ty->getMask().zext(IntTy->getBitWidth()).shl(ShAmt);
2645 Old = IRB.CreateAnd(Old, Mask, Name + ".mask");
2646 LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n");
2647 V = IRB.CreateOr(Old, V, Name + ".insert");
2648 LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n");
2649 }
2650 return V;
2651}
2652
2653static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex,
2654 unsigned EndIndex, const Twine &Name) {
2655 auto *VecTy = cast<FixedVectorType>(V->getType());
2656 unsigned NumElements = EndIndex - BeginIndex;
2657 assert(NumElements <= VecTy->getNumElements() && "Too many elements!");
2658
2659 if (NumElements == VecTy->getNumElements())
2660 return V;
2661
2662 if (NumElements == 1) {
2663 V = IRB.CreateExtractElement(V, BeginIndex, Name + ".extract");
2664 LLVM_DEBUG(dbgs() << " extract: " << *V << "\n");
2665 return V;
2666 }
2667
2668 auto Mask = llvm::to_vector<8>(llvm::seq<int>(BeginIndex, EndIndex));
2669 V = IRB.CreateShuffleVector(V, Mask, Name + ".extract");
2670 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2671 return V;
2672}
2673
2674static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V,
2675 unsigned BeginIndex, const Twine &Name) {
2676 VectorType *VecTy = cast<VectorType>(Old->getType());
2677 assert(VecTy && "Can only insert a vector into a vector");
2678
2679 VectorType *Ty = dyn_cast<VectorType>(V->getType());
2680 if (!Ty) {
2681 // Single element to insert.
2682 V = IRB.CreateInsertElement(Old, V, BeginIndex, Name + ".insert");
2683 LLVM_DEBUG(dbgs() << " insert: " << *V << "\n");
2684 return V;
2685 }
2686
2687 unsigned NumSubElements = cast<FixedVectorType>(Ty)->getNumElements();
2688 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
2689
2690 assert(NumSubElements <= NumElements && "Too many elements!");
2691 if (NumSubElements == NumElements) {
2692 assert(V->getType() == VecTy && "Vector type mismatch");
2693 return V;
2694 }
2695 unsigned EndIndex = BeginIndex + NumSubElements;
2696
2697 // When inserting a smaller vector into the larger to store, we first
2698 // use a shuffle vector to widen it with undef elements, and then
2699 // a second shuffle vector to select between the loaded vector and the
2700 // incoming vector.
2702 Mask.reserve(NumElements);
2703 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2704 if (Idx >= BeginIndex && Idx < EndIndex)
2705 Mask.push_back(Idx - BeginIndex);
2706 else
2707 Mask.push_back(-1);
2708 V = IRB.CreateShuffleVector(V, Mask, Name + ".expand");
2709 LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n");
2710
2711 Mask.clear();
2712 for (unsigned Idx = 0; Idx != NumElements; ++Idx)
2713 if (Idx >= BeginIndex && Idx < EndIndex)
2714 Mask.push_back(Idx);
2715 else
2716 Mask.push_back(Idx + NumElements);
2717 V = IRB.CreateShuffleVector(V, Old, Mask, Name + "blend");
2718 LLVM_DEBUG(dbgs() << " blend: " << *V << "\n");
2719 return V;
2720}
2721
2722/// This function takes two vector values and combines them into a single vector
2723/// by concatenating their elements. The function handles:
2724///
2725/// 1. Element type mismatch: If either vector's element type differs from
2726/// NewAIEltType, the function bitcasts the vector to use NewAIEltType while
2727/// preserving the total bit width (adjusting the number of elements
2728/// accordingly).
2729///
2730/// 2. Size mismatch: After transforming the vectors to have the desired element
2731/// type, if the two vectors have different numbers of elements, the smaller
2732/// vector is extended with poison values to match the size of the larger
2733/// vector before concatenation.
2734///
2735/// 3. Concatenation: The vectors are merged using a shuffle operation that
2736/// places all elements of V0 first, followed by all elements of V1.
2737///
2738/// \param V0 The first vector to merge (must be a vector type)
2739/// \param V1 The second vector to merge (must be a vector type)
2740/// \param DL The data layout for size calculations
2741/// \param NewAIEltTy The desired element type for the result vector
2742/// \param Builder IRBuilder for creating new instructions
2743/// \return A new vector containing all elements from V0 followed by all
2744/// elements from V1
2746 Type *NewAIEltTy, IRBuilder<> &Builder) {
2747 // V0 and V1 are vectors
2748 // Create a new vector type with combined elements
2749 // Use ShuffleVector to concatenate the vectors
2750 auto *VecType0 = cast<FixedVectorType>(V0->getType());
2751 auto *VecType1 = cast<FixedVectorType>(V1->getType());
2752
2753 // If V0/V1 element types are different from NewAllocaElementType,
2754 // we need to introduce bitcasts before merging them
2755 auto BitcastIfNeeded = [&](Value *&V, FixedVectorType *&VecType,
2756 const char *DebugName) {
2757 Type *EltType = VecType->getElementType();
2758 if (EltType != NewAIEltTy) {
2759 // Calculate new number of elements to maintain same bit width
2760 unsigned TotalBits =
2761 VecType->getNumElements() * DL.getTypeSizeInBits(EltType);
2762 unsigned NewNumElts = TotalBits / DL.getTypeSizeInBits(NewAIEltTy);
2763
2764 auto *NewVecType = FixedVectorType::get(NewAIEltTy, NewNumElts);
2765 V = Builder.CreateBitCast(V, NewVecType);
2766 VecType = NewVecType;
2767 LLVM_DEBUG(dbgs() << " bitcast " << DebugName << ": " << *V << "\n");
2768 }
2769 };
2770
2771 BitcastIfNeeded(V0, VecType0, "V0");
2772 BitcastIfNeeded(V1, VecType1, "V1");
2773
2774 unsigned NumElts0 = VecType0->getNumElements();
2775 unsigned NumElts1 = VecType1->getNumElements();
2776
2777 SmallVector<int, 16> ShuffleMask;
2778
2779 if (NumElts0 == NumElts1) {
2780 for (unsigned i = 0; i < NumElts0 + NumElts1; ++i)
2781 ShuffleMask.push_back(i);
2782 } else {
2783 // If two vectors have different sizes, we need to extend
2784 // the smaller vector to the size of the larger vector.
2785 unsigned SmallSize = std::min(NumElts0, NumElts1);
2786 unsigned LargeSize = std::max(NumElts0, NumElts1);
2787 bool IsV0Smaller = NumElts0 < NumElts1;
2788 Value *&ExtendedVec = IsV0Smaller ? V0 : V1;
2789 SmallVector<int, 16> ExtendMask;
2790 for (unsigned i = 0; i < SmallSize; ++i)
2791 ExtendMask.push_back(i);
2792 for (unsigned i = SmallSize; i < LargeSize; ++i)
2793 ExtendMask.push_back(PoisonMaskElem);
2794 ExtendedVec = Builder.CreateShuffleVector(
2795 ExtendedVec, PoisonValue::get(ExtendedVec->getType()), ExtendMask);
2796 LLVM_DEBUG(dbgs() << " shufflevector: " << *ExtendedVec << "\n");
2797 for (unsigned i = 0; i < NumElts0; ++i)
2798 ShuffleMask.push_back(i);
2799 for (unsigned i = 0; i < NumElts1; ++i)
2800 ShuffleMask.push_back(LargeSize + i);
2801 }
2802
2803 return Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2804}
2805
2806namespace {
2807
2808/// Visitor to rewrite instructions using p particular slice of an alloca
2809/// to use a new alloca.
2810///
2811/// Also implements the rewriting to vector-based accesses when the partition
2812/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
2813/// lives here.
2814class AllocaSliceRewriter : public InstVisitor<AllocaSliceRewriter, bool> {
2815 // Befriend the base class so it can delegate to private visit methods.
2816 friend class InstVisitor<AllocaSliceRewriter, bool>;
2817
2818 using Base = InstVisitor<AllocaSliceRewriter, bool>;
2819
2820 const DataLayout &DL;
2821 AllocaSlices &AS;
2822 SROA &Pass;
2823 AllocaInst &OldAI, &NewAI;
2824 const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
2825 Type *NewAllocaTy;
2826
2827 // This is a convenience and flag variable that will be null unless the new
2828 // alloca's integer operations should be widened to this integer type due to
2829 // passing isIntegerWideningViable above. If it is non-null, the desired
2830 // integer type will be stored here for easy access during rewriting.
2831 IntegerType *IntTy;
2832
2833 // If we are rewriting an alloca partition which can be written as pure
2834 // vector operations, we stash extra information here. When VecTy is
2835 // non-null, we have some strict guarantees about the rewritten alloca:
2836 // - The new alloca is exactly the size of the vector type here.
2837 // - The accesses all either map to the entire vector or to a single
2838 // element.
2839 // - The set of accessing instructions is only one of those handled above
2840 // in isVectorPromotionViable. Generally these are the same access kinds
2841 // which are promotable via mem2reg.
2842 VectorType *VecTy;
2843 Type *ElementTy;
2844 uint64_t ElementSize;
2845
2846 // The original offset of the slice currently being rewritten relative to
2847 // the original alloca.
2848 uint64_t BeginOffset = 0;
2849 uint64_t EndOffset = 0;
2850
2851 // The new offsets of the slice currently being rewritten relative to the
2852 // original alloca.
2853 uint64_t NewBeginOffset = 0, NewEndOffset = 0;
2854
2855 uint64_t SliceSize = 0;
2856 bool IsSplittable = false;
2857 bool IsSplit = false;
2858 Use *OldUse = nullptr;
2859 Instruction *OldPtr = nullptr;
2860
2861 // Track post-rewrite users which are PHI nodes and Selects.
2862 SmallSetVector<PHINode *, 8> &PHIUsers;
2863 SmallSetVector<SelectInst *, 8> &SelectUsers;
2864
2865 // Utility IR builder, whose name prefix is setup for each visited use, and
2866 // the insertion point is set to point to the user.
2867 IRBuilderTy IRB;
2868
2869 // Return the new alloca, addrspacecasted if required to avoid changing the
2870 // addrspace of a volatile access.
2871 Value *getPtrToNewAI(unsigned AddrSpace, bool IsVolatile) {
2872 if (!IsVolatile || AddrSpace == NewAI.getType()->getPointerAddressSpace())
2873 return &NewAI;
2874
2875 Type *AccessTy = IRB.getPtrTy(AddrSpace);
2876 return IRB.CreateAddrSpaceCast(&NewAI, AccessTy);
2877 }
2878
2879public:
2880 AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass,
2881 AllocaInst &OldAI, AllocaInst &NewAI, Type *NewAllocaTy,
2882 uint64_t NewAllocaBeginOffset,
2883 uint64_t NewAllocaEndOffset, bool IsIntegerPromotable,
2884 VectorType *PromotableVecTy,
2885 SmallSetVector<PHINode *, 8> &PHIUsers,
2886 SmallSetVector<SelectInst *, 8> &SelectUsers)
2887 : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI),
2888 NewAllocaBeginOffset(NewAllocaBeginOffset),
2889 NewAllocaEndOffset(NewAllocaEndOffset), NewAllocaTy(NewAllocaTy),
2890 IntTy(IsIntegerPromotable
2891 ? Type::getIntNTy(
2892 NewAI.getContext(),
2893 DL.getTypeSizeInBits(NewAllocaTy).getFixedValue())
2894 : nullptr),
2895 VecTy(PromotableVecTy),
2896 ElementTy(VecTy ? VecTy->getElementType() : nullptr),
2897 ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy).getFixedValue() / 8
2898 : 0),
2899 PHIUsers(PHIUsers), SelectUsers(SelectUsers),
2900 IRB(NewAI.getContext(), ConstantFolder()) {
2901 if (VecTy) {
2902 assert((DL.getTypeSizeInBits(ElementTy).getFixedValue() % 8) == 0 &&
2903 "Only multiple-of-8 sized vector elements are viable");
2904 ++NumVectorized;
2905 }
2906 assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy));
2907 }
2908
2909 bool visit(AllocaSlices::const_iterator I) {
2910 bool CanSROA = true;
2911 BeginOffset = I->beginOffset();
2912 EndOffset = I->endOffset();
2913 IsSplittable = I->isSplittable();
2914 IsSplit =
2915 BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset;
2916 LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""));
2917 LLVM_DEBUG(AS.printSlice(dbgs(), I, ""));
2918 LLVM_DEBUG(dbgs() << "\n");
2919
2920 // Compute the intersecting offset range.
2921 assert(BeginOffset < NewAllocaEndOffset);
2922 assert(EndOffset > NewAllocaBeginOffset);
2923 NewBeginOffset = std::max(BeginOffset, NewAllocaBeginOffset);
2924 NewEndOffset = std::min(EndOffset, NewAllocaEndOffset);
2925
2926 SliceSize = NewEndOffset - NewBeginOffset;
2927 LLVM_DEBUG(dbgs() << " Begin:(" << BeginOffset << ", " << EndOffset
2928 << ") NewBegin:(" << NewBeginOffset << ", "
2929 << NewEndOffset << ") NewAllocaBegin:("
2930 << NewAllocaBeginOffset << ", " << NewAllocaEndOffset
2931 << ")\n");
2932 assert(IsSplit || NewBeginOffset == BeginOffset);
2933 OldUse = I->getUse();
2934 OldPtr = cast<Instruction>(OldUse->get());
2935
2936 Instruction *OldUserI = cast<Instruction>(OldUse->getUser());
2937 IRB.SetInsertPoint(OldUserI);
2938 IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc());
2939 // Avoid materializing the name prefix when it is discarded anyway.
2940 if (!IRB.getContext().shouldDiscardValueNames())
2941 IRB.getInserter().SetNamePrefix(Twine(NewAI.getName()) + "." +
2942 Twine(BeginOffset) + ".");
2943
2944 CanSROA &= visit(cast<Instruction>(OldUse->getUser()));
2945 if (VecTy || IntTy)
2946 assert(CanSROA);
2947 return CanSROA;
2948 }
2949
2950 /// Attempts to rewrite a partition using tree-structured merge optimization.
2951 ///
2952 /// This function handles two patterns. Both produce an O(log n) tree of
2953 /// shufflevectors in place of the linear expand+blend chain that SROA would
2954 /// otherwise emit for each partial store.
2955 ///
2956 /// Pattern 1 (stores-only):
2957 /// Multiple non-overlapping partial stores completely fill the alloca
2958 /// and there is exactly one full-width load coming after the stores.
2959 /// The stores are tree-merged into a single vector and stored once.
2960 ///
2961 /// Example transformation:
2962 /// Before: (stores do not have to be in order)
2963 /// %alloca = alloca <8 x float>
2964 /// store <2 x float> %val0, ptr %alloca ; offset 0-1
2965 /// store <2 x float> %val2, ptr %alloca+16 ; offset 4-5
2966 /// store <2 x float> %val1, ptr %alloca+8 ; offset 2-3
2967 /// store <2 x float> %val3, ptr %alloca+24 ; offset 6-7
2968 /// %r = load <8 x float>, ptr %alloca
2969 ///
2970 /// After: tree of shufflevectors producing <8 x float> directly.
2971 ///
2972 /// Pattern 2 (init + RMW, possibly multi-round):
2973 /// A single full-width init store, followed by partial loads and
2974 /// partial stores that read-modify-write the alloca one or more
2975 /// times, optionally followed by a full-width load. The only
2976 /// structural requirement is that the distinct [begin, end) ranges
2977 /// touched by the partial loads and stores, taken together, tile
2978 /// the alloca disjointly.
2979 ///
2980 /// We keep a map from each slice range to the SSA value that
2981 /// currently lives there, `SliceValues[r] -> Value*`:
2982 /// - initialize each entry to the corresponding piece of the
2983 /// init store's value (via a shufflevector picking the
2984 /// range's elements out of the init value),
2985 /// - walk partial loads and stores in block order,
2986 /// - for a partial load at range r: RAUW with `SliceValues[r]`,
2987 /// - for a partial store at range r: update `SliceValues[r]` to
2988 /// the stored value and drop the store.
2989 /// At the end, the final `SliceValues[r]` entries are tree-merged
2990 /// (in range order) into a single store to the alloca, and the
2991 /// optional full-width load is replaced by a load of the alloca.
2992 ///
2993 /// Because the ranges are disjoint by construction, a store at one
2994 /// range cannot affect another range's tracked value, so a single
2995 /// block-order walk correctly tracks the memory state at each
2996 /// range. The algorithm handles multi-round RMW, partial loads
2997 /// and stores interleaved in any order, read-only slices (the
2998 /// tracked value stays at the init extract), and write-only
2999 /// slices (the tracked value never flows into a load).
3000 ///
3001 /// \param P The partition to analyze and potentially rewrite
3002 /// \return An optional vector of values that were deleted during the
3003 /// rewrite, or std::nullopt if the partition cannot be optimized.
3004 std::optional<SmallVector<Value *, 4>>
3005 rewriteTreeStructuredMerge(Partition &P) {
3006 // No tail slices that overlap with the partition
3007 if (P.splitSliceTails().size() > 0)
3008 return std::nullopt;
3009
3010 // Structure to hold store information
3011 struct StoreInfo {
3012 StoreInst *Store;
3013 uint64_t BeginOffset;
3014 uint64_t EndOffset;
3015 Value *StoredValue;
3016 StoreInfo(StoreInst *SI, uint64_t Begin, uint64_t End, Value *Val)
3017 : Store(SI), BeginOffset(Begin), EndOffset(End), StoredValue(Val) {}
3018 };
3019 struct LoadInfo {
3020 LoadInst *Load;
3021 uint64_t BeginOffset;
3022 uint64_t EndOffset;
3023 };
3024
3025 SmallVector<StoreInfo, 4> StoreInfos; // partial stores only
3026 SmallVector<LoadInfo, 4> LoadInfos; // partial loads only
3027 LoadInst *FullLoad = nullptr; // optional full-width load
3028 StoreInst *InitStore = nullptr; // optional full-width init store
3029
3030 // If the new alloca is a fixed vector type, we use its element type as the
3031 // allocated element type, otherwise we use i8 as the allocated element
3032 Type *AllocatedEltTy =
3033 isa<FixedVectorType>(NewAllocaTy)
3034 ? cast<FixedVectorType>(NewAllocaTy)->getElementType()
3035 : Type::getInt8Ty(NewAI.getContext());
3036 unsigned AllocatedEltTySize = DL.getTypeSizeInBits(AllocatedEltTy);
3037
3038 // Helper to check if a type is
3039 // 1. A fixed vector type
3040 // 2. The element type is not a pointer
3041 // 3. The element type size is byte-aligned
3042 // We only handle the cases that the ld/st meet these conditions
3043 auto IsTypeValidForTreeStructuredMerge = [&](Type *Ty) -> bool {
3044 auto *FixedVecTy = dyn_cast<FixedVectorType>(Ty);
3045 return FixedVecTy &&
3046 DL.getTypeSizeInBits(FixedVecTy->getElementType()) % 8 == 0 &&
3047 !FixedVecTy->getElementType()->isPointerTy();
3048 };
3049
3050 for (Slice &S : P) {
3051 auto *User = cast<Instruction>(S.getUse()->getUser());
3052 // A "full-width" slice spans the entire alloca; it's either the single
3053 // init store (Pattern 2) or the single final load (both patterns).
3054 bool IsFullWidth = (S.beginOffset() == NewAllocaBeginOffset &&
3055 S.endOffset() == NewAllocaEndOffset);
3056 if (auto *LI = dyn_cast<LoadInst>(User)) {
3057 // Only handle simple (non-volatile, non-atomic) loads.
3058 if (!LI->isSimple() ||
3059 !IsTypeValidForTreeStructuredMerge(LI->getType()))
3060 return std::nullopt;
3061 if (IsFullWidth) {
3062 // We accept at most one full-width load (the "final" load, after
3063 // all the partial stores).
3064 if (FullLoad)
3065 return std::nullopt;
3066 FullLoad = LI;
3067 } else {
3068 // Partial load (RMW pattern only).
3069 LoadInfos.push_back({LI, S.beginOffset(), S.endOffset()});
3070 }
3071 } else if (auto *SI = dyn_cast<StoreInst>(User)) {
3072 // Do not handle the case if
3073 // 1. The store does not meet the conditions in the helper function
3074 // 2. The store is not simple — we drop stores as part of the
3075 // rewrite, so volatile stores (which must be kept) and atomic
3076 // stores (which carry memory-ordering semantics) are unsound
3077 // to replace with SSA bookkeeping.
3078 // 3. The total store size is not a multiple of the allocated
3079 // element type size (required so the tree merge can produce a
3080 // vector whose element type matches the alloca).
3081 if (!SI->isSimple() || !IsTypeValidForTreeStructuredMerge(
3082 SI->getValueOperand()->getType()))
3083 return std::nullopt;
3084 auto *StVecTy = cast<FixedVectorType>(SI->getValueOperand()->getType());
3085 unsigned NumElts = StVecTy->getNumElements();
3086 unsigned EltSize = DL.getTypeSizeInBits(StVecTy->getElementType());
3087 if (NumElts * EltSize % AllocatedEltTySize != 0)
3088 return std::nullopt;
3089 if (IsFullWidth) {
3090 // At most one full-width store is allowed — it's the init store
3091 // for the RMW pattern.
3092 if (InitStore)
3093 return std::nullopt;
3094 InitStore = SI;
3095 } else {
3096 StoreInfos.emplace_back(SI, S.beginOffset(), S.endOffset(),
3097 SI->getValueOperand());
3098 }
3099 } else {
3100 // If we have instructions other than load and store, we cannot do
3101 // the tree structured merge.
3102 return std::nullopt;
3103 }
3104 }
3105
3106 // Need at least two partial stores to benefit from tree-merging; a
3107 // single store is already optimal as-is. This applies to both patterns
3108 // below, so check it before classifying.
3109 if (StoreInfos.size() < 2)
3110 return std::nullopt;
3111
3112 // Classify the pattern by looking at what we collected:
3113 // Pattern 1 (stores-only): only partial stores + exactly one full load.
3114 // Pattern 2 (RMW): one full init store + partial loads + partial stores
3115 // (+ optional full final load). RMW also needs VecTy to be set
3116 // because we use getIndex() to convert byte offsets to element
3117 // indices, which requires a promoted vector alloca.
3118 bool IsRMWPattern = InitStore && VecTy && !LoadInfos.empty();
3119 bool IsStoresOnlyPattern = !InitStore && FullLoad && LoadInfos.empty();
3120 if (!IsRMWPattern && !IsStoresOnlyPattern)
3121 return std::nullopt;
3122
3123 // All partial stores must live in the same basic block — the tree merge
3124 // is built in a single BB using block-order ordering (comesBefore).
3125 BasicBlock *StoreBB = StoreInfos[0].Store->getParent();
3126 for (auto &Info : StoreInfos)
3127 if (Info.Store->getParent() != StoreBB)
3128 return std::nullopt;
3129
3130 SmallVector<Value *, 4> DeletedValues;
3131
3132 // Helper: pairwise tree-merge a list of vectors into a single vector.
3133 // At each iteration we merge each adjacent pair via mergeTwoVectors,
3134 // collect the merged values into Next, and (if Vals had odd length)
3135 // carry the trailing element through unchanged. Loop until one value
3136 // remains — the fully-merged vector.
3137 auto TreeMerge = [&](SmallVectorImpl<Value *> &Vals,
3138 IRBuilder<> &B) -> Value * {
3139 LLVM_DEBUG(dbgs() << " Rewrite stores into shufflevectors:\n");
3140 while (Vals.size() > 1) {
3141 SmallVector<Value *, 8> Next;
3142 for (unsigned I = 0, E = Vals.size(); I + 1 < E; I += 2) {
3143 Value *M =
3144 mergeTwoVectors(Vals[I], Vals[I + 1], DL, AllocatedEltTy, B);
3145 LLVM_DEBUG(dbgs() << " shufflevector: " << *M << "\n");
3146 Next.push_back(M);
3147 }
3148 if (Vals.size() % 2 == 1)
3149 Next.push_back(Vals.back());
3150 Vals = std::move(Next);
3151 }
3152 return Vals[0];
3153 };
3154
3155 // Replace a full-width load with a load of the freshly-merged alloca.
3156 // The merge stored a value of type Merged->getType() into NewAI; we load
3157 // that same type back so every access to NewAI stays consistently typed
3158 // (otherwise the alloca is no longer promotable).
3159 auto ReplaceFullLoad = [&](LoadInst *LoadToReplace, Value *Merged) {
3160 IRBuilder<> LoadBuilder(LoadToReplace);
3161 Value *NewLoad = LoadBuilder.CreateAlignedLoad(
3162 Merged->getType(), &NewAI, getSliceAlign(),
3163 LoadToReplace->isVolatile(),
3164 LoadToReplace->getName() + ".sroa.new.load");
3165 if (NewLoad->getType() != LoadToReplace->getType())
3166 NewLoad = LoadBuilder.CreateBitCast(NewLoad, LoadToReplace->getType());
3167 LoadToReplace->replaceAllUsesWith(NewLoad);
3168 DeletedValues.push_back(LoadToReplace);
3169 };
3170
3171 if (IsStoresOnlyPattern) {
3172 // Stores should not overlap and should cover the whole alloca.
3173 // Sort by begin offset to verify this with a single linear scan.
3174 llvm::sort(StoreInfos, [](const StoreInfo &A, const StoreInfo &B) {
3175 return A.BeginOffset < B.BeginOffset;
3176 });
3177 // Check for gap or overlap: each begin offset must equal the previous
3178 // end offset, i.e. the store ranges must tile [NewAllocaBeginOffset,
3179 // NewAllocaEndOffset) exactly.
3180 uint64_t Expected = NewAllocaBeginOffset;
3181 for (auto &Info : StoreInfos) {
3182 if (Info.BeginOffset != Expected)
3183 return std::nullopt;
3184 Expected = Info.EndOffset;
3185 }
3186 // Stores cover the entire alloca (no trailing gap either).
3187 if (Expected != NewAllocaEndOffset)
3188 return std::nullopt;
3189
3190 // The load should not be in the middle of the stores.
3191 // Note:
3192 // If the load is in a different basic block from the stores, we can
3193 // still do the tree-structured merge. We don't have store->load
3194 // forwarding here — the merged vector is stored back to NewAI and
3195 // the new load loads from NewAI. The forwarding will be handled
3196 // later when NewAI is promoted.
3197 BasicBlock *LoadBB = FullLoad->getParent();
3198 if (LoadBB == StoreBB) {
3199 for (auto &Info : StoreInfos)
3200 if (!Info.Store->comesBefore(FullLoad))
3201 return std::nullopt;
3202 }
3203
3204 LLVM_DEBUG({
3205 dbgs() << "Tree structured merge rewrite (stores-only):\n";
3206 dbgs() << " Load: " << *FullLoad << "\n Ordered stores:\n";
3207 for (auto [I, Info] : enumerate(StoreInfos)) {
3208 dbgs() << " [" << I << "] Range[" << Info.BeginOffset << ", "
3209 << Info.EndOffset << ") \tStore: " << *Info.Store
3210 << "\tValue: " << *Info.StoredValue << "\n";
3211 }
3212 });
3213
3214 // StoreInfos is sorted by offset, not by block order. Anchoring to
3215 // StoreInfos.back().Store (last by offset) can place shuffles before
3216 // operands that appear later in the block (invalid SSA). Insert before
3217 // FullLoad when it shares the store block (after all stores, before
3218 // any later IR in that block). Otherwise insert before the store
3219 // block's terminator so the merge runs after every store and any
3220 // trailing instructions in that block.
3221 IRBuilder<> Builder(LoadBB == StoreBB ? cast<Instruction>(FullLoad)
3222 : StoreBB->getTerminator());
3223 SmallVector<Value *, 8> Vals;
3224 for (const auto &Info : StoreInfos) {
3225 DeletedValues.push_back(Info.Store);
3226 Vals.push_back(Info.StoredValue);
3227 }
3228 // Merge all stored values and store the merged value into the alloca.
3229 Value *Merged = TreeMerge(Vals, Builder);
3230 Builder.CreateAlignedStore(Merged, &NewAI, getSliceAlign());
3231
3232 // Replace the original load with a load of the newly-merged alloca.
3233 ReplaceFullLoad(FullLoad, Merged);
3234 return DeletedValues;
3235 }
3236
3237 // RMW pattern handling starts from here.
3238 // Like StoreBB above: keep the init store, all partial loads and all
3239 // partial stores in one basic block so we can reason about ordering
3240 // with comesBefore and build SSA without PHIs.
3241 if (InitStore->getParent() != StoreBB)
3242 return std::nullopt;
3243 if (any_of(LoadInfos, [&](const LoadInfo &I) {
3244 return I.Load->getParent() != StoreBB;
3245 }))
3246 return std::nullopt;
3247 // FullLoad (if any) is allowed to live in a different basic block. See
3248 // the note on the stores-only path: we don't do store->load forwarding
3249 // directly — the merged vector is stored to NewAI and the new load
3250 // loads from NewAI, so cross-BB ordering is resolved later when NewAI
3251 // is promoted.
3252
3253 // Collect the combined partial-load/partial-store accesses sorted
3254 // by block order. Used both for ordering checks and for the rewrite
3255 // walk below.
3256 struct Access {
3257 Instruction *Inst;
3258 uint64_t BeginOffset, EndOffset;
3259 bool IsStore;
3260 };
3262 Accesses.reserve(LoadInfos.size() + StoreInfos.size());
3263 for (const auto &L : LoadInfos)
3264 Accesses.push_back({L.Load, L.BeginOffset, L.EndOffset, false});
3265 for (const auto &S : StoreInfos)
3266 Accesses.push_back({S.Store, S.BeginOffset, S.EndOffset, true});
3267 llvm::sort(Accesses, [](const Access &A, const Access &B) {
3268 return A.Inst->comesBefore(B.Inst);
3269 });
3270
3271 // Ordering constraint 1: InitStore must come before every partial
3272 // access — they read/write the RMW state initialised by InitStore.
3273 // Accesses is sorted by block order, so the first element is the
3274 // earliest; checking it is enough.
3275 if (!InitStore->comesBefore(Accesses.front().Inst))
3276 return std::nullopt;
3277 // Ordering constraint 2: when FullLoad shares the block with the
3278 // partial accesses, it must come after every one of them — otherwise
3279 // it could read a stale value. Accesses is sorted, so the last
3280 // element is the latest; checking it is enough. If FullLoad is in
3281 // another block, mem2reg forwards the merged store to it.
3282 if (FullLoad && FullLoad->getParent() == StoreBB &&
3283 !Accesses.back().Inst->comesBefore(FullLoad))
3284 return std::nullopt;
3285
3286 // Coverage check: the distinct [begin, end) ranges touched by the
3287 // partial loads and stores must tile the alloca disjointly. That is
3288 // the only precondition the per-range SliceValues tracking below
3289 // needs — a disjoint tile guarantees the entries don't alias each
3290 // other. We don't check per-range load/store counts: a range with
3291 // only loads ends with SliceValues[r] = the init extract
3292 // (contributed to the final tree-merge), and a range with only
3293 // stores ends with SliceValues[r] = its last stored value. Both are
3294 // correct.
3295 using SliceRange = std::pair<uint64_t, uint64_t>;
3296 SmallVector<SliceRange, 8> SortedRanges;
3297 SortedRanges.reserve(Accesses.size());
3298 for (auto &Acc : Accesses)
3299 SortedRanges.emplace_back(Acc.BeginOffset, Acc.EndOffset);
3300 llvm::sort(SortedRanges);
3301 SortedRanges.erase(llvm::unique(SortedRanges), SortedRanges.end());
3302 // Disjoint + contiguous tile of the whole alloca.
3303 uint64_t Expected = NewAllocaBeginOffset;
3304 for (auto &Range : SortedRanges) {
3305 if (Range.first != Expected)
3306 return std::nullopt;
3307 Expected = Range.second;
3308 }
3309 if (Expected != NewAllocaEndOffset)
3310 return std::nullopt;
3311
3312 LLVM_DEBUG({
3313 dbgs() << "Tree structured merge rewrite (RMW):\n";
3314 dbgs() << " Init store: " << *InitStore << "\n";
3315 if (FullLoad)
3316 dbgs() << " Final load: " << *FullLoad << "\n";
3317 dbgs() << " Slice ranges (" << SortedRanges.size() << "):\n";
3318 for (auto &Range : SortedRanges)
3319 dbgs() << " [" << Range.first << ", " << Range.second << ")\n";
3320 });
3321
3322 // Initialize SliceValues: one SSA value per slice range, tracking
3323 // the value the alloca currently holds at that range. Each entry
3324 // starts at the corresponding piece of the init store, obtained by
3325 // bitcasting the init value to the alloca's vector type (if needed)
3326 // and extracting the slice's sub-range.
3327 IRB.SetInsertPoint(InitStore->getNextNode());
3328 Value *InitVec = InitStore->getValueOperand();
3329 if (InitVec->getType() != NewAllocaTy)
3330 InitVec = IRB.CreateBitCast(InitVec, NewAllocaTy, "init.cast");
3331 DenseMap<SliceRange, Value *> SliceValues;
3332 for (auto &Range : SortedRanges) {
3333 unsigned BeginIdx = getIndex(Range.first);
3334 unsigned EndIdx = getIndex(Range.second);
3335 SliceValues[Range] = IRB.CreateShuffleVector(
3336 InitVec, createSequentialMask(BeginIdx, EndIdx - BeginIdx, 0),
3337 "init.extract");
3338 }
3339 // The init store itself becomes dead — its value is consumed via the
3340 // extracts above.
3341 DeletedValues.push_back(InitStore);
3342
3343 // Walk accesses in block order:
3344 // - partial load at range r: replace with SliceValues[r] (bitcast
3345 // if the load's type differs from the current tracked value's
3346 // type, e.g. because a previous store wrote a vector with a
3347 // different element type);
3348 // - partial store at range r: update SliceValues[r] to the stored
3349 // value and drop the store.
3350 for (auto &Acc : Accesses) {
3351 SliceRange Range{Acc.BeginOffset, Acc.EndOffset};
3352 if (!Acc.IsStore) {
3353 Value *V = SliceValues[Range];
3354 if (V->getType() != Acc.Inst->getType()) {
3355 IRB.SetInsertPoint(cast<LoadInst>(Acc.Inst));
3356 V = IRB.CreateBitCast(V, Acc.Inst->getType());
3357 }
3358 Acc.Inst->replaceAllUsesWith(V);
3359 } else {
3360 SliceValues[Range] = cast<StoreInst>(Acc.Inst)->getValueOperand();
3361 }
3362 DeletedValues.push_back(Acc.Inst);
3363 }
3364
3365 // Tree-merge the final per-range values (in range order) into the
3366 // alloca's final vector value. Anchor the IRBuilder to FullLoad (when it
3367 // shares the partial-access block) or otherwise to the block's
3368 // terminator — never to a partial access, since those are queued for
3369 // deletion. Both anchors are guaranteed to dominate every SliceValues
3370 // entry: each one is either an init extract (before any access) or a
3371 // stored value defined before its (now-deleted) store.
3372 IRBuilder<> Builder(FullLoad && FullLoad->getParent() == StoreBB
3373 ? cast<Instruction>(FullLoad)
3374 : StoreBB->getTerminator());
3375 SmallVector<Value *, 8> Vals;
3376 for (auto &Range : SortedRanges)
3377 Vals.push_back(SliceValues[Range]);
3378 Value *Merged = TreeMerge(Vals, Builder);
3379 Builder.CreateAlignedStore(Merged, &NewAI, getSliceAlign());
3380
3381 // Replace the optional final full-width load with a load of the newly
3382 // merged alloca. Later promotion will forward the store above to it.
3383 if (FullLoad)
3384 ReplaceFullLoad(FullLoad, Merged);
3385
3386 return DeletedValues;
3387 }
3388
3389private:
3390 // Make sure the other visit overloads are visible.
3391 using Base::visit;
3392
3393 // Every instruction which can end up as a user must have a rewrite rule.
3394 bool visitInstruction(Instruction &I) {
3395 LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n");
3396 llvm_unreachable("No rewrite rule for this instruction!");
3397 }
3398
3399 Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) {
3400 // Note that the offset computation can use BeginOffset or NewBeginOffset
3401 // interchangeably for unsplit slices.
3402 assert(IsSplit || BeginOffset == NewBeginOffset);
3403 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3404
3405 StringRef OldName = OldPtr->getName();
3406 // Skip through the last '.sroa.' component of the name.
3407 size_t LastSROAPrefix = OldName.rfind(".sroa.");
3408 if (LastSROAPrefix != StringRef::npos) {
3409 OldName = OldName.substr(LastSROAPrefix + strlen(".sroa."));
3410 // Look for an SROA slice index.
3411 size_t IndexEnd = OldName.find_first_not_of("0123456789");
3412 if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') {
3413 // Strip the index and look for the offset.
3414 OldName = OldName.substr(IndexEnd + 1);
3415 size_t OffsetEnd = OldName.find_first_not_of("0123456789");
3416 if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.')
3417 // Strip the offset.
3418 OldName = OldName.substr(OffsetEnd + 1);
3419 }
3420 }
3421 // Strip any SROA suffixes as well.
3422 OldName = OldName.substr(0, OldName.find(".sroa_"));
3423
3424 return getAdjustedPtr(IRB, DL, &NewAI,
3425 APInt(DL.getIndexTypeSizeInBits(PointerTy), Offset),
3426 PointerTy, Twine(OldName) + ".");
3427 }
3428
3429 /// Compute suitable alignment to access this slice of the *new*
3430 /// alloca.
3431 ///
3432 /// You can optionally pass a type to this routine and if that type's ABI
3433 /// alignment is itself suitable, this will return zero.
3434 Align getSliceAlign() {
3435 return commonAlignment(NewAI.getAlign(),
3436 NewBeginOffset - NewAllocaBeginOffset);
3437 }
3438
3439 unsigned getIndex(uint64_t Offset) {
3440 assert(VecTy && "Can only call getIndex when rewriting a vector");
3441 uint64_t RelOffset = Offset - NewAllocaBeginOffset;
3442 assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
3443 uint32_t Index = RelOffset / ElementSize;
3444 assert(Index * ElementSize == RelOffset);
3445 return Index;
3446 }
3447
3448 void deleteIfTriviallyDead(Value *V) {
3451 Pass.DeadInsts.push_back(I);
3452 }
3453
3454 Value *rewriteVectorizedLoadInst(LoadInst &LI) {
3455 unsigned BeginIndex = getIndex(NewBeginOffset);
3456 unsigned EndIndex = getIndex(NewEndOffset);
3457 assert(EndIndex > BeginIndex && "Empty vector!");
3458
3459 LoadInst *Load =
3460 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
3461
3462 Load->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
3463 LLVMContext::MD_access_group});
3464 return extractVector(IRB, Load, BeginIndex, EndIndex, "vec");
3465 }
3466
3467 Value *rewriteIntegerLoad(LoadInst &LI) {
3468 assert(IntTy && "We cannot insert an integer to the alloca");
3469 assert(!LI.isVolatile());
3470 Value *V =
3471 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
3472 V = IRB.CreateBitPreservingCastChain(DL, V, IntTy);
3473 assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3474 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3475 if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) {
3476 IntegerType *ExtractTy = Type::getIntNTy(LI.getContext(), SliceSize * 8);
3477 V = extractInteger(DL, IRB, V, ExtractTy, Offset, "extract");
3478 }
3479 // It is possible that the extracted type is not the load type. This
3480 // happens if there is a load past the end of the alloca, and as
3481 // a consequence the slice is narrower but still a candidate for integer
3482 // lowering. To handle this case, we just zero extend the extracted
3483 // integer.
3484 assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&
3485 "Can only handle an extract for an overly wide load");
3486 if (cast<IntegerType>(LI.getType())->getBitWidth() > SliceSize * 8)
3487 V = IRB.CreateZExt(V, LI.getType());
3488 return V;
3489 }
3490
3491 bool visitLoadInst(LoadInst &LI) {
3492 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
3493 Value *OldOp = LI.getOperand(0);
3494 assert(OldOp == OldPtr);
3495
3496 AAMDNodes AATags = LI.getAAMetadata();
3497
3498 unsigned AS = LI.getPointerAddressSpace();
3499
3500 Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8)
3501 : LI.getType();
3502 bool IsPtrAdjusted = false;
3503 Value *V;
3504 if (VecTy) {
3505 V = rewriteVectorizedLoadInst(LI);
3506 } else if (IntTy && LI.getType()->isIntegerTy()) {
3507 V = rewriteIntegerLoad(LI);
3508 } else if (NewBeginOffset == NewAllocaBeginOffset &&
3509 NewEndOffset == NewAllocaEndOffset &&
3510 (canConvertValue(DL, NewAllocaTy, TargetTy) ||
3511 (NewAllocaTy->isIntegerTy() && TargetTy->isIntegerTy() &&
3512 DL.getTypeStoreSize(TargetTy).getFixedValue() > SliceSize &&
3513 !LI.isVolatile()))) {
3514 Value *NewPtr =
3515 getPtrToNewAI(LI.getPointerAddressSpace(), LI.isVolatile());
3516 LoadInst *NewLI = IRB.CreateAlignedLoad(
3517 NewAllocaTy, NewPtr, NewAI.getAlign(), LI.isVolatile(), LI.getName());
3518 if (LI.isVolatile())
3519 NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
3520 if (NewLI->isAtomic())
3521 NewLI->setAlignment(LI.getAlign());
3522
3523 // Copy any metadata that is valid for the new load. This may require
3524 // conversion to a different kind of metadata, e.g. !nonnull might change
3525 // to !range or vice versa.
3526 copyMetadataForLoad(*NewLI, LI);
3527
3528 // Do this after copyMetadataForLoad() to preserve the TBAA shift.
3529 if (AATags)
3530 NewLI->setAAMetadata(AATags.adjustForAccess(
3531 NewBeginOffset - BeginOffset, NewLI->getType(), DL));
3532
3533 // Try to preserve nonnull metadata
3534 V = NewLI;
3535
3536 // If this is an integer load past the end of the slice (which means the
3537 // bytes outside the slice are undef or this load is dead) just forcibly
3538 // fix the integer size with correct handling of endianness.
3539 if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy))
3540 if (auto *TITy = dyn_cast<IntegerType>(TargetTy))
3541 if (AITy->getBitWidth() < TITy->getBitWidth()) {
3542 V = IRB.CreateZExt(V, TITy, "load.ext");
3543 if (DL.isBigEndian())
3544 V = IRB.CreateShl(V, TITy->getBitWidth() - AITy->getBitWidth(),
3545 "endian_shift");
3546 }
3547 } else {
3548 Type *LTy = IRB.getPtrTy(AS);
3549 LoadInst *NewLI =
3550 IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy),
3551 getSliceAlign(), LI.isVolatile(), LI.getName());
3552
3553 if (AATags)
3554 NewLI->setAAMetadata(AATags.adjustForAccess(
3555 NewBeginOffset - BeginOffset, NewLI->getType(), DL));
3556
3557 if (LI.isVolatile())
3558 NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
3559 NewLI->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
3560 LLVMContext::MD_access_group});
3561
3562 V = NewLI;
3563 IsPtrAdjusted = true;
3564 }
3565 V = IRB.CreateBitPreservingCastChain(DL, V, TargetTy);
3566
3567 if (IsSplit) {
3568 assert(!LI.isVolatile());
3569 assert(LI.getType()->isIntegerTy() &&
3570 "Only integer type loads and stores are split");
3571 assert(SliceSize < DL.getTypeStoreSize(LI.getType()).getFixedValue() &&
3572 "Split load isn't smaller than original load");
3573 assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&
3574 "Non-byte-multiple bit width");
3575 // Move the insertion point just past the load so that we can refer to it.
3576 BasicBlock::iterator LIIt = std::next(LI.getIterator());
3577 // Ensure the insertion point comes before any debug-info immediately
3578 // after the load, so that variable values referring to the load are
3579 // dominated by it.
3580 LIIt.setHeadBit(true);
3581 IRB.SetInsertPoint(LI.getParent(), LIIt);
3582 // Create a placeholder value with the same type as LI to use as the
3583 // basis for the new value. This allows us to replace the uses of LI with
3584 // the computed value, and then replace the placeholder with LI, leaving
3585 // LI only used for this computation.
3586 Value *Placeholder =
3587 new LoadInst(LI.getType(), PoisonValue::get(IRB.getPtrTy(AS)), "",
3588 false, Align(1));
3589 V = insertInteger(DL, IRB, Placeholder, V, NewBeginOffset - BeginOffset,
3590 "insert");
3591 LI.replaceAllUsesWith(V);
3592 Placeholder->replaceAllUsesWith(&LI);
3593 Placeholder->deleteValue();
3594 } else {
3595 LI.replaceAllUsesWith(V);
3596 }
3597
3598 Pass.DeadInsts.push_back(&LI);
3599 deleteIfTriviallyDead(OldOp);
3600 LLVM_DEBUG(dbgs() << " to: " << *V << "\n");
3601 return !LI.isVolatile() && !IsPtrAdjusted;
3602 }
3603
3604 bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp,
3605 AAMDNodes AATags) {
3606 // Capture V for the purpose of debug-info accounting once it's converted
3607 // to a vector store.
3608 Value *OrigV = V;
3609 if (V->getType() != VecTy) {
3610 unsigned BeginIndex = getIndex(NewBeginOffset);
3611 unsigned EndIndex = getIndex(NewEndOffset);
3612 assert(EndIndex > BeginIndex && "Empty vector!");
3613 unsigned NumElements = EndIndex - BeginIndex;
3614 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3615 "Too many elements!");
3616 Type *SliceTy = (NumElements == 1)
3617 ? ElementTy
3618 : FixedVectorType::get(ElementTy, NumElements);
3619 if (V->getType() != SliceTy)
3620 V = IRB.CreateBitPreservingCastChain(DL, V, SliceTy);
3621
3622 // Mix in the existing elements.
3623 Value *Old =
3624 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
3625 V = insertVector(IRB, Old, V, BeginIndex, "vec");
3626 }
3627 StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
3628 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
3629 LLVMContext::MD_access_group});
3630 if (AATags)
3631 Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
3632 V->getType(), DL));
3633 Pass.DeadInsts.push_back(&SI);
3634
3635 // NOTE: Careful to use OrigV rather than V.
3636 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
3637 Store, Store->getPointerOperand(), OrigV, DL);
3638 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3639 return true;
3640 }
3641
3642 bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) {
3643 assert(IntTy && "We cannot extract an integer from the alloca");
3644 assert(!SI.isVolatile());
3645 if (DL.getTypeSizeInBits(V->getType()).getFixedValue() !=
3646 IntTy->getBitWidth()) {
3647 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(),
3648 "oldload");
3649 Old = IRB.CreateBitPreservingCastChain(DL, Old, IntTy);
3650 assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
3651 uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
3652 V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert");
3653 }
3654 V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
3655 StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
3656 Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
3657 LLVMContext::MD_access_group});
3658 if (AATags)
3659 Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
3660 V->getType(), DL));
3661
3662 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
3663 Store, Store->getPointerOperand(),
3664 Store->getValueOperand(), DL);
3665
3666 Pass.DeadInsts.push_back(&SI);
3667 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
3668 return true;
3669 }
3670
3671 bool visitStoreInst(StoreInst &SI) {
3672 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
3673 Value *OldOp = SI.getOperand(1);
3674 assert(OldOp == OldPtr);
3675
3676 AAMDNodes AATags = SI.getAAMetadata();
3677 Value *V = SI.getValueOperand();
3678
3679 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3680 // alloca that should be re-examined after promoting this alloca.
3681 if (V->getType()->isPointerTy())
3682 if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets()))
3683 Pass.PostPromotionWorklist.insert(AI);
3684
3685 TypeSize StoreSize = DL.getTypeStoreSize(V->getType());
3686 if (StoreSize.isFixed() && SliceSize < StoreSize.getFixedValue()) {
3687 assert(!SI.isVolatile());
3688 assert(V->getType()->isIntegerTy() &&
3689 "Only integer type loads and stores are split");
3690 assert(DL.typeSizeEqualsStoreSize(V->getType()) &&
3691 "Non-byte-multiple bit width");
3692 IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8);
3693 V = extractInteger(DL, IRB, V, NarrowTy, NewBeginOffset - BeginOffset,
3694 "extract");
3695 }
3696
3697 if (VecTy)
3698 return rewriteVectorizedStoreInst(V, SI, OldOp, AATags);
3699 if (IntTy && V->getType()->isIntegerTy())
3700 return rewriteIntegerStore(V, SI, AATags);
3701
3702 StoreInst *NewSI;
3703 if (NewBeginOffset == NewAllocaBeginOffset &&
3704 NewEndOffset == NewAllocaEndOffset &&
3705 canConvertValue(DL, V->getType(), NewAllocaTy)) {
3706 V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
3707 Value *NewPtr =
3708 getPtrToNewAI(SI.getPointerAddressSpace(), SI.isVolatile());
3709
3710 NewSI =
3711 IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), SI.isVolatile());
3712 } else {
3713 unsigned AS = SI.getPointerAddressSpace();
3714 Value *NewPtr = getNewAllocaSlicePtr(IRB, IRB.getPtrTy(AS));
3715 NewSI =
3716 IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(), SI.isVolatile());
3717 }
3718 NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
3719 LLVMContext::MD_access_group});
3720 if (AATags)
3721 NewSI->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
3722 V->getType(), DL));
3723 if (SI.isVolatile())
3724 NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID());
3725 if (NewSI->isAtomic())
3726 NewSI->setAlignment(SI.getAlign());
3727
3728 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &SI,
3729 NewSI, NewSI->getPointerOperand(),
3730 NewSI->getValueOperand(), DL);
3731
3732 Pass.DeadInsts.push_back(&SI);
3733 deleteIfTriviallyDead(OldOp);
3734
3735 LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n");
3736 return NewSI->getPointerOperand() == &NewAI &&
3737 NewSI->getValueOperand()->getType() == NewAllocaTy &&
3738 !SI.isVolatile();
3739 }
3740
3741 /// Compute an integer value from splatting an i8 across the given
3742 /// number of bytes.
3743 ///
3744 /// Note that this routine assumes an i8 is a byte. If that isn't true, don't
3745 /// call this routine.
3746 /// FIXME: Heed the advice above.
3747 ///
3748 /// \param V The i8 value to splat.
3749 /// \param Size The number of bytes in the output (assuming i8 is one byte)
3750 Value *getIntegerSplat(Value *V, unsigned Size) {
3751 assert(Size > 0 && "Expected a positive number of bytes.");
3752 IntegerType *VTy = cast<IntegerType>(V->getType());
3753 assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte");
3754 if (Size == 1)
3755 return V;
3756
3757 Type *SplatIntTy = Type::getIntNTy(VTy->getContext(), Size * 8);
3758 V = IRB.CreateMul(
3759 IRB.CreateZExt(V, SplatIntTy, "zext"),
3760 IRB.CreateUDiv(Constant::getAllOnesValue(SplatIntTy),
3761 IRB.CreateZExt(Constant::getAllOnesValue(V->getType()),
3762 SplatIntTy)),
3763 "isplat");
3764 return V;
3765 }
3766
3767 /// Compute a vector splat for a given element value.
3768 Value *getVectorSplat(Value *V, unsigned NumElements) {
3769 V = IRB.CreateVectorSplat(NumElements, V, "vsplat");
3770 LLVM_DEBUG(dbgs() << " splat: " << *V << "\n");
3771 return V;
3772 }
3773
3774 bool visitMemSetInst(MemSetInst &II) {
3775 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3776 assert(II.getRawDest() == OldPtr);
3777
3778 AAMDNodes AATags = II.getAAMetadata();
3779
3780 // If the memset has a variable size, it cannot be split, just adjust the
3781 // pointer to the new alloca.
3782 if (!isa<ConstantInt>(II.getLength())) {
3783 assert(!IsSplit);
3784 assert(NewBeginOffset == BeginOffset);
3785 II.setDest(getNewAllocaSlicePtr(IRB, OldPtr->getType()));
3786 II.setDestAlignment(getSliceAlign());
3787 // In theory we should call migrateDebugInfo here. However, we do not
3788 // emit dbg.assign intrinsics for mem intrinsics storing through non-
3789 // constant geps, or storing a variable number of bytes.
3791 "AT: Unexpected link to non-const GEP");
3792 deleteIfTriviallyDead(OldPtr);
3793 return false;
3794 }
3795
3796 // Record this instruction for deletion.
3797 Pass.DeadInsts.push_back(&II);
3798
3799 Type *ScalarTy = NewAllocaTy->getScalarType();
3800
3801 const bool CanContinue = [&]() {
3802 if (VecTy || IntTy)
3803 return true;
3804 if (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset)
3805 return false;
3806 // Length must be in range for FixedVectorType.
3807 auto *C = cast<ConstantInt>(II.getLength());
3808 const uint64_t Len = C->getLimitedValue();
3809 if (Len > std::numeric_limits<unsigned>::max())
3810 return false;
3811 auto *Int8Ty = IntegerType::getInt8Ty(NewAI.getContext());
3812 auto *SrcTy = FixedVectorType::get(Int8Ty, Len);
3813 return canConvertValue(DL, SrcTy, NewAllocaTy) &&
3814 DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy).getFixedValue());
3815 }();
3816
3817 // If this doesn't map cleanly onto the alloca type, and that type isn't
3818 // a single value type, just emit a memset.
3819 if (!CanContinue) {
3820 Type *SizeTy = II.getLength()->getType();
3821 unsigned Sz = NewEndOffset - NewBeginOffset;
3822 Constant *Size = ConstantInt::get(SizeTy, Sz);
3823 MemIntrinsic *New = cast<MemIntrinsic>(IRB.CreateMemSet(
3824 getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size,
3825 MaybeAlign(getSliceAlign()), II.isVolatile()));
3826 if (AATags)
3827 New->setAAMetadata(
3828 AATags.adjustForAccess(NewBeginOffset - BeginOffset, Sz));
3829
3830 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II,
3831 New, New->getRawDest(), nullptr, DL);
3832
3833 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3834 return false;
3835 }
3836
3837 // If we can represent this as a simple value, we have to build the actual
3838 // value to store, which requires expanding the byte present in memset to
3839 // a sensible representation for the alloca type. This is essentially
3840 // splatting the byte to a sufficiently wide integer, splatting it across
3841 // any desired vector width, and bitcasting to the final type.
3842 Value *V;
3843
3844 if (VecTy) {
3845 // If this is a memset of a vectorized alloca, insert it.
3846 assert(ElementTy == ScalarTy);
3847
3848 unsigned BeginIndex = getIndex(NewBeginOffset);
3849 unsigned EndIndex = getIndex(NewEndOffset);
3850 assert(EndIndex > BeginIndex && "Empty vector!");
3851 unsigned NumElements = EndIndex - BeginIndex;
3852 assert(NumElements <= cast<FixedVectorType>(VecTy)->getNumElements() &&
3853 "Too many elements!");
3854
3855 Value *Splat = getIntegerSplat(
3856 II.getValue(), DL.getTypeSizeInBits(ElementTy).getFixedValue() / 8);
3857 Splat = IRB.CreateBitPreservingCastChain(DL, Splat, ElementTy);
3858 if (NumElements > 1)
3859 Splat = getVectorSplat(Splat, NumElements);
3860
3861 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(),
3862 "oldload");
3863 V = insertVector(IRB, Old, Splat, BeginIndex, "vec");
3864 } else if (IntTy) {
3865 // If this is a memset on an alloca where we can widen stores, insert the
3866 // set integer.
3867 assert(!II.isVolatile());
3868
3869 uint64_t Size = NewEndOffset - NewBeginOffset;
3870 V = getIntegerSplat(II.getValue(), Size);
3871
3872 if (IntTy && (NewBeginOffset != NewAllocaBeginOffset ||
3873 NewEndOffset != NewAllocaEndOffset)) {
3874 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI,
3875 NewAI.getAlign(), "oldload");
3876 Old = IRB.CreateBitPreservingCastChain(DL, Old, IntTy);
3877 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
3878 V = insertInteger(DL, IRB, Old, V, Offset, "insert");
3879 } else {
3880 assert(V->getType() == IntTy &&
3881 "Wrong type for an alloca wide integer!");
3882 }
3883 V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
3884 } else {
3885 // Established these invariants above.
3886 assert(NewBeginOffset == NewAllocaBeginOffset);
3887 assert(NewEndOffset == NewAllocaEndOffset);
3888
3889 V = getIntegerSplat(II.getValue(),
3890 DL.getTypeSizeInBits(ScalarTy).getFixedValue() / 8);
3891 if (VectorType *AllocaVecTy = dyn_cast<VectorType>(NewAllocaTy))
3892 V = getVectorSplat(
3893 V, cast<FixedVectorType>(AllocaVecTy)->getNumElements());
3894
3895 V = IRB.CreateBitPreservingCastChain(DL, V, NewAllocaTy);
3896 }
3897
3898 Value *NewPtr = getPtrToNewAI(II.getDestAddressSpace(), II.isVolatile());
3899 StoreInst *New =
3900 IRB.CreateAlignedStore(V, NewPtr, NewAI.getAlign(), II.isVolatile());
3901 New->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
3902 LLVMContext::MD_access_group});
3903 if (AATags)
3904 New->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
3905 V->getType(), DL));
3906
3907 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II,
3908 New, New->getPointerOperand(), V, DL);
3909
3910 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
3911 return !II.isVolatile();
3912 }
3913
3914 bool visitMemTransferInst(MemTransferInst &II) {
3915 // Rewriting of memory transfer instructions can be a bit tricky. We break
3916 // them into two categories: split intrinsics and unsplit intrinsics.
3917
3918 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
3919
3920 AAMDNodes AATags = II.getAAMetadata();
3921
3922 bool IsDest = &II.getRawDestUse() == OldUse;
3923 assert((IsDest && II.getRawDest() == OldPtr) ||
3924 (!IsDest && II.getRawSource() == OldPtr));
3925
3926 Align SliceAlign = getSliceAlign();
3927 // For unsplit intrinsics, we simply modify the source and destination
3928 // pointers in place. This isn't just an optimization, it is a matter of
3929 // correctness. With unsplit intrinsics we may be dealing with transfers
3930 // within a single alloca before SROA ran, or with transfers that have
3931 // a variable length. We may also be dealing with memmove instead of
3932 // memcpy, and so simply updating the pointers is the necessary for us to
3933 // update both source and dest of a single call.
3934 if (!IsSplittable) {
3935 Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
3936 if (IsDest) {
3937 // Update the address component of linked dbg.assigns.
3938 for (DbgVariableRecord *DbgAssign : at::getDVRAssignmentMarkers(&II)) {
3939 if (llvm::is_contained(DbgAssign->location_ops(), II.getDest()) ||
3940 DbgAssign->getAddress() == II.getDest())
3941 DbgAssign->replaceVariableLocationOp(II.getDest(), AdjustedPtr);
3942 }
3943 II.setDest(AdjustedPtr);
3944 II.setDestAlignment(SliceAlign);
3945 } else {
3946 II.setSource(AdjustedPtr);
3947 II.setSourceAlignment(SliceAlign);
3948 }
3949
3950 LLVM_DEBUG(dbgs() << " to: " << II << "\n");
3951 deleteIfTriviallyDead(OldPtr);
3952 return false;
3953 }
3954 // For split transfer intrinsics we have an incredibly useful assurance:
3955 // the source and destination do not reside within the same alloca, and at
3956 // least one of them does not escape. This means that we can replace
3957 // memmove with memcpy, and we don't need to worry about all manner of
3958 // downsides to splitting and transforming the operations.
3959
3960 // If this doesn't map cleanly onto the alloca type, and that type isn't
3961 // a single value type, just emit a memcpy.
3962 bool EmitMemCpy =
3963 !VecTy && !IntTy &&
3964 (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset ||
3965 SliceSize != DL.getTypeStoreSize(NewAllocaTy).getFixedValue() ||
3966 !DL.typeSizeEqualsStoreSize(NewAllocaTy) ||
3967 !NewAllocaTy->isSingleValueType());
3968
3969 // If we're just going to emit a memcpy, the alloca hasn't changed, and the
3970 // size hasn't been shrunk based on analysis of the viable range, this is
3971 // a no-op.
3972 if (EmitMemCpy && &OldAI == &NewAI) {
3973 // Ensure the start lines up.
3974 assert(NewBeginOffset == BeginOffset);
3975
3976 // Rewrite the size as needed.
3977 if (NewEndOffset != EndOffset)
3978 II.setLength(NewEndOffset - NewBeginOffset);
3979 return false;
3980 }
3981 // Record this instruction for deletion.
3982 Pass.DeadInsts.push_back(&II);
3983
3984 // Strip all inbounds GEPs and pointer casts to try to dig out any root
3985 // alloca that should be re-examined after rewriting this instruction.
3986 Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
3987 if (AllocaInst *AI =
3989 assert(AI != &OldAI && AI != &NewAI &&
3990 "Splittable transfers cannot reach the same alloca on both ends.");
3991 Pass.Worklist.insert(AI);
3992 }
3993
3994 Type *OtherPtrTy = OtherPtr->getType();
3995 unsigned OtherAS = OtherPtrTy->getPointerAddressSpace();
3996
3997 // Compute the relative offset for the other pointer within the transfer.
3998 unsigned OffsetWidth = DL.getIndexSizeInBits(OtherAS);
3999 APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
4000 Align OtherAlign =
4001 (IsDest ? II.getSourceAlign() : II.getDestAlign()).valueOrOne();
4002 OtherAlign =
4003 commonAlignment(OtherAlign, OtherOffset.zextOrTrunc(64).getZExtValue());
4004
4005 if (EmitMemCpy) {
4006 // Compute the other pointer, folding as much as possible to produce
4007 // a single, simple GEP in most cases.
4008 OtherPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy,
4009 OtherPtr->getName() + ".");
4010
4011 Value *OurPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
4012 Type *SizeTy = II.getLength()->getType();
4013 Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset);
4014
4015 Value *DestPtr, *SrcPtr;
4016 MaybeAlign DestAlign, SrcAlign;
4017 // Note: IsDest is true iff we're copying into the new alloca slice
4018 if (IsDest) {
4019 DestPtr = OurPtr;
4020 DestAlign = SliceAlign;
4021 SrcPtr = OtherPtr;
4022 SrcAlign = OtherAlign;
4023 } else {
4024 DestPtr = OtherPtr;
4025 DestAlign = OtherAlign;
4026 SrcPtr = OurPtr;
4027 SrcAlign = SliceAlign;
4028 }
4029 CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign,
4030 Size, II.isVolatile());
4031 if (AATags)
4032 New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
4033
4034 APInt Offset(DL.getIndexTypeSizeInBits(DestPtr->getType()), 0);
4035 if (IsDest) {
4036 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8,
4037 &II, New, DestPtr, nullptr, DL);
4038 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4040 DL, Offset, /*AllowNonInbounds*/ true))) {
4041 migrateDebugInfo(Base, IsSplit, Offset.getZExtValue() * 8,
4042 SliceSize * 8, &II, New, DestPtr, nullptr, DL);
4043 }
4044 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4045 return false;
4046 }
4047
4048 bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset &&
4049 NewEndOffset == NewAllocaEndOffset;
4050 uint64_t Size = NewEndOffset - NewBeginOffset;
4051 unsigned BeginIndex = VecTy ? getIndex(NewBeginOffset) : 0;
4052 unsigned EndIndex = VecTy ? getIndex(NewEndOffset) : 0;
4053 unsigned NumElements = EndIndex - BeginIndex;
4054 IntegerType *SubIntTy =
4055 IntTy ? Type::getIntNTy(IntTy->getContext(), Size * 8) : nullptr;
4056
4057 // Reset the other pointer type to match the register type we're going to
4058 // use, but using the address space of the original other pointer.
4059 Type *OtherTy;
4060 if (VecTy && !IsWholeAlloca) {
4061 if (NumElements == 1)
4062 OtherTy = VecTy->getElementType();
4063 else
4064 OtherTy = FixedVectorType::get(VecTy->getElementType(), NumElements);
4065 } else if (IntTy && !IsWholeAlloca) {
4066 OtherTy = SubIntTy;
4067 } else {
4068 OtherTy = NewAllocaTy;
4069 }
4070
4071 Value *AdjPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy,
4072 OtherPtr->getName() + ".");
4073 MaybeAlign SrcAlign = OtherAlign;
4074 MaybeAlign DstAlign = SliceAlign;
4075 if (!IsDest)
4076 std::swap(SrcAlign, DstAlign);
4077
4078 Value *SrcPtr;
4079 Value *DstPtr;
4080
4081 if (IsDest) {
4082 DstPtr = getPtrToNewAI(II.getDestAddressSpace(), II.isVolatile());
4083 SrcPtr = AdjPtr;
4084 } else {
4085 DstPtr = AdjPtr;
4086 SrcPtr = getPtrToNewAI(II.getSourceAddressSpace(), II.isVolatile());
4087 }
4088
4089 Value *Src;
4090 if (VecTy && !IsWholeAlloca && !IsDest) {
4091 Src =
4092 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
4093 Src = extractVector(IRB, Src, BeginIndex, EndIndex, "vec");
4094 } else if (IntTy && !IsWholeAlloca && !IsDest) {
4095 Src =
4096 IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(), "load");
4097 Src = IRB.CreateBitPreservingCastChain(DL, Src, IntTy);
4098 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4099 Src = extractInteger(DL, IRB, Src, SubIntTy, Offset, "extract");
4100 } else {
4101 LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign,
4102 II.isVolatile(), "copyload");
4103 Load->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
4104 LLVMContext::MD_access_group});
4105 if (AATags)
4106 Load->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
4107 Load->getType(), DL));
4108 Src = Load;
4109 }
4110
4111 if (VecTy && !IsWholeAlloca && IsDest) {
4112 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(),
4113 "oldload");
4114 Src = insertVector(IRB, Old, Src, BeginIndex, "vec");
4115 } else if (IntTy && !IsWholeAlloca && IsDest) {
4116 Value *Old = IRB.CreateAlignedLoad(NewAllocaTy, &NewAI, NewAI.getAlign(),
4117 "oldload");
4118 Old = IRB.CreateBitPreservingCastChain(DL, Old, IntTy);
4119 uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
4120 Src = insertInteger(DL, IRB, Old, Src, Offset, "insert");
4121 Src = IRB.CreateBitPreservingCastChain(DL, Src, NewAllocaTy);
4122 }
4123
4124 StoreInst *Store = cast<StoreInst>(
4125 IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile()));
4126 Store->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
4127 LLVMContext::MD_access_group});
4128 if (AATags)
4129 Store->setAAMetadata(AATags.adjustForAccess(NewBeginOffset - BeginOffset,
4130 Src->getType(), DL));
4131
4132 APInt Offset(DL.getIndexTypeSizeInBits(DstPtr->getType()), 0);
4133 if (IsDest) {
4134
4135 migrateDebugInfo(&OldAI, IsSplit, NewBeginOffset * 8, SliceSize * 8, &II,
4136 Store, DstPtr, Src, DL);
4137 } else if (AllocaInst *Base = dyn_cast<AllocaInst>(
4139 DL, Offset, /*AllowNonInbounds*/ true))) {
4140 migrateDebugInfo(Base, IsSplit, Offset.getZExtValue() * 8, SliceSize * 8,
4141 &II, Store, DstPtr, Src, DL);
4142 }
4143
4144 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4145 return !II.isVolatile();
4146 }
4147
4148 bool visitIntrinsicInst(IntrinsicInst &II) {
4149 assert((II.isLifetimeStartOrEnd() || II.isDroppable()) &&
4150 "Unexpected intrinsic!");
4151 LLVM_DEBUG(dbgs() << " original: " << II << "\n");
4152
4153 // Record this instruction for deletion.
4154 Pass.DeadInsts.push_back(&II);
4155
4156 if (II.isDroppable()) {
4157 assert(II.getIntrinsicID() == Intrinsic::assume && "Expected assume");
4158 // TODO For now we forget assumed information, this can be improved.
4159 OldPtr->dropDroppableUsesIn(II);
4160 return true;
4161 }
4162
4163 assert(II.getArgOperand(0) == OldPtr);
4164 Type *PointerTy = IRB.getPtrTy(OldPtr->getType()->getPointerAddressSpace());
4165 Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
4166 Value *New;
4167 if (II.getIntrinsicID() == Intrinsic::lifetime_start)
4168 New = IRB.CreateLifetimeStart(Ptr);
4169 else
4170 New = IRB.CreateLifetimeEnd(Ptr);
4171
4172 (void)New;
4173 LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
4174
4175 return true;
4176 }
4177
4178 void fixLoadStoreAlign(Instruction &Root) {
4179 // This algorithm implements the same visitor loop as
4180 // hasUnsafePHIOrSelectUse, and fixes the alignment of each load
4181 // or store found.
4182 SmallPtrSet<Instruction *, 4> Visited;
4183 SmallVector<Instruction *, 4> Uses;
4184 Visited.insert(&Root);
4185 Uses.push_back(&Root);
4186 do {
4187 Instruction *I = Uses.pop_back_val();
4188
4189 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
4190 LI->setAlignment(std::min(LI->getAlign(), getSliceAlign()));
4191 continue;
4192 }
4193 if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
4194 SI->setAlignment(std::min(SI->getAlign(), getSliceAlign()));
4195 continue;
4196 }
4197
4201 for (User *U : I->users())
4202 if (Visited.insert(cast<Instruction>(U)).second)
4203 Uses.push_back(cast<Instruction>(U));
4204 } while (!Uses.empty());
4205 }
4206
4207 bool visitPHINode(PHINode &PN) {
4208 LLVM_DEBUG(dbgs() << " original: " << PN << "\n");
4209 assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable");
4210 assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable");
4211
4212 // We would like to compute a new pointer in only one place, but have it be
4213 // as local as possible to the PHI. To do that, we re-use the location of
4214 // the old pointer, which necessarily must be in the right position to
4215 // dominate the PHI.
4216 IRBuilderBase::InsertPointGuard Guard(IRB);
4217 if (isa<PHINode>(OldPtr))
4218 IRB.SetInsertPoint(OldPtr->getParent(),
4219 OldPtr->getParent()->getFirstInsertionPt());
4220 else
4221 IRB.SetInsertPoint(OldPtr);
4222 IRB.SetCurrentDebugLocation(OldPtr->getDebugLoc());
4223
4224 Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
4225 // Replace the operands which were using the old pointer.
4226 std::replace(PN.op_begin(), PN.op_end(), cast<Value>(OldPtr), NewPtr);
4227
4228 LLVM_DEBUG(dbgs() << " to: " << PN << "\n");
4229 deleteIfTriviallyDead(OldPtr);
4230
4231 // Fix the alignment of any loads or stores using this PHI node.
4232 fixLoadStoreAlign(PN);
4233
4234 // PHIs can't be promoted on their own, but often can be speculated. We
4235 // check the speculation outside of the rewriter so that we see the
4236 // fully-rewritten alloca.
4237 PHIUsers.insert(&PN);
4238 return true;
4239 }
4240
4241 bool visitSelectInst(SelectInst &SI) {
4242 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4243 assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&
4244 "Pointer isn't an operand!");
4245 assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable");
4246 assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable");
4247
4248 Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
4249 // Replace the operands which were using the old pointer.
4250 if (SI.getOperand(1) == OldPtr)
4251 SI.setOperand(1, NewPtr);
4252 if (SI.getOperand(2) == OldPtr)
4253 SI.setOperand(2, NewPtr);
4254
4255 LLVM_DEBUG(dbgs() << " to: " << SI << "\n");
4256 deleteIfTriviallyDead(OldPtr);
4257
4258 // Fix the alignment of any loads or stores using this select.
4259 fixLoadStoreAlign(SI);
4260
4261 // Selects can't be promoted on their own, but often can be speculated. We
4262 // check the speculation outside of the rewriter so that we see the
4263 // fully-rewritten alloca.
4264 SelectUsers.insert(&SI);
4265 return true;
4266 }
4267};
4268
4269/// Visitor to rewrite aggregate loads and stores as scalar.
4270///
4271/// This pass aggressively rewrites all aggregate loads and stores on
4272/// a particular pointer (or any pointer derived from it which we can identify)
4273/// with scalar loads and stores.
4274class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
4275 // Befriend the base class so it can delegate to private visit methods.
4276 friend class InstVisitor<AggLoadStoreRewriter, bool>;
4277
4278 /// Queue of pointer uses to analyze and potentially rewrite.
4280
4281 /// Set to prevent us from cycling with phi nodes and loops.
4282 SmallPtrSet<User *, 8> Visited;
4283
4284 /// The current pointer use being rewritten. This is used to dig up the used
4285 /// value (as opposed to the user).
4286 Use *U = nullptr;
4287
4288 /// Used to calculate offsets, and hence alignment, of subobjects.
4289 const DataLayout &DL;
4290
4291 IRBuilderTy &IRB;
4292
4293public:
4294 AggLoadStoreRewriter(const DataLayout &DL, IRBuilderTy &IRB)
4295 : DL(DL), IRB(IRB) {}
4296
4297 /// Rewrite loads and stores through a pointer and all pointers derived from
4298 /// it.
4299 bool rewrite(Instruction &I) {
4300 LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n");
4301 enqueueUsers(I);
4302 bool Changed = false;
4303 while (!Queue.empty()) {
4304 U = Queue.pop_back_val();
4305 Changed |= visit(cast<Instruction>(U->getUser()));
4306 }
4307 return Changed;
4308 }
4309
4310private:
4311 /// Enqueue all the users of the given instruction for further processing.
4312 /// This uses a set to de-duplicate users.
4313 void enqueueUsers(Instruction &I) {
4314 for (Use &U : I.uses())
4315 if (Visited.insert(U.getUser()).second)
4316 Queue.push_back(&U);
4317 }
4318
4319 // Conservative default is to not rewrite anything.
4320 bool visitInstruction(Instruction &I) { return false; }
4321
4322 /// Generic recursive split emission class.
4323 template <typename Derived> class OpSplitter {
4324 protected:
4325 /// The builder used to form new instructions.
4326 IRBuilderTy &IRB;
4327
4328 /// The indices which to be used with insert- or extractvalue to select the
4329 /// appropriate value within the aggregate.
4330 SmallVector<unsigned, 4> Indices;
4331
4332 /// The indices to a GEP instruction which will move Ptr to the correct slot
4333 /// within the aggregate.
4334 SmallVector<Value *, 4> GEPIndices;
4335
4336 /// The base pointer of the original op, used as a base for GEPing the
4337 /// split operations.
4338 Value *Ptr;
4339
4340 /// The base pointee type being GEPed into.
4341 Type *BaseTy;
4342
4343 /// Known alignment of the base pointer.
4344 Align BaseAlign;
4345
4346 /// To calculate offset of each component so we can correctly deduce
4347 /// alignments.
4348 const DataLayout &DL;
4349
4350 /// Initialize the splitter with an insertion point, Ptr and start with a
4351 /// single zero GEP index.
4352 OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4353 Align BaseAlign, const DataLayout &DL, IRBuilderTy &IRB)
4354 : IRB(IRB), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr), BaseTy(BaseTy),
4355 BaseAlign(BaseAlign), DL(DL) {
4356 IRB.SetInsertPoint(InsertionPoint);
4357 }
4358
4359 public:
4360 /// Generic recursive split emission routine.
4361 ///
4362 /// This method recursively splits an aggregate op (load or store) into
4363 /// scalar or vector ops. It splits recursively until it hits a single value
4364 /// and emits that single value operation via the template argument.
4365 ///
4366 /// The logic of this routine relies on GEPs and insertvalue and
4367 /// extractvalue all operating with the same fundamental index list, merely
4368 /// formatted differently (GEPs need actual values).
4369 ///
4370 /// \param Ty The type being split recursively into smaller ops.
4371 /// \param Agg The aggregate value being built up or stored, depending on
4372 /// whether this is splitting a load or a store respectively.
4373 void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) {
4374 if (Ty->isSingleValueType()) {
4375 unsigned Offset = DL.getIndexedOffsetInType(BaseTy, GEPIndices);
4376 return static_cast<Derived *>(this)->emitFunc(
4377 Ty, Agg, commonAlignment(BaseAlign, Offset), Name);
4378 }
4379
4380 if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
4381 unsigned OldSize = Indices.size();
4382 (void)OldSize;
4383 for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size;
4384 ++Idx) {
4385 assert(Indices.size() == OldSize && "Did not return to the old size");
4386 Indices.push_back(Idx);
4387 GEPIndices.push_back(IRB.getInt32(Idx));
4388 emitSplitOps(ATy->getElementType(), Agg, Name + "." + Twine(Idx));
4389 GEPIndices.pop_back();
4390 Indices.pop_back();
4391 }
4392 return;
4393 }
4394
4395 if (StructType *STy = dyn_cast<StructType>(Ty)) {
4396 unsigned OldSize = Indices.size();
4397 (void)OldSize;
4398 for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size;
4399 ++Idx) {
4400 assert(Indices.size() == OldSize && "Did not return to the old size");
4401 Indices.push_back(Idx);
4402 GEPIndices.push_back(IRB.getInt32(Idx));
4403 emitSplitOps(STy->getElementType(Idx), Agg, Name + "." + Twine(Idx));
4404 GEPIndices.pop_back();
4405 Indices.pop_back();
4406 }
4407 return;
4408 }
4409
4410 llvm_unreachable("Only arrays and structs are aggregate loadable types");
4411 }
4412 };
4413
4414 struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
4415 AAMDNodes AATags;
4416 // A vector to hold the split components that we want to emit
4417 // separate fake uses for.
4418 SmallVector<Value *, 4> Components;
4419 // A vector to hold all the fake uses of the struct that we are splitting.
4420 // Usually there should only be one, but we are handling the general case.
4422
4423 LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4424 AAMDNodes AATags, Align BaseAlign, const DataLayout &DL,
4425 IRBuilderTy &IRB)
4426 : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, DL,
4427 IRB),
4428 AATags(AATags) {}
4429
4430 /// Emit a leaf load of a single value. This is called at the leaves of the
4431 /// recursive emission to actually load values.
4432 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4434 // Load the single value and insert it using the indices.
4435 Value *GEP =
4436 IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
4437 LoadInst *Load =
4438 IRB.CreateAlignedLoad(Ty, GEP, Alignment, Name + ".load");
4439
4440 APInt Offset(
4441 DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
4442 if (AATags &&
4443 GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset))
4444 Load->setAAMetadata(
4445 AATags.adjustForAccess(Offset.getZExtValue(), Load->getType(), DL));
4446 // Record the load so we can generate a fake use for this aggregate
4447 // component.
4448 Components.push_back(Load);
4449
4450 Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert");
4451 LLVM_DEBUG(dbgs() << " to: " << *Load << "\n");
4452 }
4453
4454 // Stash the fake uses that use the value generated by this instruction.
4455 void recordFakeUses(LoadInst &LI) {
4456 for (Use &U : LI.uses())
4457 if (auto *II = dyn_cast<IntrinsicInst>(U.getUser()))
4458 if (II->getIntrinsicID() == Intrinsic::fake_use)
4459 FakeUses.push_back(II);
4460 }
4461
4462 // Replace all fake uses of the aggregate with a series of fake uses, one
4463 // for each split component.
4464 void emitFakeUses() {
4465 for (Instruction *I : FakeUses) {
4466 IRB.SetInsertPoint(I);
4467 for (auto *V : Components)
4468 IRB.CreateIntrinsic(Intrinsic::fake_use, {V});
4469 I->eraseFromParent();
4470 }
4471 }
4472 };
4473
4474 bool visitLoadInst(LoadInst &LI) {
4475 assert(LI.getPointerOperand() == *U);
4476 if (!LI.isSimple() || LI.getType()->isSingleValueType())
4477 return false;
4478
4479 // We have an aggregate being loaded, split it apart.
4480 LLVM_DEBUG(dbgs() << " original: " << LI << "\n");
4481 LoadOpSplitter Splitter(&LI, *U, LI.getType(), LI.getAAMetadata(),
4482 getAdjustedAlignment(&LI, 0), DL, IRB);
4483 Splitter.recordFakeUses(LI);
4485 Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca");
4486 Splitter.emitFakeUses();
4487 Visited.erase(&LI);
4488 LI.replaceAllUsesWith(V);
4489 LI.eraseFromParent();
4490 return true;
4491 }
4492
4493 struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
4494 StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
4495 AAMDNodes AATags, StoreInst *AggStore, Align BaseAlign,
4496 const DataLayout &DL, IRBuilderTy &IRB)
4497 : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
4498 DL, IRB),
4499 AATags(AATags), AggStore(AggStore) {}
4500 AAMDNodes AATags;
4501 StoreInst *AggStore;
4502 /// Emit a leaf store of a single value. This is called at the leaves of the
4503 /// recursive emission to actually produce stores.
4504 void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
4506 // Extract the single value and store it using the indices.
4507 //
4508 // The gep and extractvalue values are factored out of the CreateStore
4509 // call to make the output independent of the argument evaluation order.
4510 Value *ExtractValue =
4511 IRB.CreateExtractValue(Agg, Indices, Name + ".extract");
4512 Value *InBoundsGEP =
4513 IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
4514 StoreInst *Store =
4515 IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Alignment);
4516
4517 APInt Offset(
4518 DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
4519 GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset);
4520 if (AATags) {
4521 Store->setAAMetadata(AATags.adjustForAccess(
4522 Offset.getZExtValue(), ExtractValue->getType(), DL));
4523 }
4524
4525 // migrateDebugInfo requires the base Alloca. Walk to it from this gep.
4526 // If we cannot (because there's an intervening non-const or unbounded
4527 // gep) then we wouldn't expect to see dbg.assign intrinsics linked to
4528 // this instruction.
4530 if (auto *OldAI = dyn_cast<AllocaInst>(Base)) {
4531 uint64_t SizeInBits =
4532 DL.getTypeSizeInBits(Store->getValueOperand()->getType());
4533 migrateDebugInfo(OldAI, /*IsSplit*/ true, Offset.getZExtValue() * 8,
4534 SizeInBits, AggStore, Store,
4535 Store->getPointerOperand(), Store->getValueOperand(),
4536 DL);
4537 } else {
4539 "AT: unexpected debug.assign linked to store through "
4540 "unbounded GEP");
4541 }
4542 LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
4543 }
4544 };
4545
4546 bool visitStoreInst(StoreInst &SI) {
4547 if (!SI.isSimple() || SI.getPointerOperand() != *U)
4548 return false;
4549 Value *V = SI.getValueOperand();
4550 if (V->getType()->isSingleValueType())
4551 return false;
4552
4553 // We have an aggregate being stored, split it apart.
4554 LLVM_DEBUG(dbgs() << " original: " << SI << "\n");
4555 StoreOpSplitter Splitter(&SI, *U, V->getType(), SI.getAAMetadata(), &SI,
4556 getAdjustedAlignment(&SI, 0), DL, IRB);
4557 Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca");
4558 Visited.erase(&SI);
4559 // The stores replacing SI each have markers describing fragments of the
4560 // assignment so delete the assignment markers linked to SI.
4562 SI.eraseFromParent();
4563 return true;
4564 }
4565
4566 bool visitBitCastInst(BitCastInst &BC) {
4567 enqueueUsers(BC);
4568 return false;
4569 }
4570
4571 bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) {
4572 enqueueUsers(ASC);
4573 return false;
4574 }
4575
4576 // Unfold gep (select cond, ptr1, ptr2), idx
4577 // => select cond, gep(ptr1, idx), gep(ptr2, idx)
4578 // and gep ptr, (select cond, idx1, idx2)
4579 // => select cond, gep(ptr, idx1), gep(ptr, idx2)
4580 // We also allow for i1 zext indices, which are equivalent to selects.
4581 bool unfoldGEPSelect(GetElementPtrInst &GEPI) {
4582 // Check whether the GEP has exactly one select operand and all indices
4583 // will become constant after the transform.
4585 for (Value *Op : GEPI.indices()) {
4586 if (auto *SI = dyn_cast<SelectInst>(Op)) {
4587 if (Sel)
4588 return false;
4589
4590 Sel = SI;
4591 if (!isa<ConstantInt>(SI->getTrueValue()) ||
4592 !isa<ConstantInt>(SI->getFalseValue()))
4593 return false;
4594 continue;
4595 }
4596 if (auto *ZI = dyn_cast<ZExtInst>(Op)) {
4597 if (Sel)
4598 return false;
4599 Sel = ZI;
4600 if (!ZI->getSrcTy()->isIntegerTy(1))
4601 return false;
4602 continue;
4603 }
4604
4605 if (!isa<ConstantInt>(Op))
4606 return false;
4607 }
4608
4609 if (!Sel)
4610 return false;
4611
4612 LLVM_DEBUG(dbgs() << " Rewriting gep(select) -> select(gep):\n";
4613 dbgs() << " original: " << *Sel << "\n";
4614 dbgs() << " " << GEPI << "\n";);
4615
4616 auto GetNewOps = [&](Value *SelOp) {
4617 SmallVector<Value *> NewOps;
4618 for (Value *Op : GEPI.operands())
4619 if (Op == Sel)
4620 NewOps.push_back(SelOp);
4621 else
4622 NewOps.push_back(Op);
4623 return NewOps;
4624 };
4625
4626 Value *Cond, *True, *False;
4627 Instruction *MDFrom = nullptr;
4628 if (auto *SI = dyn_cast<SelectInst>(Sel)) {
4629 Cond = SI->getCondition();
4630 True = SI->getTrueValue();
4631 False = SI->getFalseValue();
4632 MDFrom = SI;
4633 } else {
4634 Cond = Sel->getOperand(0);
4635 True = ConstantInt::get(Sel->getType(), 1);
4636 False = ConstantInt::get(Sel->getType(), 0);
4637 }
4638 SmallVector<Value *> TrueOps = GetNewOps(True);
4639 SmallVector<Value *> FalseOps = GetNewOps(False);
4640
4641 IRB.SetInsertPoint(&GEPI);
4642 GEPNoWrapFlags NW = GEPI.getNoWrapFlags();
4643
4644 Type *Ty = GEPI.getSourceElementType();
4645 Value *NTrue = IRB.CreateGEP(Ty, TrueOps[0], ArrayRef(TrueOps).drop_front(),
4646 True->getName() + ".sroa.gep", NW);
4647
4648 Value *NFalse =
4649 IRB.CreateGEP(Ty, FalseOps[0], ArrayRef(FalseOps).drop_front(),
4650 False->getName() + ".sroa.gep", NW);
4651
4652 Value *NSel = MDFrom
4653 ? IRB.CreateSelect(Cond, NTrue, NFalse,
4654 Sel->getName() + ".sroa.sel", MDFrom)
4655 : IRB.CreateSelectWithUnknownProfile(
4656 Cond, NTrue, NFalse, DEBUG_TYPE,
4657 Sel->getName() + ".sroa.sel");
4658 Visited.erase(&GEPI);
4659 GEPI.replaceAllUsesWith(NSel);
4660 GEPI.eraseFromParent();
4661 Instruction *NSelI = cast<Instruction>(NSel);
4662 Visited.insert(NSelI);
4663 enqueueUsers(*NSelI);
4664
4665 LLVM_DEBUG(dbgs() << " to: " << *NTrue << "\n";
4666 dbgs() << " " << *NFalse << "\n";
4667 dbgs() << " " << *NSel << "\n";);
4668
4669 return true;
4670 }
4671
4672 // Unfold gep (phi ptr1, ptr2), idx
4673 // => phi ((gep ptr1, idx), (gep ptr2, idx))
4674 // and gep ptr, (phi idx1, idx2)
4675 // => phi ((gep ptr, idx1), (gep ptr, idx2))
4676 bool unfoldGEPPhi(GetElementPtrInst &GEPI) {
4677 // To prevent infinitely expanding recursive phis, bail if the GEP pointer
4678 // operand (looking through the phi if it is the phi we want to unfold) is
4679 // an instruction besides a static alloca.
4680 PHINode *Phi = dyn_cast<PHINode>(GEPI.getPointerOperand());
4681 auto IsInvalidPointerOperand = [](Value *V) {
4682 if (!isa<Instruction>(V))
4683 return false;
4684 if (auto *AI = dyn_cast<AllocaInst>(V))
4685 return !AI->isStaticAlloca();
4686 return true;
4687 };
4688 if (Phi) {
4689 if (any_of(Phi->operands(), IsInvalidPointerOperand))
4690 return false;
4691 } else {
4692 if (IsInvalidPointerOperand(GEPI.getPointerOperand()))
4693 return false;
4694 }
4695 // Check whether the GEP has exactly one phi operand (including the pointer
4696 // operand) and all indices will become constant after the transform.
4697 for (Value *Op : GEPI.indices()) {
4698 if (auto *SI = dyn_cast<PHINode>(Op)) {
4699 if (Phi)
4700 return false;
4701
4702 Phi = SI;
4703 if (!all_of(Phi->incoming_values(),
4704 [](Value *V) { return isa<ConstantInt>(V); }))
4705 return false;
4706 continue;
4707 }
4708
4709 if (!isa<ConstantInt>(Op))
4710 return false;
4711 }
4712
4713 if (!Phi)
4714 return false;
4715
4716 LLVM_DEBUG(dbgs() << " Rewriting gep(phi) -> phi(gep):\n";
4717 dbgs() << " original: " << *Phi << "\n";
4718 dbgs() << " " << GEPI << "\n";);
4719
4720 auto GetNewOps = [&](Value *PhiOp) {
4721 SmallVector<Value *> NewOps;
4722 for (Value *Op : GEPI.operands())
4723 if (Op == Phi)
4724 NewOps.push_back(PhiOp);
4725 else
4726 NewOps.push_back(Op);
4727 return NewOps;
4728 };
4729
4730 IRB.SetInsertPoint(Phi);
4731 PHINode *NewPhi = IRB.CreatePHI(GEPI.getType(), Phi->getNumIncomingValues(),
4732 Phi->getName() + ".sroa.phi");
4733
4734 Type *SourceTy = GEPI.getSourceElementType();
4735 // We only handle arguments, constants, and static allocas here, so we can
4736 // insert GEPs at the end of the entry block.
4737 IRB.SetInsertPoint(GEPI.getFunction()->getEntryBlock().getTerminator());
4738 for (unsigned I = 0, E = Phi->getNumIncomingValues(); I != E; ++I) {
4739 Value *Op = Phi->getIncomingValue(I);
4740 BasicBlock *BB = Phi->getIncomingBlock(I);
4741 Value *NewGEP;
4742 if (int NI = NewPhi->getBasicBlockIndex(BB); NI >= 0) {
4743 NewGEP = NewPhi->getIncomingValue(NI);
4744 } else {
4745 SmallVector<Value *> NewOps = GetNewOps(Op);
4746 NewGEP =
4747 IRB.CreateGEP(SourceTy, NewOps[0], ArrayRef(NewOps).drop_front(),
4748 Phi->getName() + ".sroa.gep", GEPI.getNoWrapFlags());
4749 }
4750 NewPhi->addIncoming(NewGEP, BB);
4751 }
4752
4753 Visited.erase(&GEPI);
4754 GEPI.replaceAllUsesWith(NewPhi);
4755 GEPI.eraseFromParent();
4756 Visited.insert(NewPhi);
4757 enqueueUsers(*NewPhi);
4758
4759 LLVM_DEBUG(dbgs() << " to: ";
4760 for (Value *In
4761 : NewPhi->incoming_values()) dbgs()
4762 << "\n " << *In;
4763 dbgs() << "\n " << *NewPhi << '\n');
4764
4765 return true;
4766 }
4767
4768 bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
4769 if (unfoldGEPSelect(GEPI))
4770 return true;
4771
4772 if (unfoldGEPPhi(GEPI))
4773 return true;
4774
4775 enqueueUsers(GEPI);
4776 return false;
4777 }
4778
4779 bool visitPHINode(PHINode &PN) {
4780 enqueueUsers(PN);
4781 return false;
4782 }
4783
4784 bool visitSelectInst(SelectInst &SI) {
4785 enqueueUsers(SI);
4786 return false;
4787 }
4788};
4789
4790} // end anonymous namespace
4791
4792/// Strip aggregate type wrapping.
4793///
4794/// This removes no-op aggregate types wrapping an underlying type. It will
4795/// strip as many layers of types as it can without changing either the type
4796/// size or the allocated size.
4798 if (Ty->isSingleValueType())
4799 return Ty;
4800
4801 uint64_t AllocSize = DL.getTypeAllocSize(Ty).getFixedValue();
4802 uint64_t TypeSize = DL.getTypeSizeInBits(Ty).getFixedValue();
4803
4804 Type *InnerTy;
4805 if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) {
4806 InnerTy = ArrTy->getElementType();
4807 } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
4808 const StructLayout *SL = DL.getStructLayout(STy);
4809 unsigned Index = SL->getElementContainingOffset(0);
4810 InnerTy = STy->getElementType(Index);
4811 } else {
4812 return Ty;
4813 }
4814
4815 if (AllocSize > DL.getTypeAllocSize(InnerTy).getFixedValue() ||
4816 TypeSize > DL.getTypeSizeInBits(InnerTy).getFixedValue())
4817 return Ty;
4818
4819 return stripAggregateTypeWrapping(DL, InnerTy);
4820}
4821
4822/// Try to find a partition of the aggregate type passed in for a given
4823/// offset and size.
4824///
4825/// This recurses through the aggregate type and tries to compute a subtype
4826/// based on the offset and size. When the offset and size span a sub-section
4827/// of an array, it will even compute a new array type for that sub-section,
4828/// and the same for structs.
4829///
4830/// Note that this routine is very strict and tries to find a partition of the
4831/// type which produces the *exact* right offset and size. It is not forgiving
4832/// when the size or offset cause either end of type-based partition to be off.
4833/// Also, this is a best-effort routine. It is reasonable to give up and not
4834/// return a type if necessary.
4836 uint64_t Size) {
4837 if (Offset == 0 && DL.getTypeAllocSize(Ty).getFixedValue() == Size)
4838 return stripAggregateTypeWrapping(DL, Ty);
4839 if (Offset > DL.getTypeAllocSize(Ty).getFixedValue() ||
4840 (DL.getTypeAllocSize(Ty).getFixedValue() - Offset) < Size)
4841 return nullptr;
4842
4843 if (isa<ArrayType>(Ty) || isa<VectorType>(Ty)) {
4844 Type *ElementTy;
4845 uint64_t TyNumElements;
4846 if (auto *AT = dyn_cast<ArrayType>(Ty)) {
4847 ElementTy = AT->getElementType();
4848 TyNumElements = AT->getNumElements();
4849 } else {
4850 // FIXME: This isn't right for vectors with non-byte-sized or
4851 // non-power-of-two sized elements.
4852 auto *VT = cast<FixedVectorType>(Ty);
4853 ElementTy = VT->getElementType();
4854 TyNumElements = VT->getNumElements();
4855 }
4856 uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedValue();
4857 uint64_t NumSkippedElements = Offset / ElementSize;
4858 if (NumSkippedElements >= TyNumElements)
4859 return nullptr;
4860 Offset -= NumSkippedElements * ElementSize;
4861
4862 // First check if we need to recurse.
4863 if (Offset > 0 || Size < ElementSize) {
4864 // Bail if the partition ends in a different array element.
4865 if ((Offset + Size) > ElementSize)
4866 return nullptr;
4867 // Recurse through the element type trying to peel off offset bytes.
4868 return getTypePartition(DL, ElementTy, Offset, Size);
4869 }
4870 assert(Offset == 0);
4871
4872 if (Size == ElementSize)
4873 return stripAggregateTypeWrapping(DL, ElementTy);
4874 assert(Size > ElementSize);
4875 uint64_t NumElements = Size / ElementSize;
4876 if (NumElements * ElementSize != Size)
4877 return nullptr;
4878 return ArrayType::get(ElementTy, NumElements);
4879 }
4880
4882 if (!STy)
4883 return nullptr;
4884
4885 const StructLayout *SL = DL.getStructLayout(STy);
4886
4887 if (SL->getSizeInBits().isScalable())
4888 return nullptr;
4889
4890 if (Offset >= SL->getSizeInBytes())
4891 return nullptr;
4892 uint64_t EndOffset = Offset + Size;
4893 if (EndOffset > SL->getSizeInBytes())
4894 return nullptr;
4895
4896 unsigned Index = SL->getElementContainingOffset(Offset);
4897 Offset -= SL->getElementOffset(Index);
4898
4899 Type *ElementTy = STy->getElementType(Index);
4900 uint64_t ElementSize = DL.getTypeAllocSize(ElementTy).getFixedValue();
4901 if (Offset >= ElementSize)
4902 return nullptr; // The offset points into alignment padding.
4903
4904 // See if any partition must be contained by the element.
4905 if (Offset > 0 || Size < ElementSize) {
4906 if ((Offset + Size) > ElementSize)
4907 return nullptr;
4908 return getTypePartition(DL, ElementTy, Offset, Size);
4909 }
4910 assert(Offset == 0);
4911
4912 if (Size == ElementSize)
4913 return stripAggregateTypeWrapping(DL, ElementTy);
4914
4915 StructType::element_iterator EI = STy->element_begin() + Index,
4916 EE = STy->element_end();
4917 if (EndOffset < SL->getSizeInBytes()) {
4918 unsigned EndIndex = SL->getElementContainingOffset(EndOffset);
4919 if (Index == EndIndex)
4920 return nullptr; // Within a single element and its padding.
4921
4922 // Don't try to form "natural" types if the elements don't line up with the
4923 // expected size.
4924 // FIXME: We could potentially recurse down through the last element in the
4925 // sub-struct to find a natural end point.
4926 if (SL->getElementOffset(EndIndex) != EndOffset)
4927 return nullptr;
4928
4929 assert(Index < EndIndex);
4930 EE = STy->element_begin() + EndIndex;
4931 }
4932
4933 // Try to build up a sub-structure.
4934 StructType *SubTy =
4935 StructType::get(STy->getContext(), ArrayRef(EI, EE), STy->isPacked());
4936 const StructLayout *SubSL = DL.getStructLayout(SubTy);
4937 if (Size != SubSL->getSizeInBytes())
4938 return nullptr; // The sub-struct doesn't have quite the size needed.
4939
4940 return SubTy;
4941}
4942
4943/// Try to find a slice in the map that partially overlaps with S, i.e. some but
4944/// not all of its range is contained within the range of S, and which forms a
4945/// load/store pair with S (i.e. the two slices are used to perform a
4946/// memmove-like copy). SliceMap is expected to contain the slices for a single
4947/// Partition.
4948static Slice *findOverlappingCopySlice(Slice &S,
4949 InstructionSliceMap &SliceMap) {
4950 // Single byte slices can't partially overlap anything
4951 if (S.endOffset() - S.beginOffset() == 1)
4952 return nullptr;
4953 // The source/destination of this slice needs to be a memory instruction
4954 // whose slice overlaps this one.
4955 Instruction *I = cast<Instruction>(S.getUse()->getUser());
4956 Instruction *J = nullptr;
4957 if (auto *LI = dyn_cast<LoadInst>(I)) {
4958 if (!LI->hasOneUser())
4959 return nullptr;
4960 J = cast<Instruction>(*LI->user_begin());
4961 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
4962 J = dyn_cast<Instruction>(SI->getValueOperand());
4963 if (!J || !J->hasOneUser())
4964 return nullptr;
4965 } else {
4966 return nullptr;
4967 }
4968 // Check if there's a slice that corresponds to J that overlaps this slice
4969 for (Slice *JS : SliceMap[J]) {
4970 if (S.beginOffset() > JS->beginOffset() &&
4971 S.beginOffset() < JS->endOffset())
4972 return JS;
4973 if (JS->beginOffset() > S.beginOffset() &&
4974 JS->beginOffset() < S.endOffset())
4975 return JS;
4976 }
4977 return nullptr;
4978}
4979
4980/// Pre-split loads and stores to simplify rewriting.
4981///
4982/// We want to break up the splittable load+store pairs as much as
4983/// possible. This is important to do as a preprocessing step, as once we
4984/// start rewriting the accesses to partitions of the alloca we lose the
4985/// necessary information to correctly split apart paired loads and stores
4986/// which both point into this alloca. The case to consider is something like
4987/// the following:
4988///
4989/// %a = alloca [12 x i8]
4990/// %gep1 = getelementptr i8, ptr %a, i32 0
4991/// %gep2 = getelementptr i8, ptr %a, i32 4
4992/// %gep3 = getelementptr i8, ptr %a, i32 8
4993/// store float 0.0, ptr %gep1
4994/// store float 1.0, ptr %gep2
4995/// %v = load i64, ptr %gep1
4996/// store i64 %v, ptr %gep2
4997/// %f1 = load float, ptr %gep2
4998/// %f2 = load float, ptr %gep3
4999///
5000/// Here we want to form 3 partitions of the alloca, each 4 bytes large, and
5001/// promote everything so we recover the 2 SSA values that should have been
5002/// there all along.
5003///
5004/// \returns true if any changes are made.
5005bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
5006 LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n");
5007
5008 // Track the loads and stores which are candidates for pre-splitting here, in
5009 // the order they first appear during the partition scan. These give stable
5010 // iteration order and a basis for tracking which loads and stores we
5011 // actually split.
5014
5015 // We need to accumulate the splits required of each load or store where we
5016 // can find them via a direct lookup. This is important to cross-check loads
5017 // and stores against each other. We also track the slice so that we can kill
5018 // all the slices that end up split.
5019 struct SplitOffsets {
5020 Slice *S;
5021 std::vector<uint64_t> Splits;
5022 };
5023 SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap;
5024
5025 // Track loads out of this alloca which cannot, for any reason, be pre-split.
5026 // This is important as we also cannot pre-split stores of those loads!
5027 // FIXME: This is all pretty gross. It means that we can be more aggressive
5028 // in pre-splitting when the load feeding the store happens to come from
5029 // a separate alloca. Put another way, the effectiveness of SROA would be
5030 // decreased by a frontend which just concatenated all of its local allocas
5031 // into one big flat alloca. But defeating such patterns is exactly the job
5032 // SROA is tasked with! Sadly, to not have this discrepancy we would have
5033 // change store pre-splitting to actually force pre-splitting of the load
5034 // that feeds it *and all stores*. That makes pre-splitting much harder, but
5035 // maybe it would make it more principled?
5036 SmallPtrSet<LoadInst *, 8> UnsplittableLoads;
5037
5038 LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n");
5039 for (auto &P : AS.partitions()) {
5040 InstructionSliceMap SliceMap;
5041 for (Slice &S : P) {
5042 SliceMap[cast<Instruction>(S.getUse()->getUser())].insert(&S);
5043 }
5044 for (Slice &S : P) {
5045 Instruction *I = cast<Instruction>(S.getUse()->getUser());
5046 bool ExtendsPastPartitionEnd = S.endOffset() > P.endOffset();
5047 Slice *CopyOverlap = findOverlappingCopySlice(S, SliceMap);
5048 if (!S.isSplittable() || (!ExtendsPastPartitionEnd && !CopyOverlap)) {
5049 // If this is a load we have to track that it can't participate in any
5050 // pre-splitting. If this is a store of a load we have to track that
5051 // that load also can't participate in any pre-splitting.
5052 if (auto *LI = dyn_cast<LoadInst>(I))
5053 UnsplittableLoads.insert(LI);
5054 else if (auto *SI = dyn_cast<StoreInst>(I))
5055 if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand()))
5056 UnsplittableLoads.insert(LI);
5057 continue;
5058 }
5059 assert(P.endOffset() > S.beginOffset() &&
5060 "Empty or backwards partition!");
5061
5062 // Determine if this is a pre-splittable slice.
5063 if (auto *LI = dyn_cast<LoadInst>(I)) {
5064 assert(!LI->isVolatile() && "Cannot split volatile loads!");
5065
5066 // The load must be used exclusively to store into other pointers for
5067 // us to be able to arbitrarily pre-split it. The stores must also be
5068 // simple to avoid changing semantics.
5069 auto IsLoadSimplyStored = [](LoadInst *LI) {
5070 for (User *LU : LI->users()) {
5071 auto *SI = dyn_cast<StoreInst>(LU);
5072 if (!SI || !SI->isSimple())
5073 return false;
5074 }
5075 return true;
5076 };
5077 if (!IsLoadSimplyStored(LI)) {
5078 UnsplittableLoads.insert(LI);
5079 continue;
5080 }
5081
5082 Loads.push_back(LI);
5083 } else if (auto *SI = dyn_cast<StoreInst>(I)) {
5084 if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex()))
5085 // Skip stores *of* pointers. FIXME: This shouldn't even be possible!
5086 continue;
5087 auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand());
5088 if (!StoredLoad || !StoredLoad->isSimple())
5089 continue;
5090 assert(!SI->isVolatile() && "Cannot split volatile stores!");
5091
5092 Stores.push_back(SI);
5093 } else {
5094 // Other uses cannot be pre-split.
5095 continue;
5096 }
5097
5098 // Record the initial split.
5099 LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n");
5100 auto &Offsets = SplitOffsetsMap[I];
5101 assert(Offsets.Splits.empty() &&
5102 "Should not have splits the first time we see an instruction!");
5103 Offsets.S = &S;
5104 if (CopyOverlap) {
5105 // S is being moved to CopyOverlap, which overlaps with S, so we split S
5106 // into three parts:
5107 // * Initially outside the overlap and copied into the overlap.
5108 // * Either initially inside the overlap and remains inside it after
5109 // the copy, or initially outside and remains outside, depending on
5110 // if the overlap or non-overlap area is larger, and which will be
5111 // empty if they are equal.
5112 // * Initially inside the overlap and copied outside the overlap.
5113 // This should result in the first and last parts being promoted to
5114 // scalars. If the middle part is an overlapping copy then we repeat
5115 // this process until it isn't.
5116 uint64_t OverlapStart =
5117 std::max(S.beginOffset(), CopyOverlap->beginOffset());
5118 uint64_t OverlapEnd = std::min(S.endOffset(), CopyOverlap->endOffset());
5119 uint64_t OverlapSize = OverlapEnd - OverlapStart;
5120 uint64_t SliceSize = S.endOffset() - S.beginOffset();
5121 uint64_t NonOverlapSize = SliceSize - OverlapSize;
5122 if (OverlapSize < NonOverlapSize) {
5123 // When the overlap area is smaller the middle part is initially and
5124 // remains outside the overlap, so splitting once is enough. The size
5125 // of the first and last part both equal the overlap size, so the
5126 // split points are that amount from the start and end of the slice.
5127 Offsets.Splits.push_back(OverlapSize);
5128 Offsets.Splits.push_back(SliceSize - OverlapSize);
5129 } else if (OverlapSize > NonOverlapSize) {
5130 // When the overlap area is larger the middle part is initially and
5131 // remains inside the overlap, so we repeatedly split it.
5132 for (uint64_t Split = NonOverlapSize; Split <= SliceSize / 2;
5133 Split += NonOverlapSize) {
5134 Offsets.Splits.push_back(Split);
5135 if (Split != SliceSize - Split) {
5136 Offsets.Splits.push_back(SliceSize - Split);
5137 }
5138 }
5139 // Sort the splits as they need to be in ascending order.
5140 llvm::sort(Offsets.Splits);
5141 } else {
5142 // Here the overlap and non-overlap size are the same, so the middle
5143 // part is empty and the slice is split exactly in the center.
5144 Offsets.Splits.push_back(OverlapSize);
5145 }
5146 } else {
5147 Offsets.Splits.push_back(P.endOffset() - S.beginOffset());
5148 }
5149 }
5150
5151 // Now scan the already split slices, and add a split for any of them which
5152 // we're going to pre-split.
5153 for (Slice *S : P.splitSliceTails()) {
5154 auto SplitOffsetsMapI =
5155 SplitOffsetsMap.find(cast<Instruction>(S->getUse()->getUser()));
5156 if (SplitOffsetsMapI == SplitOffsetsMap.end())
5157 continue;
5158 auto &Offsets = SplitOffsetsMapI->second;
5159
5160 assert(Offsets.S == S && "Found a mismatched slice!");
5161 assert(!Offsets.Splits.empty() &&
5162 "Cannot have an empty set of splits on the second partition!");
5163 assert(Offsets.Splits.back() ==
5164 P.beginOffset() - Offsets.S->beginOffset() &&
5165 "Previous split does not end where this one begins!");
5166
5167 // Record each split. The last partition's end isn't needed as the size
5168 // of the slice dictates that.
5169 if (S->endOffset() > P.endOffset())
5170 Offsets.Splits.push_back(P.endOffset() - Offsets.S->beginOffset());
5171 }
5172 }
5173
5174 // We may have split loads where some of their stores are split stores. For
5175 // such loads and stores, we can only pre-split them if their splits exactly
5176 // match relative to their starting offset. We have to verify this prior to
5177 // any rewriting.
5178 llvm::erase_if(Stores, [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
5179 // Lookup the load we are storing in our map of split
5180 // offsets.
5181 auto *LI = cast<LoadInst>(SI->getValueOperand());
5182 // If it was completely unsplittable, then we're done,
5183 // and this store can't be pre-split.
5184 if (UnsplittableLoads.count(LI))
5185 return true;
5186
5187 auto LoadOffsetsI = SplitOffsetsMap.find(LI);
5188 if (LoadOffsetsI == SplitOffsetsMap.end())
5189 return false; // Unrelated loads are definitely safe.
5190 auto &LoadOffsets = LoadOffsetsI->second;
5191
5192 // Now lookup the store's offsets.
5193 auto &StoreOffsets = SplitOffsetsMap[SI];
5194
5195 // If the relative offsets of each split in the load and
5196 // store match exactly, then we can split them and we
5197 // don't need to remove them here.
5198 if (LoadOffsets.Splits == StoreOffsets.Splits)
5199 return false;
5200
5201 LLVM_DEBUG(dbgs() << " Mismatched splits for load and store:\n"
5202 << " " << *LI << "\n"
5203 << " " << *SI << "\n");
5204
5205 // We've found a store and load that we need to split
5206 // with mismatched relative splits. Just give up on them
5207 // and remove both instructions from our list of
5208 // candidates.
5209 UnsplittableLoads.insert(LI);
5210 return true;
5211 });
5212 // Now we have to go *back* through all the stores, because a later store may
5213 // have caused an earlier store's load to become unsplittable and if it is
5214 // unsplittable for the later store, then we can't rely on it being split in
5215 // the earlier store either.
5216 llvm::erase_if(Stores, [&UnsplittableLoads](StoreInst *SI) {
5217 auto *LI = cast<LoadInst>(SI->getValueOperand());
5218 return UnsplittableLoads.count(LI);
5219 });
5220 // Once we've established all the loads that can't be split for some reason,
5221 // filter any that made it into our list out.
5222 llvm::erase_if(Loads, [&UnsplittableLoads](LoadInst *LI) {
5223 return UnsplittableLoads.count(LI);
5224 });
5225
5226 // If no loads or stores are left, there is no pre-splitting to be done for
5227 // this alloca.
5228 if (Loads.empty() && Stores.empty())
5229 return false;
5230
5231 // From here on, we can't fail and will be building new accesses, so rig up
5232 // an IR builder.
5233 IRBuilderTy IRB(&AI);
5234
5235 // Collect the new slices which we will merge into the alloca slices.
5236 SmallVector<Slice, 4> NewSlices;
5237
5238 // Track any allocas we end up splitting loads and stores for so we iterate
5239 // on them.
5240 SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas;
5241
5242 // At this point, we have collected all of the loads and stores we can
5243 // pre-split, and the specific splits needed for them. We actually do the
5244 // splitting in a specific order in order to handle when one of the loads in
5245 // the value operand to one of the stores.
5246 //
5247 // First, we rewrite all of the split loads, and just accumulate each split
5248 // load in a parallel structure. We also build the slices for them and append
5249 // them to the alloca slices.
5250 SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap;
5251 std::vector<LoadInst *> SplitLoads;
5252 const DataLayout &DL = AI.getDataLayout();
5253 for (LoadInst *LI : Loads) {
5254 SplitLoads.clear();
5255
5256 auto &Offsets = SplitOffsetsMap[LI];
5257 unsigned SliceSize = Offsets.S->endOffset() - Offsets.S->beginOffset();
5258 assert(LI->getType()->getIntegerBitWidth() % 8 == 0 &&
5259 "Load must have type size equal to store size");
5260 assert(LI->getType()->getIntegerBitWidth() / 8 >= SliceSize &&
5261 "Load must be >= slice size");
5262
5263 uint64_t BaseOffset = Offsets.S->beginOffset();
5264 assert(BaseOffset + SliceSize > BaseOffset &&
5265 "Cannot represent alloca access size using 64-bit integers!");
5266
5268 IRB.SetInsertPoint(LI);
5269
5270 LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n");
5271
5272 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5273 int Idx = 0, Size = Offsets.Splits.size();
5274 for (;;) {
5275 auto *PartTy = Type::getIntNTy(LI->getContext(), PartSize * 8);
5276 auto AS = LI->getPointerAddressSpace();
5277 auto *PartPtrTy = LI->getPointerOperandType();
5278 LoadInst *PLoad = IRB.CreateAlignedLoad(
5279 PartTy,
5280 getAdjustedPtr(IRB, DL, BasePtr,
5281 APInt(DL.getIndexSizeInBits(AS), PartOffset),
5282 PartPtrTy, BasePtr->getName() + "."),
5283 getAdjustedAlignment(LI, PartOffset),
5284 /*IsVolatile*/ false, LI->getName());
5285 PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
5286 LLVMContext::MD_access_group});
5287
5288 // Append this load onto the list of split loads so we can find it later
5289 // to rewrite the stores.
5290 SplitLoads.push_back(PLoad);
5291
5292 // Now build a new slice for the alloca.
5293 NewSlices.push_back(
5294 Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5295 &PLoad->getOperandUse(PLoad->getPointerOperandIndex()),
5296 /*IsSplittable*/ false));
5297 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5298 << ", " << NewSlices.back().endOffset()
5299 << "): " << *PLoad << "\n");
5300
5301 // See if we've handled all the splits.
5302 if (Idx >= Size)
5303 break;
5304
5305 // Setup the next partition.
5306 PartOffset = Offsets.Splits[Idx];
5307 ++Idx;
5308 PartSize = (Idx < Size ? Offsets.Splits[Idx] : SliceSize) - PartOffset;
5309 }
5310
5311 // Now that we have the split loads, do the slow walk over all uses of the
5312 // load and rewrite them as split stores, or save the split loads to use
5313 // below if the store is going to be split there anyways.
5314 bool DeferredStores = false;
5315 for (User *LU : LI->users()) {
5316 StoreInst *SI = cast<StoreInst>(LU);
5317 if (!Stores.empty() && SplitOffsetsMap.count(SI)) {
5318 DeferredStores = true;
5319 LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SI
5320 << "\n");
5321 continue;
5322 }
5323
5324 Value *StoreBasePtr = SI->getPointerOperand();
5325 IRB.SetInsertPoint(SI);
5326 AAMDNodes AATags = SI->getAAMetadata();
5327
5328 LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n");
5329
5330 for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) {
5331 LoadInst *PLoad = SplitLoads[Idx];
5332 uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1];
5333 auto *PartPtrTy = SI->getPointerOperandType();
5334
5335 auto AS = SI->getPointerAddressSpace();
5336 StoreInst *PStore = IRB.CreateAlignedStore(
5337 PLoad,
5338 getAdjustedPtr(IRB, DL, StoreBasePtr,
5339 APInt(DL.getIndexSizeInBits(AS), PartOffset),
5340 PartPtrTy, StoreBasePtr->getName() + "."),
5341 getAdjustedAlignment(SI, PartOffset),
5342 /*IsVolatile*/ false);
5343 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access,
5344 LLVMContext::MD_access_group,
5345 LLVMContext::MD_DIAssignID});
5346
5347 if (AATags)
5348 PStore->setAAMetadata(
5349 AATags.adjustForAccess(PartOffset, PLoad->getType(), DL));
5350 LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
5351 }
5352
5353 // We want to immediately iterate on any allocas impacted by splitting
5354 // this store, and we have to track any promotable alloca (indicated by
5355 // a direct store) as needing to be resplit because it is no longer
5356 // promotable.
5357 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(StoreBasePtr)) {
5358 ResplitPromotableAllocas.insert(OtherAI);
5359 Worklist.insert(OtherAI);
5360 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5361 StoreBasePtr->stripInBoundsOffsets())) {
5362 Worklist.insert(OtherAI);
5363 }
5364
5365 // Mark the original store as dead.
5366 DeadInsts.push_back(SI);
5367 }
5368
5369 // Save the split loads if there are deferred stores among the users.
5370 if (DeferredStores)
5371 SplitLoadsMap.insert(std::make_pair(LI, std::move(SplitLoads)));
5372
5373 // Mark the original load as dead and kill the original slice.
5374 DeadInsts.push_back(LI);
5375 Offsets.S->kill();
5376 }
5377
5378 // Second, we rewrite all of the split stores. At this point, we know that
5379 // all loads from this alloca have been split already. For stores of such
5380 // loads, we can simply look up the pre-existing split loads. For stores of
5381 // other loads, we split those loads first and then write split stores of
5382 // them.
5383 for (StoreInst *SI : Stores) {
5384 auto *LI = cast<LoadInst>(SI->getValueOperand());
5385 IntegerType *Ty = cast<IntegerType>(LI->getType());
5386 assert(Ty->getBitWidth() % 8 == 0);
5387 uint64_t StoreSize = Ty->getBitWidth() / 8;
5388 assert(StoreSize > 0 && "Cannot have a zero-sized integer store!");
5389
5390 auto &Offsets = SplitOffsetsMap[SI];
5391 assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&
5392 "Slice size should always match load size exactly!");
5393 uint64_t BaseOffset = Offsets.S->beginOffset();
5394 assert(BaseOffset + StoreSize > BaseOffset &&
5395 "Cannot represent alloca access size using 64-bit integers!");
5396
5397 Value *LoadBasePtr = LI->getPointerOperand();
5398 Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand());
5399
5400 LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n");
5401
5402 // Check whether we have an already split load.
5403 auto SplitLoadsMapI = SplitLoadsMap.find(LI);
5404 std::vector<LoadInst *> *SplitLoads = nullptr;
5405 if (SplitLoadsMapI != SplitLoadsMap.end()) {
5406 SplitLoads = &SplitLoadsMapI->second;
5407 assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&
5408 "Too few split loads for the number of splits in the store!");
5409 } else {
5410 LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n");
5411 }
5412
5413 uint64_t PartOffset = 0, PartSize = Offsets.Splits.front();
5414 int Idx = 0, Size = Offsets.Splits.size();
5415 for (;;) {
5416 auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8);
5417 auto *LoadPartPtrTy = LI->getPointerOperandType();
5418 auto *StorePartPtrTy = SI->getPointerOperandType();
5419
5420 // Either lookup a split load or create one.
5421 LoadInst *PLoad;
5422 if (SplitLoads) {
5423 PLoad = (*SplitLoads)[Idx];
5424 } else {
5425 IRB.SetInsertPoint(LI);
5426 auto AS = LI->getPointerAddressSpace();
5427 PLoad = IRB.CreateAlignedLoad(
5428 PartTy,
5429 getAdjustedPtr(IRB, DL, LoadBasePtr,
5430 APInt(DL.getIndexSizeInBits(AS), PartOffset),
5431 LoadPartPtrTy, LoadBasePtr->getName() + "."),
5432 getAdjustedAlignment(LI, PartOffset),
5433 /*IsVolatile*/ false, LI->getName());
5434 PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
5435 LLVMContext::MD_access_group});
5436 }
5437
5438 // And store this partition.
5439 IRB.SetInsertPoint(SI);
5440 auto AS = SI->getPointerAddressSpace();
5441 StoreInst *PStore = IRB.CreateAlignedStore(
5442 PLoad,
5443 getAdjustedPtr(IRB, DL, StoreBasePtr,
5444 APInt(DL.getIndexSizeInBits(AS), PartOffset),
5445 StorePartPtrTy, StoreBasePtr->getName() + "."),
5446 getAdjustedAlignment(SI, PartOffset),
5447 /*IsVolatile*/ false);
5448 PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access,
5449 LLVMContext::MD_access_group});
5450
5451 // Now build a new slice for the alloca.
5452 NewSlices.push_back(
5453 Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize,
5454 &PStore->getOperandUse(PStore->getPointerOperandIndex()),
5455 /*IsSplittable*/ false));
5456 LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()
5457 << ", " << NewSlices.back().endOffset()
5458 << "): " << *PStore << "\n");
5459 if (!SplitLoads) {
5460 LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n");
5461 }
5462
5463 // See if we've finished all the splits.
5464 if (Idx >= Size)
5465 break;
5466
5467 // Setup the next partition.
5468 PartOffset = Offsets.Splits[Idx];
5469 ++Idx;
5470 PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset;
5471 }
5472
5473 // We want to immediately iterate on any allocas impacted by splitting
5474 // this load, which is only relevant if it isn't a load of this alloca and
5475 // thus we didn't already split the loads above. We also have to keep track
5476 // of any promotable allocas we split loads on as they can no longer be
5477 // promoted.
5478 if (!SplitLoads) {
5479 if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(LoadBasePtr)) {
5480 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5481 ResplitPromotableAllocas.insert(OtherAI);
5482 Worklist.insert(OtherAI);
5483 } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(
5484 LoadBasePtr->stripInBoundsOffsets())) {
5485 assert(OtherAI != &AI && "We can't re-split our own alloca!");
5486 Worklist.insert(OtherAI);
5487 }
5488 }
5489
5490 // Mark the original store as dead now that we've split it up and kill its
5491 // slice. Note that we leave the original load in place unless this store
5492 // was its only use. It may in turn be split up if it is an alloca load
5493 // for some other alloca, but it may be a normal load. This may introduce
5494 // redundant loads, but where those can be merged the rest of the optimizer
5495 // should handle the merging, and this uncovers SSA splits which is more
5496 // important. In practice, the original loads will almost always be fully
5497 // split and removed eventually, and the splits will be merged by any
5498 // trivial CSE, including instcombine.
5499 if (LI->hasOneUse()) {
5500 assert(*LI->user_begin() == SI && "Single use isn't this store!");
5501 DeadInsts.push_back(LI);
5502 }
5503 DeadInsts.push_back(SI);
5504 Offsets.S->kill();
5505 }
5506
5507 // Remove the killed slices that have ben pre-split.
5508 llvm::erase_if(AS, [](const Slice &S) { return S.isDead(); });
5509
5510 // Insert our new slices. This will sort and merge them into the sorted
5511 // sequence.
5512 AS.insert(NewSlices);
5513
5514 LLVM_DEBUG(dbgs() << " Pre-split slices:\n");
5515#ifndef NDEBUG
5516 for (auto I = AS.begin(), E = AS.end(); I != E; ++I)
5517 LLVM_DEBUG(AS.print(dbgs(), I, " "));
5518#endif
5519
5520 // Finally, don't try to promote any allocas that new require re-splitting.
5521 // They have already been added to the worklist above.
5522 PromotableAllocas.set_subtract(ResplitPromotableAllocas);
5523
5524 return true;
5525}
5526
5527/// Try to canonicalize a homogeneous struct partition to a vector type.
5528///
5529/// We can do this if all the elements of the struct are the same and the
5530/// corresponding vector has the same byte-level layout. This can sometimes
5531/// eliminate allocas because structs cannot get promoted to LLVM values, but
5532/// vectors can.
5533///
5534/// We only apply this transformation when all users of the partition are memory
5535/// intrinsics. Otherwise, if there is a load or store of some other type to the
5536/// partition, SROA would select that type.
5537///
5538/// Applying this transformation too early may hinder memcpyopt, which may
5539/// generate better code when eliminating allocas. For example, see
5540/// `struct-to-vector-fp-store-only-tail.ll`, which demonstrates that applying
5541/// this before memcpyopt can initialize previously uninitialized memory when
5542/// the alloca gets promoted to an SSA value. For another example, see
5543/// `struct-to-vector-before-memcpyopt.ll`, which demonstrates that applying
5544/// this before memcpyopt can result in promoting an alloca so that we load a
5545/// temporary value instead of copying the temporary value into memory, whereas
5546/// memcpyopt eliminates the temporary altogether.
5547///
5548/// As such, we only apply this transformation after memcpyopt has run. We gate
5549/// this transformation by the "AggregateToVector" pass option.
5551 Partition &P,
5552 const DataLayout &DL) {
5553 unsigned NumElts = STy->getNumElements();
5554
5555 Type *EltTy = STy->getElementType(0);
5556 if (!llvm::all_equal(STy->elements()))
5557 return nullptr;
5558
5559 bool IsIntegralPointerTy =
5560 EltTy->isPointerTy() && !DL.isNonIntegralPointerType(EltTy);
5561 if (!EltTy->isIntegerTy() && !EltTy->isFloatingPointTy() &&
5562 !IsIntegralPointerTy)
5563 return nullptr;
5564
5565 // Ensure the struct is tightly packed so that the bit-layout is the same as
5566 // the corresponding vector. For example, this prevents a miscompile for
5567 // { i5, i5 }, which has padding after each i5 field, whereas <i5, i5> has
5568 // tightly packed elements and trailing padding.
5569 if (DL.getTypeSizeInBits(EltTy) != DL.getTypeAllocSizeInBits(EltTy))
5570 return nullptr;
5571
5572 auto *VTy = FixedVectorType::get(EltTy, NumElts);
5573 TypeSize StructSize = DL.getStructLayout(STy)->getSizeInBytes();
5574 TypeSize VectorSize = DL.getTypeStoreSize(VTy);
5575 // After ruling out per-element padding, make sure a vector load/store
5576 // covers the same number of bytes as the struct layout.
5577 if (StructSize != VectorSize)
5578 return nullptr;
5579
5580 auto IsIgnorableOrMemIntrinsicSlice = [](const Slice &S) {
5581 if (S.isDead())
5582 return true;
5583 auto *U = S.getUse();
5584 if (!U)
5585 return true;
5586
5587 User *Usr = U->getUser();
5589 return true;
5590
5591 return isa<MemIntrinsic>(Usr);
5592 };
5593
5594 for (const Slice &S : P)
5595 if (!IsIgnorableOrMemIntrinsicSlice(S))
5596 return nullptr;
5597
5598 for (const Slice *S : P.splitSliceTails())
5599 if (!IsIgnorableOrMemIntrinsicSlice(*S))
5600 return nullptr;
5601
5602 return VTy;
5603}
5604
5605/// Select a partition type for an alloca partition.
5606///
5607/// Try to compute a friendly type for this partition of the alloca. This
5608/// won't always succeed, in which case we fall back to a legal integer type
5609/// or an i8 array of an appropriate size.
5610///
5611/// \returns A tuple with the following elements:
5612/// - PartitionType: The computed type for this partition.
5613/// - IsIntegerWideningViable: True if integer widening promotion is used.
5614/// - VectorType: The vector type if vector promotion is used, otherwise
5615/// nullptr.
5616static std::tuple<Type *, bool, VectorType *>
5618 LLVMContext &C, bool AggregateToVector) {
5619 auto LogSelection = [&](StringRef Path, Type *SelectedTy,
5620 VectorType *SelectedVecTy, bool SelectedIntWidening) {
5621 LLVM_DEBUG({
5622 dbgs() << "selectPartitionType path=" << Path
5623 << " func=" << AI.getFunction()->getName() << " alloca=";
5624 if (AI.hasName())
5625 dbgs() << AI.getName();
5626 else
5627 dbgs() << "<unnamed>";
5628 dbgs() << " partition=[" << P.beginOffset() << "," << P.endOffset()
5629 << ") size=" << P.size();
5630 if (std::optional<TypeSize> AllocSize = AI.getAllocationSize(DL))
5631 dbgs() << " alloc-size=" << AllocSize->getKnownMinValue();
5632 if (SelectedTy)
5633 dbgs() << " chosen=" << *SelectedTy;
5634 if (SelectedVecTy)
5635 dbgs() << " vec=" << *SelectedVecTy;
5636 dbgs() << " intwiden=" << SelectedIntWidening << "\n";
5637 });
5638 };
5639 // First check if the partition is viable for vector promotion.
5640 //
5641 // We prefer vector promotion over integer widening promotion when:
5642 // - The vector element type is a floating-point type.
5643 // - All the loads/stores to the alloca are vector loads/stores to the
5644 // entire alloca or load/store a single element of the vector.
5645 //
5646 // Otherwise when there is an integer vector with mixed type loads/stores we
5647 // prefer integer widening promotion because it's more likely the user is
5648 // doing bitwise arithmetic and we generate better code.
5649 VectorType *VecTy =
5651 // If the vector element type is a floating-point type, we prefer vector
5652 // promotion. If the vector has one element, let the below code select
5653 // whether we promote with the vector or scalar.
5654 if (VecTy && VecTy->getElementType()->isFloatingPointTy() &&
5655 VecTy->getElementCount().getFixedValue() > 1) {
5656 LogSelection("direct-fp-vecty", VecTy, VecTy, false);
5657 return {VecTy, false, VecTy};
5658 }
5659
5660 // Check if there is a common type that all slices of the partition use that
5661 // spans the partition.
5662 auto [CommonUseTy, LargestIntTy] =
5663 findCommonType(P.begin(), P.end(), P.endOffset());
5664 if (CommonUseTy) {
5665 TypeSize CommonUseSize = DL.getTypeAllocSize(CommonUseTy);
5666 if (CommonUseSize.isFixed() && CommonUseSize.getFixedValue() >= P.size()) {
5667 // We prefer vector promotion here because if vector promotion is viable
5668 // and there is a common type used, then it implies the second listed
5669 // condition for preferring vector promotion is true.
5670 if (VecTy) {
5671 LogSelection("common-type-vecty", VecTy, VecTy, false);
5672 return {VecTy, false, VecTy};
5673 }
5674 bool IntWiden = isIntegerWideningViable(P, CommonUseTy, DL);
5675 LogSelection("common-type", CommonUseTy, nullptr, IntWiden);
5676 return {CommonUseTy, IntWiden, nullptr};
5677 }
5678 }
5679
5680 // Can we find an appropriate subtype in the original allocated
5681 // type?
5682 if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(),
5683 P.beginOffset(), P.size())) {
5684 // If the partition is an integer array that can be spanned by a legal
5685 // integer type, prefer to represent it as a legal integer type because
5686 // it's more likely to be promotable.
5687 if (TypePartitionTy->isArrayTy() &&
5688 TypePartitionTy->getArrayElementType()->isIntegerTy() &&
5689 DL.isLegalInteger(P.size() * 8))
5690 TypePartitionTy = Type::getIntNTy(C, P.size() * 8);
5691 // There was no common type used, so we prefer integer widening promotion.
5692 if (isIntegerWideningViable(P, TypePartitionTy, DL)) {
5693 LogSelection("type-partition-int-widen", TypePartitionTy, nullptr, true);
5694 return {TypePartitionTy, true, nullptr};
5695 }
5696 if (VecTy) {
5697 LogSelection("type-partition-vecty", VecTy, VecTy, false);
5698 return {VecTy, false, VecTy};
5699 }
5700 // If we couldn't promote with TypePartitionTy, try with the largest
5701 // integer type used.
5702 if (LargestIntTy &&
5703 DL.getTypeAllocSize(LargestIntTy).getFixedValue() >= P.size() &&
5704 isIntegerWideningViable(P, LargestIntTy, DL)) {
5705 LogSelection("largest-int-int-widen", LargestIntTy, nullptr, true);
5706 return {LargestIntTy, true, nullptr};
5707 }
5708
5709 // Try homogeneous struct to vector canonicalization when requested. Running
5710 // this too early can hide memcpy chains from MemCpyOpt.
5711 if (AggregateToVector) {
5712 if (auto *STy = dyn_cast<StructType>(TypePartitionTy)) {
5713 if (auto *VTy = tryCanonicalizeStructToVector(STy, P, DL)) {
5714 LogSelection("struct-fallback-vecty", VTy, nullptr, false);
5715 return {VTy, false, nullptr};
5716 }
5717 }
5718 }
5719
5720 // Fallback to TypePartitionTy and we probably won't promote.
5721 LogSelection("type-partition-fallback", TypePartitionTy, nullptr, false);
5722 return {TypePartitionTy, false, nullptr};
5723 }
5724
5725 // Select the largest integer type used if it spans the partition.
5726 if (LargestIntTy &&
5727 DL.getTypeAllocSize(LargestIntTy).getFixedValue() >= P.size()) {
5728 LogSelection("largest-int-fallback", LargestIntTy, nullptr, false);
5729 return {LargestIntTy, false, nullptr};
5730 }
5731
5732 // Select a legal integer type if it spans the partition.
5733 if (DL.isLegalInteger(P.size() * 8)) {
5734 Type *IntTy = Type::getIntNTy(C, P.size() * 8);
5735 LogSelection("legal-int-fallback", IntTy, nullptr, false);
5736 return {IntTy, false, nullptr};
5737 }
5738
5739 // Fallback to an i8 array.
5740 Type *ArrayTy = ArrayType::get(Type::getInt8Ty(C), P.size());
5741 LogSelection("byte-array-fallback", ArrayTy, nullptr, false);
5742 return {ArrayTy, false, nullptr};
5743}
5744
5745/// Rewrite an alloca partition's users.
5746///
5747/// This routine drives both of the rewriting goals of the SROA pass. It tries
5748/// to rewrite uses of an alloca partition to be conducive for SSA value
5749/// promotion. If the partition needs a new, more refined alloca, this will
5750/// build that new alloca, preserving as much type information as possible, and
5751/// rewrite the uses of the old alloca to point at the new one and have the
5752/// appropriate new offsets. It also evaluates how successful the rewrite was
5753/// at enabling promotion and if it was successful queues the alloca to be
5754/// promoted.
5755std::pair<AllocaInst *, uint64_t>
5756SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, Partition &P) {
5757 const DataLayout &DL = AI.getDataLayout();
5758 // Select the type for the new alloca that spans the partition.
5759 auto [PartitionTy, IsIntegerWideningViable, VecTy] =
5760 selectPartitionType(P, DL, AI, *C, AggregateToVector);
5761
5762 // Check for the case where we're going to rewrite to a new alloca of the
5763 // exact same type as the original, and with the same access offsets. In that
5764 // case, re-use the existing alloca, but still run through the rewriter to
5765 // perform phi and select speculation.
5766 // P.beginOffset() can be non-zero even with the same type in a case with
5767 // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll).
5768 AllocaInst *NewAI;
5769 if (PartitionTy == AI.getAllocatedType() && P.beginOffset() == 0) {
5770 NewAI = &AI;
5771 // FIXME: We should be able to bail at this point with "nothing changed".
5772 // FIXME: We might want to defer PHI speculation until after here.
5773 // FIXME: return nullptr;
5774 } else {
5775 // Make sure the alignment is compatible with P.beginOffset().
5776 const Align Alignment = commonAlignment(AI.getAlign(), P.beginOffset());
5777 NewAI =
5778 new AllocaInst(PartitionTy, AI.getAddressSpace(), nullptr, Alignment,
5779 AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()),
5780 AI.getIterator());
5781 tryEnforceAlignment(NewAI, DL.getPrefTypeAlign(PartitionTy), DL);
5782 // Copy the old AI debug location over to the new one.
5783 NewAI->setDebugLoc(AI.getDebugLoc());
5784 ++NumNewAllocas;
5785 }
5786
5787 LLVM_DEBUG(dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset()
5788 << "," << P.endOffset() << ") to: " << *NewAI << "\n");
5789
5790 // Track the high watermark on the worklist as it is only relevant for
5791 // promoted allocas. We will reset it to this point if the alloca is not in
5792 // fact scheduled for promotion.
5793 unsigned PPWOldSize = PostPromotionWorklist.size();
5794 unsigned NumUses = 0;
5795 SmallSetVector<PHINode *, 8> PHIUsers;
5796 SmallSetVector<SelectInst *, 8> SelectUsers;
5797
5798 AllocaSliceRewriter Rewriter(
5799 DL, AS, *this, AI, *NewAI, PartitionTy, P.beginOffset(), P.endOffset(),
5800 IsIntegerWideningViable, VecTy, PHIUsers, SelectUsers);
5801 bool Promotable = true;
5802 // Check whether we can have tree-structured merge.
5803 if (auto DeletedValues = Rewriter.rewriteTreeStructuredMerge(P)) {
5804 NumUses += DeletedValues->size() + 1;
5805 for (Value *V : *DeletedValues)
5806 DeadInsts.push_back(V);
5807 } else {
5808 for (Slice *S : P.splitSliceTails()) {
5809 Promotable &= Rewriter.visit(S);
5810 ++NumUses;
5811 }
5812 for (Slice &S : P) {
5813 Promotable &= Rewriter.visit(&S);
5814 ++NumUses;
5815 }
5816 }
5817
5818 NumAllocaPartitionUses += NumUses;
5819 MaxUsesPerAllocaPartition.updateMax(NumUses);
5820
5821 // Now that we've processed all the slices in the new partition, check if any
5822 // PHIs or Selects would block promotion.
5823 SmallVector<PHINode *, 8> NewSpeculatablePHIs;
5824 SmallVector<PHINode *, 2> NewPHIsWithStoreToRewrite;
5825 for (PHINode *PHI : PHIUsers) {
5826 if (isSafePHIToSpeculate(*PHI)) {
5827 NewSpeculatablePHIs.push_back(PHI);
5828 continue;
5829 }
5830 if (getPHIStoreToRewrite(*PHI, PreserveCFG, DTU->getDomTree())) {
5831 NewPHIsWithStoreToRewrite.push_back(PHI);
5832 continue;
5833 }
5834
5835 Promotable = false;
5836 SelectUsers.clear();
5837 break;
5838 }
5839
5841 NewSelectsToRewrite;
5842 NewSelectsToRewrite.reserve(SelectUsers.size());
5843 for (SelectInst *Sel : SelectUsers) {
5844 std::optional<RewriteableMemOps> Ops =
5845 isSafeSelectToSpeculate(*Sel, PreserveCFG);
5846 if (!Ops) {
5847 Promotable = false;
5848 break;
5849 }
5850 NewSelectsToRewrite.emplace_back(std::make_pair(Sel, *Ops));
5851 }
5852
5853 if (Promotable) {
5854 for (Use *U : AS.getDeadUsesIfPromotable()) {
5855 auto *OldInst = dyn_cast<Instruction>(U->get());
5856 Value::dropDroppableUse(*U);
5857 if (OldInst)
5858 if (isInstructionTriviallyDead(OldInst))
5859 DeadInsts.push_back(OldInst);
5860 }
5861 if (NewSpeculatablePHIs.empty() && NewPHIsWithStoreToRewrite.empty() &&
5862 SelectUsers.empty()) {
5863 // Promote the alloca.
5864 PromotableAllocas.insert(NewAI);
5865 } else {
5866 // If we have either PHIs or Selects to rewrite, add them to those
5867 // worklists and re-queue the new alloca so that we promote it on the next
5868 // iteration.
5869 SpeculatablePHIs.insert_range(NewSpeculatablePHIs);
5870 PHIsWithStoreToRewrite.insert_range(NewPHIsWithStoreToRewrite);
5871 SelectsToRewrite.reserve(SelectsToRewrite.size() +
5872 NewSelectsToRewrite.size());
5873 for (auto &&KV : llvm::make_range(
5874 std::make_move_iterator(NewSelectsToRewrite.begin()),
5875 std::make_move_iterator(NewSelectsToRewrite.end())))
5876 SelectsToRewrite.insert(std::move(KV));
5877 Worklist.insert(NewAI);
5878 }
5879 } else {
5880 // Drop any post-promotion work items if promotion didn't happen.
5881 while (PostPromotionWorklist.size() > PPWOldSize)
5882 PostPromotionWorklist.pop_back();
5883
5884 // We couldn't promote and we didn't create a new partition, nothing
5885 // happened.
5886 if (NewAI == &AI)
5887 return {nullptr, 0};
5888
5889 // If we can't promote the alloca, iterate on it to check for new
5890 // refinements exposed by splitting the current alloca. Don't iterate on an
5891 // alloca which didn't actually change and didn't get promoted.
5892 Worklist.insert(NewAI);
5893 }
5894
5895 return {NewAI, DL.getTypeSizeInBits(PartitionTy).getFixedValue()};
5896}
5897
5898// There isn't a shared interface to get the "address" parts out of a
5899// dbg.declare and dbg.assign, so provide some wrappers.
5902 return DVR->isKillAddress();
5903 return DVR->isKillLocation();
5904}
5905
5908 return DVR->getAddressExpression();
5909 return DVR->getExpression();
5910}
5911
5912/// Create or replace an existing fragment in a DIExpression with \p Frag.
5913/// If the expression already contains a DW_OP_LLVM_extract_bits_[sz]ext
5914/// operation, add \p BitExtractOffset to the offset part.
5915///
5916/// Returns the new expression, or nullptr if this fails (see details below).
5917///
5918/// This function is similar to DIExpression::createFragmentExpression except
5919/// for 3 important distinctions:
5920/// 1. The new fragment isn't relative to an existing fragment.
5921/// 2. It assumes the computed location is a memory location. This means we
5922/// don't need to perform checks that creating the fragment preserves the
5923/// expression semantics.
5924/// 3. Existing extract_bits are modified independently of fragment changes
5925/// using \p BitExtractOffset. A change to the fragment offset or size
5926/// may affect a bit extract. But a bit extract offset can change
5927/// independently of the fragment dimensions.
5928///
5929/// Returns the new expression, or nullptr if one couldn't be created.
5930/// Ideally this is only used to signal that a bit-extract has become
5931/// zero-sized (and thus the new debug record has no size and can be
5932/// dropped), however, it fails for other reasons too - see the FIXME below.
5933///
5934/// FIXME: To keep the change that introduces this function NFC it bails
5935/// in some situations unecessarily, e.g. when fragment and bit extract
5936/// sizes differ.
5939 int64_t BitExtractOffset) {
5941 bool HasFragment = false;
5942 bool HasBitExtract = false;
5943
5944 for (auto &Op : Expr->expr_ops()) {
5945 if (Op.getOp() == dwarf::DW_OP_LLVM_fragment) {
5946 HasFragment = true;
5947 continue;
5948 }
5949 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Op)) {
5950 HasBitExtract = true;
5951 int64_t ExtractOffsetInBits = Extract.getOffsetInBits();
5952 int64_t ExtractSizeInBits = Extract.getSizeInBits();
5953
5954 // DIExpression::createFragmentExpression doesn't know how to handle
5955 // a fragment that is smaller than the extract. Copy the behaviour
5956 // (bail) to avoid non-NFC changes.
5957 // FIXME: Don't do this.
5958 if (Frag.SizeInBits < uint64_t(ExtractSizeInBits))
5959 return nullptr;
5960
5961 assert(BitExtractOffset <= 0);
5962 int64_t AdjustedOffset = ExtractOffsetInBits + BitExtractOffset;
5963
5964 // DIExpression::createFragmentExpression doesn't know what to do
5965 // if the new extract starts "outside" the existing one. Copy the
5966 // behaviour (bail) to avoid non-NFC changes.
5967 // FIXME: Don't do this.
5968 if (AdjustedOffset < 0)
5969 return nullptr;
5970
5971 Ops.push_back(Op.getOp());
5972 Ops.push_back(std::max<int64_t>(0, AdjustedOffset));
5973 Ops.push_back(ExtractSizeInBits);
5974 continue;
5975 }
5976 Op.appendToVector(Ops);
5977 }
5978
5979 // Unsupported by createFragmentExpression, so don't support it here yet to
5980 // preserve NFC-ness.
5981 if (HasFragment && HasBitExtract)
5982 return nullptr;
5983
5984 if (!HasBitExtract) {
5986 Ops.push_back(Frag.OffsetInBits);
5987 Ops.push_back(Frag.SizeInBits);
5988 }
5989 return DIExpression::get(Expr->getContext(), Ops);
5990}
5991
5992/// Insert a new DbgRecord.
5993/// \p Orig Original to copy record type, debug loc and variable from, and
5994/// additionally value and value expression for dbg_assign records.
5995/// \p NewAddr Location's new base address.
5996/// \p NewAddrExpr New expression to apply to address.
5997/// \p BeforeInst Insert position.
5998/// \p NewFragment New fragment (absolute, non-relative).
5999/// \p BitExtractAdjustment Offset to apply to any extract_bits op.
6000static void
6002 DIExpression *NewAddrExpr, Instruction *BeforeInst,
6003 std::optional<DIExpression::FragmentInfo> NewFragment,
6004 int64_t BitExtractAdjustment) {
6005 (void)DIB;
6006
6007 // A dbg_assign puts fragment info in the value expression only. The address
6008 // expression has already been built: NewAddrExpr. A dbg_declare puts the
6009 // new fragment info into NewAddrExpr (as it only has one expression).
6010 DIExpression *NewFragmentExpr =
6011 Orig->isDbgAssign() ? Orig->getExpression() : NewAddrExpr;
6012 if (NewFragment)
6013 NewFragmentExpr = createOrReplaceFragment(NewFragmentExpr, *NewFragment,
6014 BitExtractAdjustment);
6015 if (!NewFragmentExpr)
6016 return;
6017
6018 if (Orig->isDbgDeclare()) {
6020 NewAddr, Orig->getVariable(), NewFragmentExpr, Orig->getDebugLoc());
6021 BeforeInst->getParent()->insertDbgRecordBefore(DVR,
6022 BeforeInst->getIterator());
6023 return;
6024 }
6025
6026 if (Orig->isDbgValue()) {
6028 NewAddr, Orig->getVariable(), NewFragmentExpr, Orig->getDebugLoc());
6029 // Drop debug information if the expression doesn't start with a
6030 // DW_OP_deref. This is because without a DW_OP_deref, the #dbg_value
6031 // describes the address of alloca rather than the value inside the alloca.
6032 if (!NewFragmentExpr->startsWithDeref())
6033 DVR->setKillAddress();
6034 BeforeInst->getParent()->insertDbgRecordBefore(DVR,
6035 BeforeInst->getIterator());
6036 return;
6037 }
6038
6039 // Apply a DIAssignID to the store if it doesn't already have it.
6040 if (!NewAddr->hasMetadata(LLVMContext::MD_DIAssignID)) {
6041 NewAddr->setMetadata(LLVMContext::MD_DIAssignID,
6043 }
6044
6046 NewAddr, Orig->getValue(), Orig->getVariable(), NewFragmentExpr, NewAddr,
6047 NewAddrExpr, Orig->getDebugLoc());
6048 LLVM_DEBUG(dbgs() << "Created new DVRAssign: " << *NewAssign << "\n");
6049 (void)NewAssign;
6050}
6051
6052/// Walks the slices of an alloca and form partitions based on them,
6053/// rewriting each of their uses.
6054bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
6055 if (AS.begin() == AS.end())
6056 return false;
6057
6058 unsigned NumPartitions = 0;
6059 bool Changed = false;
6060 const DataLayout &DL = AI.getModule()->getDataLayout();
6061
6062 // First try to pre-split loads and stores.
6063 Changed |= presplitLoadsAndStores(AI, AS);
6064
6065 // Now that we have identified any pre-splitting opportunities,
6066 // mark loads and stores unsplittable except for the following case.
6067 // We leave a slice splittable if all other slices are disjoint or fully
6068 // included in the slice, such as whole-alloca loads and stores.
6069 // If we fail to split these during pre-splitting, we want to force them
6070 // to be rewritten into a partition.
6071 bool IsSorted = true;
6072
6073 uint64_t AllocaSize = AI.getAllocationSize(DL)->getFixedValue();
6074 // We can split at the begin and end offsets of each slice, but only if those
6075 // offsets don't lie inside another slice. Because slices are ordered by
6076 // increasing begin offset, and then decreasing end offset, we can consider
6077 // the slices as being split up into sets with the same begin offset where we
6078 // can ignore every slice except the first (the begin offset will already be
6079 // handled as the begin offset of the set, and the end offset we know is not a
6080 // splittable offset as it's inside the first slice of the set).
6081 SparseBitVector<> SplittableOffset;
6082 uint64_t CurBegin = 0, CurEnd = 0;
6083 for (Slice &S : AS) {
6084 // Check if we have a new set of slices
6085 if (S.beginOffset() > CurBegin || S.endOffset() > CurEnd) {
6086 // If the start isn't inside the previous set it's splittable
6087 if (S.beginOffset() >= CurEnd) {
6088 SplittableOffset.set(S.beginOffset());
6089 }
6090 // If the previous end is inside this slice then remove it
6091 if (CurEnd > S.beginOffset() && CurEnd < S.endOffset()) {
6092 SplittableOffset.reset(CurEnd);
6093 }
6094 CurBegin = S.beginOffset();
6095 // If the end offset isn't inside the previous set it's splittable. We
6096 // also don't update the end offset in that case, as the next set may also
6097 // be inside the previous set.
6098 if (S.endOffset() > CurEnd) {
6099 CurEnd = S.endOffset();
6100 SplittableOffset.set(CurEnd);
6101 }
6102 }
6103 }
6104
6105 for (Slice &S : AS) {
6106 if (!S.isSplittable())
6107 continue;
6108
6109 if ((S.beginOffset() > AllocaSize ||
6110 SplittableOffset.test(S.beginOffset())) &&
6111 (S.endOffset() > AllocaSize || SplittableOffset.test(S.endOffset())))
6112 continue;
6113
6114 if (isa<LoadInst>(S.getUse()->getUser()) ||
6115 isa<StoreInst>(S.getUse()->getUser())) {
6116 S.makeUnsplittable();
6117 IsSorted = false;
6118 }
6119 }
6120
6121 if (!IsSorted)
6123
6124 /// Describes the allocas introduced by rewritePartition in order to migrate
6125 /// the debug info.
6126 struct Fragment {
6127 AllocaInst *Alloca;
6129 uint64_t Size;
6130 Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
6131 : Alloca(AI), Offset(O), Size(S) {}
6132 };
6133 SmallVector<Fragment, 4> Fragments;
6134
6135 // Rewrite each partition.
6136 for (auto &P : AS.partitions()) {
6137 auto [NewAI, ActiveBits] = rewritePartition(AI, AS, P);
6138 if (NewAI) {
6139 Changed = true;
6140 if (NewAI != &AI) {
6141 uint64_t SizeOfByte = 8;
6142 // Don't include any padding.
6143 uint64_t Size = std::min(ActiveBits, P.size() * SizeOfByte);
6144 Fragments.push_back(
6145 Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
6146 }
6147 }
6148 ++NumPartitions;
6149 }
6150
6151 NumAllocaPartitions += NumPartitions;
6152 MaxPartitionsPerAlloca.updateMax(NumPartitions);
6153
6154 // Migrate debug information from the old alloca to the new alloca(s)
6155 // and the individual partitions.
6156 auto MigrateOne = [&](DbgVariableRecord *DbgVariable) {
6157 // Can't overlap with undef memory.
6158 if (isKillAddress(DbgVariable))
6159 return;
6160
6161 const Value *DbgPtr = DbgVariable->getAddress();
6163 DbgVariable->getFragmentOrEntireVariable();
6164 // Get the address expression constant offset if one exists and the ops
6165 // that come after it.
6166 int64_t CurrentExprOffsetInBytes = 0;
6167 SmallVector<uint64_t> PostOffsetOps;
6168 if (!getAddressExpression(DbgVariable)
6169 ->extractLeadingOffset(CurrentExprOffsetInBytes, PostOffsetOps))
6170 return; // Couldn't interpret this DIExpression - drop the var.
6171
6172 // Offset defined by a DW_OP_LLVM_extract_bits_[sz]ext.
6173 int64_t ExtractOffsetInBits = 0;
6174 for (auto Op : getAddressExpression(DbgVariable)->expr_ops()) {
6175 if (auto Extract = dyn_cast<DIExpression::ExtractBitsOp>(Op)) {
6176 ExtractOffsetInBits = Extract.getOffsetInBits();
6177 break;
6178 }
6179 }
6180
6181 DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false);
6182 for (auto Fragment : Fragments) {
6183 int64_t OffsetFromLocationInBits;
6184 std::optional<DIExpression::FragmentInfo> NewDbgFragment;
6185 // Find the variable fragment that the new alloca slice covers.
6186 // Drop debug info for this variable fragment if we can't compute an
6187 // intersect between it and the alloca slice.
6189 DL, &AI, Fragment.Offset, Fragment.Size, DbgPtr,
6190 CurrentExprOffsetInBytes * 8, ExtractOffsetInBits, VarFrag,
6191 NewDbgFragment, OffsetFromLocationInBits))
6192 continue; // Do not migrate this fragment to this slice.
6193
6194 // Zero sized fragment indicates there's no intersect between the variable
6195 // fragment and the alloca slice. Skip this slice for this variable
6196 // fragment.
6197 if (NewDbgFragment && !NewDbgFragment->SizeInBits)
6198 continue; // Do not migrate this fragment to this slice.
6199
6200 // No fragment indicates DbgVariable's variable or fragment exactly
6201 // overlaps the slice; copy its fragment (or nullopt if there isn't one).
6202 if (!NewDbgFragment)
6203 NewDbgFragment = DbgVariable->getFragment();
6204
6205 // Reduce the new expression offset by the bit-extract offset since
6206 // we'll be keeping that.
6207 int64_t OffestFromNewAllocaInBits =
6208 OffsetFromLocationInBits - ExtractOffsetInBits;
6209 // We need to adjust an existing bit extract if the offset expression
6210 // can't eat the slack (i.e., if the new offset would be negative).
6211 int64_t BitExtractOffset =
6212 std::min<int64_t>(0, OffestFromNewAllocaInBits);
6213 // The magnitude of a negative value indicates the number of bits into
6214 // the existing variable fragment that the memory region begins. The new
6215 // variable fragment already excludes those bits - the new DbgPtr offset
6216 // only needs to be applied if it's positive.
6217 OffestFromNewAllocaInBits =
6218 std::max(int64_t(0), OffestFromNewAllocaInBits);
6219
6220 // Rebuild the expression:
6221 // {Offset(OffestFromNewAllocaInBits), PostOffsetOps, NewDbgFragment}
6222 // Add NewDbgFragment later, because dbg.assigns don't want it in the
6223 // address expression but the value expression instead.
6224 DIExpression *NewExpr = DIExpression::get(AI.getContext(), PostOffsetOps);
6225 if (OffestFromNewAllocaInBits > 0) {
6226 int64_t OffsetInBytes = (OffestFromNewAllocaInBits + 7) / 8;
6227 NewExpr = DIExpression::prepend(NewExpr, /*flags=*/0, OffsetInBytes);
6228 }
6229
6230 // Remove any existing intrinsics on the new alloca describing
6231 // the variable fragment.
6232 auto RemoveOne = [DbgVariable](auto *OldDII) {
6233 auto SameVariableFragment = [](const auto *LHS, const auto *RHS) {
6234 return LHS->getVariable() == RHS->getVariable() &&
6235 LHS->getDebugLoc()->getInlinedAt() ==
6236 RHS->getDebugLoc()->getInlinedAt();
6237 };
6238 if (SameVariableFragment(OldDII, DbgVariable))
6239 OldDII->eraseFromParent();
6240 };
6241 for_each(findDVRDeclares(Fragment.Alloca), RemoveOne);
6242 for_each(findDVRValues(Fragment.Alloca), RemoveOne);
6243 insertNewDbgInst(DIB, DbgVariable, Fragment.Alloca, NewExpr, &AI,
6244 NewDbgFragment, BitExtractOffset);
6245 }
6246 };
6247
6248 // Migrate debug information from the old alloca to the new alloca(s)
6249 // and the individual partitions.
6250 for_each(findDVRDeclares(&AI), MigrateOne);
6251 for_each(findDVRValues(&AI), MigrateOne);
6252 for_each(at::getDVRAssignmentMarkers(&AI), MigrateOne);
6253
6254 return Changed;
6255}
6256
6257/// Clobber a use with poison, deleting the used value if it becomes dead.
6258void SROA::clobberUse(Use &U) {
6259 Value *OldV = U;
6260 // Replace the use with an poison value.
6261 U = PoisonValue::get(OldV->getType());
6262
6263 // Check for this making an instruction dead. We have to garbage collect
6264 // all the dead instructions to ensure the uses of any alloca end up being
6265 // minimal.
6266 if (Instruction *OldI = dyn_cast<Instruction>(OldV))
6267 if (isInstructionTriviallyDead(OldI)) {
6268 DeadInsts.push_back(OldI);
6269 }
6270}
6271
6272/// A basic LoadAndStorePromoter that does not remove store nodes.
6274public:
6276 Type *ZeroType)
6277 : LoadAndStorePromoter(Insts, S), ZeroType(ZeroType) {}
6278 bool shouldDelete(Instruction *I) const override {
6279 return !isa<StoreInst>(I) && !isa<AllocaInst>(I);
6280 }
6281
6283 return UndefValue::get(ZeroType);
6284 }
6285
6286private:
6287 Type *ZeroType;
6288};
6289
6290bool SROA::propagateStoredValuesToLoads(AllocaInst &AI, AllocaSlices &AS) {
6291 // Look through each "partition", looking for slices with the same start/end
6292 // that do not overlap with any before them. The slices are sorted by
6293 // increasing beginOffset. We don't use AS.partitions(), as it will use a more
6294 // sophisticated algorithm that takes splittable slices into account.
6295 LLVM_DEBUG(dbgs() << "Attempting to propagate values on " << AI << "\n");
6296 bool AllSameAndValid = true;
6297 Type *PartitionType = nullptr;
6298 SmallVector<Instruction *> Insts;
6299 uint64_t BeginOffset = 0;
6300 uint64_t EndOffset = 0;
6301
6302 auto Flush = [&]() {
6303 if (AllSameAndValid && !Insts.empty()) {
6304 LLVM_DEBUG(dbgs() << "Propagate values on slice [" << BeginOffset << ", "
6305 << EndOffset << ")\n");
6307 SSAUpdater SSA(&NewPHIs);
6308 Insts.push_back(&AI);
6309 BasicLoadAndStorePromoter Promoter(Insts, SSA, PartitionType);
6310 Promoter.run(Insts);
6311 }
6312 AllSameAndValid = true;
6313 PartitionType = nullptr;
6314 Insts.clear();
6315 };
6316
6317 for (Slice &S : AS) {
6318 auto *User = cast<Instruction>(S.getUse()->getUser());
6319 if (isAssumeLikeIntrinsic(User)) {
6320 LLVM_DEBUG({
6321 dbgs() << "Ignoring slice: ";
6322 AS.print(dbgs(), &S);
6323 });
6324 continue;
6325 }
6326 if (S.beginOffset() >= EndOffset) {
6327 Flush();
6328 BeginOffset = S.beginOffset();
6329 EndOffset = S.endOffset();
6330 } else if (S.beginOffset() != BeginOffset || S.endOffset() != EndOffset) {
6331 if (AllSameAndValid) {
6332 LLVM_DEBUG({
6333 dbgs() << "Slice does not match range [" << BeginOffset << ", "
6334 << EndOffset << ")";
6335 AS.print(dbgs(), &S);
6336 });
6337 AllSameAndValid = false;
6338 }
6339 EndOffset = std::max(EndOffset, S.endOffset());
6340 continue;
6341 }
6342
6343 if (auto *LI = dyn_cast<LoadInst>(User)) {
6344 Type *UserTy = LI->getType();
6345 // LoadAndStorePromoter requires all the types to be the same.
6346 if (!LI->isSimple() || (PartitionType && UserTy != PartitionType))
6347 AllSameAndValid = false;
6348 PartitionType = UserTy;
6349 Insts.push_back(User);
6350 } else if (auto *SI = dyn_cast<StoreInst>(User)) {
6351 Type *UserTy = SI->getValueOperand()->getType();
6352 if (!SI->isSimple() || (PartitionType && UserTy != PartitionType))
6353 AllSameAndValid = false;
6354 PartitionType = UserTy;
6355 Insts.push_back(User);
6356 } else {
6357 AllSameAndValid = false;
6358 }
6359 }
6360
6361 Flush();
6362 return true;
6363}
6364
6365/// Analyze an alloca for SROA.
6366///
6367/// This analyzes the alloca to ensure we can reason about it, builds
6368/// the slices of the alloca, and then hands it off to be split and
6369/// rewritten as needed.
6370std::pair<bool /*Changed*/, bool /*CFGChanged*/>
6371SROA::runOnAlloca(AllocaInst &AI) {
6372 bool Changed = false;
6373 bool CFGChanged = false;
6374
6375 LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
6376 ++NumAllocasAnalyzed;
6377
6378 // Special case dead allocas, as they're trivial.
6379 if (AI.use_empty()) {
6380 AI.eraseFromParent();
6381 Changed = true;
6382 return {Changed, CFGChanged};
6383 }
6384 const DataLayout &DL = AI.getDataLayout();
6385
6386 // Skip alloca forms that this analysis can't handle.
6387 std::optional<TypeSize> Size = AI.getAllocationSize(DL);
6388 if (AI.isArrayAllocation() || !Size || Size->isScalable() || Size->isZero())
6389 return {Changed, CFGChanged};
6390
6391 // First, split any FCA loads and stores touching this alloca to promote
6392 // better splitting and promotion opportunities.
6393 IRBuilderTy IRB(&AI);
6394 AggLoadStoreRewriter AggRewriter(DL, IRB);
6395 Changed |= AggRewriter.rewrite(AI);
6396
6397 // Build the slices using a recursive instruction-visiting builder.
6398 AllocaSlices AS(DL, AI);
6399 LLVM_DEBUG(AS.print(dbgs()));
6400 if (AS.isEscaped())
6401 return {Changed, CFGChanged};
6402
6403 if (AS.isEscapedReadOnly()) {
6404 Changed |= propagateStoredValuesToLoads(AI, AS);
6405 return {Changed, CFGChanged};
6406 }
6407
6408 // Delete all the dead users of this alloca before splitting and rewriting it.
6409 for (Instruction *DeadUser : AS.getDeadUsers()) {
6410 // Free up everything used by this instruction.
6411 for (Use &DeadOp : DeadUser->operands())
6412 clobberUse(DeadOp);
6413
6414 // Now replace the uses of this instruction.
6415 DeadUser->replaceAllUsesWith(PoisonValue::get(DeadUser->getType()));
6416
6417 // And mark it for deletion.
6418 DeadInsts.push_back(DeadUser);
6419 Changed = true;
6420 }
6421 for (Use *DeadOp : AS.getDeadOperands()) {
6422 clobberUse(*DeadOp);
6423 Changed = true;
6424 }
6425
6426 // No slices to split. Leave the dead alloca for a later pass to clean up.
6427 if (AS.begin() == AS.end())
6428 return {Changed, CFGChanged};
6429
6430 Changed |= splitAlloca(AI, AS);
6431
6432 LLVM_DEBUG(dbgs() << " Speculating PHIs\n");
6433 while (!SpeculatablePHIs.empty())
6434 speculatePHINodeLoads(IRB, *SpeculatablePHIs.pop_back_val());
6435
6436 LLVM_DEBUG(dbgs() << " Rewriting stores through PHIs\n");
6437 auto RemainingPHIsWithStoreToRewrite = PHIsWithStoreToRewrite.takeVector();
6438 while (!RemainingPHIsWithStoreToRewrite.empty()) {
6439 PHINode *PN = RemainingPHIsWithStoreToRewrite.pop_back_val();
6440 auto *SI = cast<StoreInst>(PN->user_back());
6441 CFGChanged |= rewritePHINodeStore(*PN, *SI, *DTU, Worklist);
6442 }
6443
6444 LLVM_DEBUG(dbgs() << " Rewriting Selects\n");
6445 auto RemainingSelectsToRewrite = SelectsToRewrite.takeVector();
6446 while (!RemainingSelectsToRewrite.empty()) {
6447 const auto [K, V] = RemainingSelectsToRewrite.pop_back_val();
6448 CFGChanged |=
6449 rewriteSelectInstMemOps(*K, V, IRB, PreserveCFG ? nullptr : DTU);
6450 }
6451
6452 return {Changed, CFGChanged};
6453}
6454
6455/// Delete the dead instructions accumulated in this run.
6456///
6457/// Recursively deletes the dead instructions we've accumulated. This is done
6458/// at the very end to maximize locality of the recursive delete and to
6459/// minimize the problems of invalidated instruction pointers as such pointers
6460/// are used heavily in the intermediate stages of the algorithm.
6461///
6462/// We also record the alloca instructions deleted here so that they aren't
6463/// subsequently handed to mem2reg to promote.
6464bool SROA::deleteDeadInstructions(
6465 SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) {
6466 bool Changed = false;
6467 while (!DeadInsts.empty()) {
6468 Instruction *I = dyn_cast_or_null<Instruction>(DeadInsts.pop_back_val());
6469 if (!I)
6470 continue;
6471 LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
6472
6473 // If the instruction is an alloca, find the possible dbg.declare connected
6474 // to it, and remove it too. We must do this before calling RAUW or we will
6475 // not be able to find it.
6476 if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) {
6477 DeletedAllocas.insert(AI);
6478 for (DbgVariableRecord *OldDII : findDVRDeclares(AI))
6479 OldDII->eraseFromParent();
6480 }
6481
6483 I->replaceAllUsesWith(UndefValue::get(I->getType()));
6484
6485 for (Use &Operand : I->operands())
6486 if (Instruction *U = dyn_cast<Instruction>(Operand)) {
6487 // Zero out the operand and see if it becomes trivially dead.
6488 Operand = nullptr;
6490 DeadInsts.push_back(U);
6491 }
6492
6493 ++NumDeleted;
6494 I->eraseFromParent();
6495 Changed = true;
6496 }
6497 return Changed;
6498}
6499/// Promote the allocas, using the best available technique.
6500///
6501/// This attempts to promote whatever allocas have been identified as viable in
6502/// the PromotableAllocas list. If that list is empty, there is nothing to do.
6503/// This function returns whether any promotion occurred.
6504bool SROA::promoteAllocas() {
6505 if (PromotableAllocas.empty())
6506 return false;
6507
6508 if (SROASkipMem2Reg) {
6509 LLVM_DEBUG(dbgs() << "Not promoting allocas with mem2reg!\n");
6510 } else {
6511 LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
6512 NumPromoted += PromotableAllocas.size();
6513 PromoteMemToReg(PromotableAllocas.getArrayRef(), DTU->getDomTree(), AC);
6514 }
6515
6516 PromotableAllocas.clear();
6517 return true;
6518}
6519
6520std::pair<bool /*Changed*/, bool /*CFGChanged*/> SROA::runSROA(Function &F) {
6521 LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
6522
6523 const DataLayout &DL = F.getDataLayout();
6524 BasicBlock &EntryBB = F.getEntryBlock();
6525 for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(EntryBB.end());
6526 I != E; ++I) {
6527 if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) {
6528 std::optional<TypeSize> Size = AI->getAllocationSize(DL);
6529 if (Size && Size->isScalable() && isAllocaPromotable(AI))
6530 PromotableAllocas.insert(AI);
6531 else
6532 Worklist.insert(AI);
6533 }
6534 }
6535
6536 bool Changed = false;
6537 bool CFGChanged = false;
6538 // A set of deleted alloca instruction pointers which should be removed from
6539 // the list of promotable allocas.
6540 SmallPtrSet<AllocaInst *, 4> DeletedAllocas;
6541
6542 do {
6543 while (!Worklist.empty()) {
6544 auto [IterationChanged, IterationCFGChanged] =
6545 runOnAlloca(*Worklist.pop_back_val());
6546 Changed |= IterationChanged;
6547 CFGChanged |= IterationCFGChanged;
6548
6549 Changed |= deleteDeadInstructions(DeletedAllocas);
6550
6551 // Remove the deleted allocas from various lists so that we don't try to
6552 // continue processing them.
6553 if (!DeletedAllocas.empty()) {
6554 Worklist.set_subtract(DeletedAllocas);
6555 PostPromotionWorklist.set_subtract(DeletedAllocas);
6556 PromotableAllocas.set_subtract(DeletedAllocas);
6557 DeletedAllocas.clear();
6558 }
6559 }
6560
6561 Changed |= promoteAllocas();
6562
6563 Worklist = PostPromotionWorklist;
6564 PostPromotionWorklist.clear();
6565 } while (!Worklist.empty());
6566
6567 assert((!CFGChanged || Changed) && "Can not only modify the CFG.");
6568 assert((!CFGChanged || !PreserveCFG) &&
6569 "Should not have modified the CFG when told to preserve it.");
6570
6571 if (Changed && isAssignmentTrackingEnabled(*F.getParent())) {
6572 for (auto &BB : F) {
6574 }
6575 }
6576
6577 return {Changed, CFGChanged};
6578}
6579
6583 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6584 auto [Changed, CFGChanged] =
6585 SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6586 if (!Changed)
6587 return PreservedAnalyses::all();
6589 if (!CFGChanged)
6592 return PA;
6593}
6594
6596 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
6597 static_cast<PassInfoMixin<SROAPass> *>(this)->printPipeline(
6598 OS, MapClassName2PassName);
6599 OS << '<'
6600 << (Options.CFG == SROAOptions::PreserveCFG ? "preserve-cfg"
6601 : "modify-cfg");
6602 if (Options.AggregateToVector)
6603 OS << ";aggregate-to-vector";
6604 OS << '>';
6605}
6606
6607SROAPass::SROAPass(SROAOptions Options) : Options(Options) {}
6608
6609namespace {
6610
6611/// A legacy pass for the legacy pass manager that wraps the \c SROA pass.
6612class SROALegacyPass : public FunctionPass {
6614
6615public:
6616 static char ID;
6617
6619 : FunctionPass(ID), Options(Options) {
6621 }
6622
6623 bool runOnFunction(Function &F) override {
6624 if (skipFunction(F))
6625 return false;
6626
6627 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
6628 AssumptionCache &AC =
6629 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
6630 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
6631 auto [Changed, _] = SROA(&F.getContext(), &DTU, &AC, Options).runSROA(F);
6632 return Changed;
6633 }
6634
6635 void getAnalysisUsage(AnalysisUsage &AU) const override {
6636 AU.addRequired<AssumptionCacheTracker>();
6637 AU.addRequired<DominatorTreeWrapperPass>();
6638 AU.addPreserved<GlobalsAAWrapperPass>();
6639 AU.addPreserved<DominatorTreeWrapperPass>();
6640 }
6641
6642 StringRef getPassName() const override { return "SROA"; }
6643};
6644
6645} // end anonymous namespace
6646
6647char SROALegacyPass::ID = 0;
6648
6649FunctionPass *llvm::createSROAPass(bool PreserveCFG, bool AggregateToVector) {
6650 return new SROALegacyPass(SROAOptions(PreserveCFG ? SROAOptions::PreserveCFG
6652 AggregateToVector));
6653}
6654
6655INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",
6656 "Scalar Replacement Of Aggregates", false, false)
6659INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
unsigned uint64_t
Rewrite undef for PHI
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition Compiler.h:678
This file contains the declarations for the subclasses of Constant, which represent the different fla...
DXIL Forward Handle Accesses
DXIL Resource Access
This file defines the DenseMap class.
static bool runOnFunction(Function &F, bool PostInlining)
Flatten the CFG
#define DEBUG_TYPE
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
#define _
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
This defines the Use class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static LVOptions Options
Definition LVOptions.cpp:25
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
print mir2vec MIR2Vec Vocabulary Printer Pass
Definition MIR2Vec.cpp:622
This file implements a map that provides insertion order iteration.
static std::optional< AllocFnsTy > getAllocationSize(const CallBase *CB, const TargetLibraryInfo *TLI)
static std::optional< uint64_t > getSizeInBytes(std::optional< uint64_t > SizeInBits)
Memory SSA
Definition MemorySSA.cpp:73
This file contains the declarations for metadata subclasses.
#define T
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
#define P(N)
if(PassOpts->AAPipeline)
PassBuilder PB(Machine, PassOpts->PTO, std::nullopt, &PIC)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file defines the PointerIntPair class.
This file provides a collection of visitors which walk the (instruction) uses of a pointer.
const SmallVectorImpl< MachineOperand > & Cond
Remove Loads Into Fake Uses
bool isDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static void visit(BasicBlock &Start, std::function< bool(BasicBlock *)> op)
static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit, uint64_t OldAllocaOffsetInBits, uint64_t SliceSizeInBits, Instruction *OldInst, Instruction *Inst, Value *Dest, Value *Value, const DataLayout &DL)
Find linked dbg.assign and generate a new one with the correct FragmentInfo.
Definition SROA.cpp:351
static VectorType * isVectorPromotionViable(Partition &P, const DataLayout &DL, unsigned VScale)
Test whether the given alloca partitioning and range of slices can be promoted to a vector.
Definition SROA.cpp:2377
static Align getAdjustedAlignment(Instruction *I, uint64_t Offset)
Compute the adjusted alignment for a load or store from an offset.
Definition SROA.cpp:2050
static VectorType * checkVectorTypesForPromotion(Partition &P, const DataLayout &DL, SmallVectorImpl< VectorType * > &CandidateTys, bool HaveCommonEltTy, Type *CommonEltTy, bool HaveVecPtrTy, bool HaveCommonVecPtrTy, VectorType *CommonVecPtrTy, unsigned VScale)
Test whether any vector type in CandidateTys is viable for promotion.
Definition SROA.cpp:2228
static std::pair< Type *, IntegerType * > findCommonType(AllocaSlices::const_iterator B, AllocaSlices::const_iterator E, uint64_t EndOffset)
Walk the range of a partitioning looking for a common type to cover this sequence of slices.
Definition SROA.cpp:1517
static Type * stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty)
Strip aggregate type wrapping.
Definition SROA.cpp:4797
static FragCalcResult calculateFragment(DILocalVariable *Variable, uint64_t NewStorageSliceOffsetInBits, uint64_t NewStorageSliceSizeInBits, std::optional< DIExpression::FragmentInfo > StorageFragment, std::optional< DIExpression::FragmentInfo > CurrentFragment, DIExpression::FragmentInfo &Target)
Definition SROA.cpp:286
static DIExpression * createOrReplaceFragment(const DIExpression *Expr, DIExpression::FragmentInfo Frag, int64_t BitExtractOffset)
Create or replace an existing fragment in a DIExpression with Frag.
Definition SROA.cpp:5937
static Value * insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old, Value *V, uint64_t Offset, const Twine &Name)
Definition SROA.cpp:2620
static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S, VectorType *Ty, uint64_t ElementSize, const DataLayout &DL, unsigned VScale)
Test whether the given slice use can be promoted to a vector.
Definition SROA.cpp:2148
static Value * getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr, APInt Offset, Type *PointerTy, const Twine &NamePrefix)
Compute an adjusted pointer from Ptr by Offset bytes where the resulting pointer has PointerTy.
Definition SROA.cpp:2039
static bool isIntegerWideningViableForSlice(const Slice &S, uint64_t AllocBeginOffset, Type *AllocaTy, const DataLayout &DL, bool &WholeAllocaOp)
Test whether a slice of an alloca is valid for integer widening.
Definition SROA.cpp:2459
static bool rewritePHINodeStore(PHINode &PN, StoreInst &SI, DomTreeUpdater &DTU, SmallSetVector< AllocaInst *, 16 > &Worklist)
Move a store through a pointer PHI onto each of the PHI's incoming edges.
Definition SROA.cpp:1770
static Value * extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex, unsigned EndIndex, const Twine &Name)
Definition SROA.cpp:2653
static Value * foldPHINodeOrSelectInst(Instruction &I)
A helper that folds a PHI node or a select.
Definition SROA.cpp:1013
static bool rewriteSelectInstMemOps(SelectInst &SI, const RewriteableMemOps &Ops, IRBuilderTy &IRB, DomTreeUpdater *DTU)
Definition SROA.cpp:2005
static void rewriteMemOpOfSelect(SelectInst &SI, T &I, SelectHandSpeculativity Spec, DomTreeUpdater &DTU)
Definition SROA.cpp:1938
static Slice * findOverlappingCopySlice(Slice &S, InstructionSliceMap &SliceMap)
Try to find a slice in the map that partially overlaps with S, i.e.
Definition SROA.cpp:4948
static Value * foldSelectInst(SelectInst &SI)
Definition SROA.cpp:1000
bool isKillAddress(const DbgVariableRecord *DVR)
Definition SROA.cpp:5900
static Value * insertVector(IRBuilderTy &IRB, Value *Old, Value *V, unsigned BeginIndex, const Twine &Name)
Definition SROA.cpp:2674
static bool isIntegerWideningViable(Partition &P, Type *AllocaTy, const DataLayout &DL)
Test whether the given alloca partition's integer operations can be widened to promotable ones.
Definition SROA.cpp:2554
static void speculatePHINodeLoads(IRBuilderTy &IRB, PHINode &PN)
Definition SROA.cpp:1715
static VectorType * createAndCheckVectorTypesForPromotion(SetVector< Type * > &OtherTys, ArrayRef< VectorType * > CandidateTysCopy, function_ref< void(Type *)> CheckCandidateType, Partition &P, const DataLayout &DL, SmallVectorImpl< VectorType * > &CandidateTys, bool &HaveCommonEltTy, Type *&CommonEltTy, bool &HaveVecPtrTy, bool &HaveCommonVecPtrTy, VectorType *&CommonVecPtrTy, unsigned VScale)
Definition SROA.cpp:2333
static DebugVariable getAggregateVariable(DbgVariableRecord *DVR)
Definition SROA.cpp:332
static std::tuple< Type *, bool, VectorType * > selectPartitionType(Partition &P, const DataLayout &DL, AllocaInst &AI, LLVMContext &C, bool AggregateToVector)
Select a partition type for an alloca partition.
Definition SROA.cpp:5617
static bool isSafePHIToSpeculate(PHINode &PN)
PHI instructions that use an alloca and are subsequently loaded can be rewritten to load both input p...
Definition SROA.cpp:1586
static FixedVectorType * tryCanonicalizeStructToVector(StructType *STy, Partition &P, const DataLayout &DL)
Try to canonicalize a homogeneous struct partition to a vector type.
Definition SROA.cpp:5550
static Value * extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V, IntegerType *Ty, uint64_t Offset, const Twine &Name)
Definition SROA.cpp:2595
static void insertNewDbgInst(DIBuilder &DIB, DbgVariableRecord *Orig, AllocaInst *NewAddr, DIExpression *NewAddrExpr, Instruction *BeforeInst, std::optional< DIExpression::FragmentInfo > NewFragment, int64_t BitExtractAdjustment)
Insert a new DbgRecord.
Definition SROA.cpp:6001
static void speculateSelectInstLoads(SelectInst &SI, LoadInst &LI, IRBuilderTy &IRB)
Definition SROA.cpp:1900
static Value * mergeTwoVectors(Value *V0, Value *V1, const DataLayout &DL, Type *NewAIEltTy, IRBuilder<> &Builder)
This function takes two vector values and combines them into a single vector by concatenating their e...
Definition SROA.cpp:2745
const DIExpression * getAddressExpression(const DbgVariableRecord *DVR)
Definition SROA.cpp:5906
static Type * getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset, uint64_t Size)
Try to find a partition of the aggregate type passed in for a given offset and size.
Definition SROA.cpp:4835
static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy, unsigned VScale=0)
Test whether we can convert a value from the old to the new type.
Definition SROA.cpp:2060
static SelectHandSpeculativity isSafeLoadOfSelectToSpeculate(LoadInst &LI, SelectInst &SI, bool PreserveCFG)
Definition SROA.cpp:1841
static StoreInst * getPHIStoreToRewrite(PHINode &PN, bool PreserveCFG, DominatorTree &DT)
Check whether a single store through PN can be moved onto each incoming edge.
Definition SROA.cpp:1662
static Type * findCommonTypeThroughPHIOrSelect(Instruction &I)
Find a common load/store type used through a pointer PHI or select.
Definition SROA.cpp:1493
This file provides the interface for LLVM's Scalar Replacement of Aggregates pass.
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the SparseBitVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:119
static SymbolRef::Type getType(const Symbol *Sym)
Definition TapiFile.cpp:39
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type.
Virtual Register Rewriter
Value * RHS
Value * LHS
Builder for the alloca slices.
Definition SROA.cpp:1025
SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS)
Definition SROA.cpp:1041
An iterator over partitions of the alloca's slices.
Definition SROA.cpp:813
bool operator==(const partition_iterator &RHS) const
Definition SROA.cpp:960
partition_iterator & operator++()
Definition SROA.cpp:980
bool shouldDelete(Instruction *I) const override
Return false if a sub-class wants to keep one of the loads/stores after the SSA construction.
Definition SROA.cpp:6278
BasicLoadAndStorePromoter(ArrayRef< const Instruction * > Insts, SSAUpdater &S, Type *ZeroType)
Definition SROA.cpp:6275
Value * getValueToUseForAlloca(Instruction *I) const override
Return the value to use for the point in the code that the alloca is positioned.
Definition SROA.cpp:6282
Class for arbitrary precision integers.
Definition APInt.h:78
an instruction to allocate memory on the stack
LLVM_ABI bool isStaticAlloca() const
Return true if this alloca is in the entry block of the function and is a constant size.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
PointerType * getType() const
Overload to return most specific pointer type.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
LLVM_ABI std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
iterator end() const
Definition ArrayRef.h:130
size_t size() const
Get the array size.
Definition ArrayRef.h:141
iterator begin() const
Definition ArrayRef.h:129
static LLVM_ABI ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
A function analysis which provides an AssumptionCache.
An immutable pass that tracks lazily created AssumptionCache objects.
A cache of @llvm.assume calls within a function.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator end()
Definition BasicBlock.h:459
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:446
LLVM_ABI InstListType::const_iterator getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
LLVM_ABI bool canSplitPredecessors() const
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
LLVM_ABI CaptureInfo getCaptureInfo(unsigned OpNo) const
Return which pointer components this operand may capture.
bool onlyReadsMemory(unsigned OpNo) const
bool isDataOperand(const Use *U) const
This is the shared class of boolean and integer constants.
Definition Constants.h:87
static LLVM_ABI Constant * getAllOnesValue(Type *Ty)
static DIAssignID * getDistinct(LLVMContext &Context)
LLVM_ABI DbgRecord * insertDbgAssign(Instruction *LinkedInstr, Value *Val, DILocalVariable *SrcVar, DIExpression *ValExpr, Value *Addr, DIExpression *AddrExpr, const DILocation *DL)
Insert a new dbg_assign record.
DWARF expression.
iterator_range< expr_op_iterator > expr_ops() const
DbgVariableFragmentInfo FragmentInfo
LLVM_ABI bool startsWithDeref() const
Return whether the first element a DW_OP_deref.
static LLVM_ABI bool calculateFragmentIntersect(const DataLayout &DL, const Value *SliceStart, uint64_t SliceOffsetInBits, uint64_t SliceSizeInBits, const Value *DbgPtr, int64_t DbgPtrOffsetInBits, int64_t DbgExtractOffsetInBits, DIExpression::FragmentInfo VarFrag, std::optional< DIExpression::FragmentInfo > &Result, int64_t &OffsetFromLocationInBits)
Computes a fragment, bit-extract operation if needed, and new constant offset to describe a part of a...
static LLVM_ABI std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
static LLVM_ABI DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
LLVM_ABI void moveBefore(DbgRecord *MoveBefore)
DebugLoc getDebugLoc() const
void setDebugLoc(DebugLoc Loc)
Record of a variable value-assignment, aka a non instruction representation of the dbg....
LLVM_ABI void setKillAddress()
Kill the address component.
LLVM_ABI bool isKillLocation() const
LLVM_ABI bool isKillAddress() const
Check whether this kills the address component.
LLVM_ABI void replaceVariableLocationOp(Value *OldValue, Value *NewValue, bool AllowEmpty=false)
Value * getValue(unsigned OpIdx=0) const
static LLVM_ABI DbgVariableRecord * createLinkedDVRAssign(Instruction *LinkedInstr, Value *Val, DILocalVariable *Variable, DIExpression *Expression, Value *Address, DIExpression *AddressExpression, const DILocation *DI)
LLVM_ABI void setAssignId(DIAssignID *New)
DIExpression * getExpression() const
static LLVM_ABI DbgVariableRecord * createDVRDeclare(Value *Address, DILocalVariable *DV, DIExpression *Expr, const DILocation *DI)
static LLVM_ABI DbgVariableRecord * createDbgVariableRecord(Value *Location, DILocalVariable *DV, DIExpression *Expr, const DILocation *DI)
DILocalVariable * getVariable() const
DIExpression * getAddressExpression() const
LLVM_ABI DILocation * getInlinedAt() const
Definition DebugLoc.cpp:58
Identifies a unique instance of a variable.
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
Definition DenseMap.h:285
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:258
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition DenseMap.h:254
iterator end()
Definition DenseMap.h:176
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:319
Analysis pass which computes a DominatorTree.
Definition Dominators.h:241
Legacy analysis pass which computes a DominatorTree.
Definition Dominators.h:277
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Class to represent fixed width SIMD vectors.
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:843
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:314
unsigned getVScaleValue() const
Return the value for vscale based on the vscale_range attribute or 0 when unknown.
const BasicBlock & getEntryBlock() const
Definition Function.h:794
LLVM_ABI bool accumulateConstantOffset(const DataLayout &DL, APInt &Offset, function_ref< bool(Value &, APInt &)> ExternalAnalysis=nullptr) const
Accumulate the constant address offset of this GEP if possible.
Definition Operator.cpp:130
iterator_range< op_iterator > indices()
Type * getSourceElementType() const
LLVM_ABI GEPNoWrapFlags getNoWrapFlags() const
Get the nowrap flags for the GEP instruction.
This provides the default implementation of the IRBuilder 'InsertHelper' method that is called whenev...
Definition IRBuilder.h:61
virtual void InsertHelper(Instruction *I, const Twine &Name, BasicBlock::iterator InsertPt) const
Definition IRBuilder.h:65
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2903
Base class for instruction visitors.
Definition InstVisitor.h:78
LLVM_ABI unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
bool hasMetadata() const
Return true if this instruction has any metadata attached to it.
LLVM_ABI bool isAtomic() const LLVM_READONLY
Return true if this instruction has an AtomicOrdering of unordered or higher.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Instruction * user_back()
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
LLVM_ABI bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
iterator_range< user_iterator > users()
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
user_iterator user_begin()
LLVM_ABI AAMDNodes getAAMetadata() const
Returns the AA metadata for this instruction.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
LLVM_ABI void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
@ MAX_INT_BITS
Maximum number of bits that can be specified.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
LLVM_ABI LoadAndStorePromoter(ArrayRef< const Instruction * > Insts, SSAUpdater &S, StringRef Name=StringRef())
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
void setAlignment(Align Align)
Value * getPointerOperand()
bool isVolatile() const
Return true if this is a load from a volatile memory location.
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this load instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Type * getPointerOperandType() const
static unsigned getPointerOperandIndex()
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this load instruction.
bool isSimple() const
Align getAlign() const
Return the alignment of the access that is being performed.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition Metadata.h:1579
LLVMContext & getContext() const
Definition Metadata.h:1245
This is the common base class for memset/memcpy/memmove.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
op_range incoming_values()
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
PointerIntPair - This class implements a pair of a pointer and small integer.
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:132
PtrUseVisitor(const DataLayout &DL)
LLVM_ABI SROAPass(SROAOptions Options)
If PreserveCFG is set, then the pass is not allowed to modify CFG in any way, even if it would update...
Definition SROA.cpp:6607
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Run the pass over the function.
Definition SROA.cpp:6580
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
Definition SROA.cpp:6595
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition SSAUpdater.h:39
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
void clear()
Completely clear the SetVector.
Definition SetVector.h:273
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
Definition SetVector.h:345
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
typename SuperClass::const_iterator const_iterator
typename SuperClass::iterator iterator
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
void set(unsigned Idx)
bool test(unsigned Idx) const
void reset(unsigned Idx)
An instruction for storing to memory.
void setAlignment(Align Align)
Value * getValueOperand()
static unsigned getPointerOperandIndex()
Value * getPointerOperand()
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this store instruction.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
static constexpr size_t npos
Definition StringRef.h:58
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition StringRef.h:597
size_t rfind(char C, size_t From=npos) const
Search for the last character C in the string.
Definition StringRef.h:365
size_t find(char C, size_t From=0) const
Search for the first character C in the string.
Definition StringRef.h:290
LLVM_ABI size_t find_first_not_of(char C, size_t From=0) const
Find the first character in the string that is not C or npos if not found.
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition DataLayout.h:743
TypeSize getSizeInBytes() const
Definition DataLayout.h:752
LLVM_ABI unsigned getElementContainingOffset(uint64_t FixedOffset) const
Given a valid byte offset into the structure, returns the structure index that contains it.
TypeSize getElementOffset(unsigned Idx) const
Definition DataLayout.h:774
TypeSize getSizeInBits() const
Definition DataLayout.h:754
Class to represent struct types.
static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition Type.cpp:467
element_iterator element_end() const
ArrayRef< Type * > elements() const
element_iterator element_begin() const
bool isPacked() const
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
Type::subtype_iterator element_iterator
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:339
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI unsigned getIntegerBitWidth() const
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:277
LLVM_ABI unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
Definition Type.h:306
static LLVM_ABI IntegerType * getInt8Ty(LLVMContext &C)
Definition Type.cpp:297
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition Type.h:363
bool isStructTy() const
True if this is an instance of StructType.
Definition Type.h:271
bool isTargetExtTy() const
Return true if this is a target extension type.
Definition Type.h:205
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition Type.h:130
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
Definition Type.h:186
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition Type.h:280
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:303
static LLVM_ABI UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
op_range operands()
Definition User.h:267
op_iterator op_begin()
Definition User.h:259
const Use & getOperandUse(unsigned i) const
Definition User.h:220
Value * getOperand(unsigned i) const
Definition User.h:207
op_iterator op_end()
Definition User.h:261
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI bool hasOneUser() const
Return true if there is exactly one user of this value.
Definition Value.cpp:163
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:553
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
LLVM_ABI const Value * stripInBoundsOffsets(function_ref< void(const Value *)> Func=[](const Value *) {}) const
Strip off pointer casts and inbounds GEPs.
Definition Value.cpp:828
LLVM_ABI void dropDroppableUsesIn(User &Usr)
Remove every use of this value in User that can safely be removed.
Definition Value.cpp:215
LLVM_ABI const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr, bool LookThroughIntToPtr=false) const
Accumulate the constant offset this value has compared to a base pointer.
bool use_empty() const
Definition Value.h:348
iterator_range< use_iterator > uses()
Definition Value.h:382
bool hasName() const
Definition Value.h:263
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:319
LLVM_ABI void takeName(Value *V)
Transfer the name from V to this value.
Definition Value.cpp:400
static LLVM_ABI VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
static VectorType * getWithSizeAndScalar(VectorType *SizeTy, Type *EltTy)
This static method attempts to construct a VectorType with the same size-in-bits as SizeTy but with a...
static LLVM_ABI bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:200
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition TypeSize.h:168
constexpr bool isFixed() const
Returns true if the quantity is not scaled by vscale.
Definition TypeSize.h:171
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:165
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition ilist_node.h:34
self_iterator getIterator()
Definition ilist_node.h:123
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition ilist_node.h:348
CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...
Definition iterator.h:80
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
Changed
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:81
Offsets
Offsets in bytes from the start of the input buffer.
SmallVector< DbgVariableRecord * > getDVRAssignmentMarkers(const Instruction *Inst)
Return a range of dbg_assign records for which Inst performs the assignment they encode.
Definition DebugInfo.h:212
LLVM_ABI void deleteAssignmentMarkers(const Instruction *Inst)
Delete the llvm.dbg.assign intrinsics linked to Inst.
initializer< Ty > init(const Ty &Val)
@ DW_OP_LLVM_fragment
Only used in LLVM metadata.
Definition Dwarf.h:144
@ User
could "use" a pointer
NodeAddr< PhiNode * > Phi
Definition RDFGraph.h:390
NodeAddr< UseNode * > Use
Definition RDFGraph.h:385
bool empty() const
Definition BasicBlock.h:101
iterator end() const
Definition BasicBlock.h:89
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
LLVM_ABI iterator begin() const
unsigned getNumElements(Type *Ty)
Definition SLPUtils.cpp:86
This is an optimization pass for GlobalISel generic memory operations.
static cl::opt< bool > SROASkipMem2Reg("sroa-skip-mem2reg", cl::init(false), cl::Hidden)
Disable running mem2reg during SROA in order to test or debug SROA.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
@ Offset
Definition DWP.cpp:577
@ Length
Definition DWP.cpp:577
bool operator<(int64_t V1, const APSInt &V2)
Definition APSInt.h:360
void stable_sort(R &&Range)
Definition STLExtras.h:2132
LLVM_ABI bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1748
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
LLVM_ABI void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
LLVM_ABI bool isAssumeLikeIntrinsic(const Instruction *I)
Return true if it is an intrinsic that cannot be speculated but also cannot trap.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
auto successors(const MachineBasicBlock *BB)
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool operator!=(uint64_t V1, const APInt &V2)
Definition APInt.h:2139
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
LLVM_ABI std::optional< RegOrConstant > getVectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition Utils.cpp:1447
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
void * PointerTy
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
auto unique(Range &&R, Predicate P)
Definition STLExtras.h:2150
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
LLVM_ABI bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition STLExtras.h:2216
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition Local.cpp:402
bool capturesFullProvenance(CaptureComponents CC)
Definition ModRef.h:396
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
Definition STLExtras.h:1652
LLVM_ABI void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void initializeSROALegacyPassPass(PassRegistry &)
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
LLVM_ABI TinyPtrVector< DbgVariableRecord * > findDVRValues(Value *V)
As above, for DVRValues.
Definition DebugInfo.cpp:82
LLVM_ABI void llvm_unreachable_internal(const char *msg=nullptr, const char *file=nullptr, unsigned line=0)
This function calls abort(), and prints the optional message to stderr.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
constexpr int PoisonMaskElem
iterator_range(Container &&) -> iterator_range< llvm::detail::IterOfRange< Container > >
LLVM_ABI BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
IRBuilder(LLVMContext &, FolderTy, InserterTy, MDNode *, ArrayRef< OperandBundleDef >) -> IRBuilder< FolderTy, InserterTy >
LLVM_ABI bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
DWARFExpression::Operation Op
LLVM_ABI Align tryEnforceAlignment(Value *V, Align PrefAlign, const DataLayout &DL)
If the specified pointer points to an object that we control, try to modify the object's alignment to...
Definition Local.cpp:1509
LLVM_ABI FunctionPass * createSROAPass(bool PreserveCFG=true, bool AggregateToVector=false)
Definition SROA.cpp:6649
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
LLVM_ABI TinyPtrVector< DbgVariableRecord * > findDVRDeclares(Value *V)
Finds dbg.declare records declaring local variables as living in the memory that 'V' points to.
Definition DebugInfo.cpp:48
LLVM_ABI bool isSafeToLoadUnconditionally(Value *V, Align Alignment, const APInt &Size, const SimplifyQuery &SQ)
Return true if we know that executing a load from this value cannot trap.
Definition Loads.cpp:456
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:201
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
Definition STLExtras.h:2182
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
LLVM_ABI llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:880
#define NDEBUG
Definition regutils.h:48
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition Metadata.h:774
AAMDNodes shift(size_t Offset) const
Create a new AAMDNode that describes this AAMDNode after applying a constant offset to the start of t...
Definition Metadata.h:833
LLVM_ABI AAMDNodes adjustForAccess(unsigned AccessSize)
Create a new AAMDNode for accessing AccessSize bytes of this AAMDNode.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Describes an element of a Bitfield.
Definition Bitfields.h:176
static Bitfield::Type get(StorageType Packed)
Unpacks the field from the Packed value.
Definition Bitfields.h:207
static void set(StorageType &Packed, typename Bitfield::Type Value)
Sets the typed value in the provided Packed value.
Definition Bitfields.h:223