LLVM 24.0.0git
LoadStoreOpt.cpp
Go to the documentation of this file.
1//===- LoadStoreOpt.cpp ----------- Generic memory optimizations -*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the LoadStoreOpt optimization pass.
10//===----------------------------------------------------------------------===//
11
13#include "llvm/ADT/STLExtras.h"
15#include "llvm/ADT/Statistic.h"
35#include "llvm/IR/Analysis.h"
39#include "llvm/Support/Debug.h"
41#include <algorithm>
42
43#define DEBUG_TYPE "load-store-opt"
44
45using namespace llvm;
46using namespace llvm::GISelAddressing;
47using namespace ore;
48using namespace MIPatternMatch;
49
50STATISTIC(NumStoresMerged, "Number of stores merged");
51
52const unsigned MaxStoreSizeToForm = 128;
53
54namespace {
55
56class LoadStoreOptImpl {
57 MachineRegisterInfo *MRI = nullptr;
58 const TargetLowering *TLI = nullptr;
59 MachineFunction *MF = nullptr;
60 AliasAnalysis *AA = nullptr;
61 const LegalizerInfo *LI = nullptr;
62
63 MachineIRBuilder Builder;
64
65 /// Initialize the field members using \p MF.
66 void init(MachineFunction &MF, function_ref<AliasAnalysis *()> GetAA);
67
68 class StoreMergeCandidate {
69 public:
70 // The base pointer used as the base for all stores in this candidate.
71 Register BasePtr;
72 // Our algorithm is very simple at the moment. We assume that in instruction
73 // order stores are writing to incremeneting consecutive addresses. So when
74 // we walk the block in reverse order, the next eligible store must write to
75 // an offset one store width lower than CurrentLowestOffset.
76 int64_t CurrentLowestOffset;
78 // A vector of MachineInstr/unsigned pairs to denote potential aliases that
79 // need to be checked before the candidate is considered safe to merge. The
80 // unsigned value is an index into the Stores vector. The indexed store is
81 // the highest-indexed store that has already been checked to not have an
82 // alias with the instruction. We record this so we don't have to repeat
83 // alias checks that have been already done, only those with stores added
84 // after the potential alias is recorded.
86
87 void addPotentialAlias(MachineInstr &MI);
88
89 /// Reset this candidate back to an empty one.
90 void reset() {
91 Stores.clear();
92 PotentialAliases.clear();
93 CurrentLowestOffset = 0;
94 BasePtr = Register();
95 }
96 };
97
98 bool isLegalOrBeforeLegalizer(const LegalityQuery &Query,
99 MachineFunction &MF) const;
100 /// If the given store is valid to be a member of the candidate, add it and
101 /// return true. Otherwise, returns false.
102 bool addStoreToCandidate(GStore &MI, StoreMergeCandidate &C);
103 /// Returns true if the instruction \p MI would potentially alias with any
104 /// stores in the candidate \p C.
105 bool operationAliasesWithCandidate(MachineInstr &MI, StoreMergeCandidate &C);
106 /// Merges the stores in the given vector into a wide store.
107 /// \p returns true if at least some of the stores were merged.
108 /// This may decide not to merge stores if heuristics predict it will not be
109 /// worth it.
110 bool mergeStores(SmallVectorImpl<GStore *> &StoresToMerge);
111 /// Perform a merge of all the stores in \p Stores into a single store.
112 /// Erases the old stores from the block when finished.
113 /// \returns true if merging was done. It may fail to perform a merge if
114 /// there are issues with materializing legal wide values.
115 bool doSingleStoreMerge(SmallVectorImpl<GStore *> &Stores);
116 bool processMergeCandidate(StoreMergeCandidate &C);
117 bool mergeBlockStores(MachineBasicBlock &MBB);
118 bool mergeFunctionStores(MachineFunction &MF);
119
120 bool mergeTruncStore(GStore &StoreMI,
121 SmallPtrSetImpl<GStore *> &DeletedStores);
122 bool mergeTruncStoresBlock(MachineBasicBlock &MBB);
123
124 /// Initialize some target-specific data structures for the store merging
125 /// optimization. \p AddrSpace indicates which address space to use when
126 /// probing the legalizer info for legal stores.
127 void initializeStoreMergeTargetInfo(unsigned AddrSpace = 0);
128 /// A map between address space numbers and a bitvector of supported stores
129 /// sizes. Each bit in the bitvector represents whether a store size of
130 /// that bit's value is legal. E.g. if bit 64 is set, then 64 bit scalar
131 /// stores are legal.
132 DenseMap<unsigned, BitVector> LegalStoreSizes;
133 bool IsPreLegalizer = false;
134 /// Contains instructions to be erased at the end of a block scan.
136
137public:
138 bool runOnMachineFunction(MachineFunction &MF,
139 function_ref<AliasAnalysis *()> GetAA);
140};
141
142} // namespace
143
146 "Generic memory optimizations", false, false)
148 "Generic memory optimizations", false, false)
149
151
152void LoadStoreOptImpl::init(MachineFunction &MF,
153 function_ref<AliasAnalysis *()> GetAA) {
154 this->MF = &MF;
155 MRI = &MF.getRegInfo();
156 AA = GetAA();
158 LI = MF.getSubtarget().getLegalizerInfo();
159 Builder.setMF(MF);
160 IsPreLegalizer = !MF.getProperties().hasLegalized();
161 InstsToErase.clear();
162}
163
170
173 BaseIndexOffset Info;
174 Register PtrAddRHS;
175 Register BaseReg;
176 if (!mi_match(Ptr, MRI, m_GPtrAdd(m_Reg(BaseReg), m_Reg(PtrAddRHS)))) {
177 Info.setBase(Ptr);
178 Info.setOffset(0);
179 return Info;
180 }
181 Info.setBase(BaseReg);
182 auto RHSCst = getIConstantVRegValWithLookThrough(PtrAddRHS, MRI);
183 if (RHSCst)
184 Info.setOffset(RHSCst->Value.getSExtValue());
185
186 // Just recognize a simple case for now. In future we'll need to match
187 // indexing patterns for base + index + constant.
188 Info.setIndex(PtrAddRHS);
189 return Info;
190}
191
193 const MachineInstr &MI2,
194 bool &IsAlias,
196 auto *LdSt1 = dyn_cast<GLoadStore>(&MI1);
197 auto *LdSt2 = dyn_cast<GLoadStore>(&MI2);
198 if (!LdSt1 || !LdSt2)
199 return false;
200
201 BaseIndexOffset BasePtr0 = getPointerInfo(LdSt1->getPointerReg(), MRI);
202 BaseIndexOffset BasePtr1 = getPointerInfo(LdSt2->getPointerReg(), MRI);
203
204 if (!BasePtr0.getBase().isValid() || !BasePtr1.getBase().isValid())
205 return false;
206
207 LocationSize Size1 = LdSt1->getMemSize();
208 LocationSize Size2 = LdSt2->getMemSize();
209
210 int64_t PtrDiff;
211 if (BasePtr0.getBase() == BasePtr1.getBase() && BasePtr0.hasValidOffset() &&
212 BasePtr1.hasValidOffset()) {
213 PtrDiff = BasePtr1.getOffset() - BasePtr0.getOffset();
214 // If the size of memory access is unknown, do not use it to do analysis.
215 // One example of unknown size memory access is to load/store scalable
216 // vector objects on the stack.
217 // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the
218 // following situations arise:
219 if (PtrDiff >= 0 && Size1.hasValue() && !Size1.isScalable()) {
220 // [----BasePtr0----]
221 // [---BasePtr1--]
222 // ========PtrDiff========>
223 IsAlias = !((int64_t)Size1.getValue() <= PtrDiff);
224 return true;
225 }
226 if (PtrDiff < 0 && Size2.hasValue() && !Size2.isScalable()) {
227 // [----BasePtr0----]
228 // [---BasePtr1--]
229 // =====(-PtrDiff)====>
230 IsAlias = !((PtrDiff + (int64_t)Size2.getValue()) <= 0);
231 return true;
232 }
233 return false;
234 }
235
236 // If both BasePtr0 and BasePtr1 are FrameIndexes, we will not be
237 // able to calculate their relative offset if at least one arises
238 // from an alloca. However, these allocas cannot overlap and we
239 // can infer there is no alias.
240 auto *Base0Def = getDefIgnoringCopies(BasePtr0.getBase(), MRI);
241 auto *Base1Def = getDefIgnoringCopies(BasePtr1.getBase(), MRI);
242 if (!Base0Def || !Base1Def)
243 return false; // Couldn't tell anything.
244
245
246 if (Base0Def->getOpcode() != Base1Def->getOpcode())
247 return false;
248
249 if (Base0Def->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
250 MachineFrameInfo &MFI = Base0Def->getMF()->getFrameInfo();
251 // If the bases have the same frame index but we couldn't find a
252 // constant offset, (indices are different) be conservative.
253 if (Base0Def != Base1Def &&
254 (!MFI.isFixedObjectIndex(Base0Def->getOperand(1).getIndex()) ||
255 !MFI.isFixedObjectIndex(Base1Def->getOperand(1).getIndex()))) {
256 IsAlias = false;
257 return true;
258 }
259 }
260
261 // This implementation is a lot more primitive than the SDAG one for now.
262 // FIXME: what about constant pools?
263 if (Base0Def->getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
264 auto GV0 = Base0Def->getOperand(1).getGlobal();
265 auto GV1 = Base1Def->getOperand(1).getGlobal();
266 if (GV0 != GV1) {
267 IsAlias = false;
268 return true;
269 }
270 }
271
272 // Can't tell anything about aliasing.
273 return false;
274}
275
277 const MachineInstr &Other,
279 AliasAnalysis *AA) {
280 struct MemUseCharacteristics {
281 bool IsVolatile;
282 bool IsAtomic;
283 Register BasePtr;
284 int64_t Offset;
285 LocationSize NumBytes;
287 };
288
289 auto getCharacteristics =
290 [&](const MachineInstr *MI) -> MemUseCharacteristics {
291 if (const auto *LS = dyn_cast<GLoadStore>(MI)) {
292 Register BaseReg;
293 int64_t Offset = 0;
294 // No pre/post-inc addressing modes are considered here, unlike in SDAG.
295 if (!mi_match(LS->getPointerReg(), MRI,
296 m_GPtrAdd(m_Reg(BaseReg), m_ICst(Offset)))) {
297 BaseReg = LS->getPointerReg();
298 Offset = 0;
299 }
300
301 LocationSize Size = LS->getMMO().getSize();
302 return {LS->isVolatile(), LS->isAtomic(), BaseReg,
303 Offset /*base offset*/, Size, &LS->getMMO()};
304 }
305 // FIXME: support recognizing lifetime instructions.
306 // Default.
307 return {false /*isvolatile*/,
308 /*isAtomic*/ false,
309 Register(),
310 (int64_t)0 /*offset*/,
312 (MachineMemOperand *)nullptr};
313 };
314 MemUseCharacteristics MUC0 = getCharacteristics(&MI),
315 MUC1 = getCharacteristics(&Other);
316
317 // If they are to the same address, then they must be aliases.
318 if (MUC0.BasePtr.isValid() && MUC0.BasePtr == MUC1.BasePtr &&
319 MUC0.Offset == MUC1.Offset)
320 return true;
321
322 // If they are both volatile then they cannot be reordered.
323 if (MUC0.IsVolatile && MUC1.IsVolatile)
324 return true;
325
326 // Be conservative about atomics for the moment
327 // TODO: This is way overconservative for unordered atomics (see D66309)
328 if (MUC0.IsAtomic && MUC1.IsAtomic)
329 return true;
330
331 // If one operation reads from invariant memory, and the other may store, they
332 // cannot alias.
333 if (MUC0.MMO && MUC1.MMO) {
334 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
335 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
336 return false;
337 }
338
339 // If NumBytes is scalable and offset is not 0, conservatively return may
340 // alias
341 if ((MUC0.NumBytes.isScalable() && MUC0.Offset != 0) ||
342 (MUC1.NumBytes.isScalable() && MUC1.Offset != 0))
343 return true;
344
345 const bool BothNotScalable =
346 !MUC0.NumBytes.isScalable() && !MUC1.NumBytes.isScalable();
347
348 // Try to prove that there is aliasing, or that there is no aliasing. Either
349 // way, we can return now. If nothing can be proved, proceed with more tests.
350 bool IsAlias;
351 if (BothNotScalable &&
353 return IsAlias;
354
355 // The following all rely on MMO0 and MMO1 being valid.
356 if (!MUC0.MMO || !MUC1.MMO)
357 return true;
358
359 // FIXME: port the alignment based alias analysis from SDAG's isAlias().
360 int64_t SrcValOffset0 = MUC0.MMO->getOffset();
361 int64_t SrcValOffset1 = MUC1.MMO->getOffset();
362 LocationSize Size0 = MUC0.NumBytes;
363 LocationSize Size1 = MUC1.NumBytes;
364 if (AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && Size0.hasValue() &&
365 Size1.hasValue()) {
366 // Use alias analysis information.
367 int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
368 int64_t Overlap0 =
369 Size0.getValue().getKnownMinValue() + SrcValOffset0 - MinOffset;
370 int64_t Overlap1 =
371 Size1.getValue().getKnownMinValue() + SrcValOffset1 - MinOffset;
372 LocationSize Loc0 =
373 Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0);
374 LocationSize Loc1 =
375 Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1);
376
377 if (AA->isNoAlias(
378 MemoryLocation(MUC0.MMO->getValue(), Loc0, MUC0.MMO->getAAInfo()),
379 MemoryLocation(MUC1.MMO->getValue(), Loc1, MUC1.MMO->getAAInfo())))
380 return false;
381 }
382
383 // Otherwise we have to assume they alias.
384 return true;
385}
386
387/// Returns true if the instruction creates an unavoidable hazard that
388/// forces a boundary between store merge candidates.
390 return MI.hasUnmodeledSideEffects() || MI.hasOrderedMemoryRef();
391}
392
393bool LoadStoreOptImpl::mergeStores(SmallVectorImpl<GStore *> &StoresToMerge) {
394 // Try to merge all the stores in the vector, splitting into separate segments
395 // as necessary.
396 assert(StoresToMerge.size() > 1 && "Expected multiple stores to merge");
397 LLT OrigTy = MRI->getType(StoresToMerge[0]->getValueReg());
398 LLT PtrTy = MRI->getType(StoresToMerge[0]->getPointerReg());
399 unsigned AS = PtrTy.getAddressSpace();
400 // Ensure the legal store info is computed for this address space.
401 initializeStoreMergeTargetInfo(AS);
402 const auto &LegalSizes = LegalStoreSizes[AS];
403
404 // FIXME: Support mismatching types (i16 + f16).
405 for (auto *StoreMI : StoresToMerge)
406 if (MRI->getType(StoreMI->getValueReg()) != OrigTy)
407 return false;
408
409 bool AnyMerged = false;
410 do {
411 unsigned NumPow2 = llvm::bit_floor(StoresToMerge.size());
412 unsigned MaxSizeBits = NumPow2 * OrigTy.getSizeInBits().getFixedValue();
413 // Compute the biggest store we can generate to handle the number of stores.
414 unsigned MergeSizeBits;
415 for (MergeSizeBits = MaxSizeBits; MergeSizeBits > 1; MergeSizeBits /= 2) {
416 LLT StoreTy = LLT::scalar(MergeSizeBits);
417 EVT StoreEVT =
419 if (LegalSizes.size() > MergeSizeBits && LegalSizes[MergeSizeBits] &&
420 TLI->canMergeStoresTo(AS, StoreEVT, *MF) &&
421 (TLI->isTypeLegal(StoreEVT)))
422 break; // We can generate a MergeSize bits store.
423 }
424 if (MergeSizeBits <= OrigTy.getSizeInBits())
425 return AnyMerged; // No greater merge.
426
427 unsigned NumStoresToMerge = MergeSizeBits / OrigTy.getSizeInBits();
428 // Perform the actual merging.
429 SmallVector<GStore *, 8> SingleMergeStores(
430 StoresToMerge.begin(), StoresToMerge.begin() + NumStoresToMerge);
431 AnyMerged |= doSingleStoreMerge(SingleMergeStores);
432 StoresToMerge.erase(StoresToMerge.begin(),
433 StoresToMerge.begin() + NumStoresToMerge);
434 } while (StoresToMerge.size() > 1);
435 return AnyMerged;
436}
437
438bool LoadStoreOptImpl::isLegalOrBeforeLegalizer(const LegalityQuery &Query,
439 MachineFunction &MF) const {
440 auto Action = LI->getAction(Query).Action;
441 // If the instruction is unsupported, it can't be legalized at all.
442 if (Action == LegalizeActions::Unsupported)
443 return false;
444 return IsPreLegalizer || Action == LegalizeAction::Legal;
445}
446
447bool LoadStoreOptImpl::doSingleStoreMerge(SmallVectorImpl<GStore *> &Stores) {
448 assert(Stores.size() > 1);
449 // We know that all the stores are consecutive and there are no aliasing
450 // operations in the range. However, the values that are being stored may be
451 // generated anywhere before each store. To ensure we have the values
452 // available, we materialize the wide value and new store at the place of the
453 // final store in the merge sequence.
454 GStore *FirstStore = Stores[0];
455 const unsigned NumStores = Stores.size();
456 LLT SmallTy = MRI->getType(FirstStore->getValueReg());
457 LLT WideValueTy =
458 LLT::integer(NumStores * SmallTy.getSizeInBits().getFixedValue());
459
460 // For each store, compute pairwise merged debug locs.
461 DebugLoc MergedLoc = Stores.front()->getDebugLoc();
462 for (auto *Store : drop_begin(Stores))
463 MergedLoc = DebugLoc::getMergedLocation(MergedLoc, Store->getDebugLoc());
464
465 Builder.setInstr(*Stores.back());
466 Builder.setDebugLoc(MergedLoc);
467
468 // If all of the store values are constants, then create a wide constant
469 // directly. Otherwise, we need to generate some instructions to merge the
470 // existing values together into a wider type.
471 SmallVector<APInt, 8> ConstantVals;
472 for (auto *Store : Stores) {
473 auto MaybeCst =
475 if (!MaybeCst) {
476 ConstantVals.clear();
477 break;
478 }
479 ConstantVals.emplace_back(MaybeCst->Value);
480 }
481
482 Register WideReg;
483 auto *WideMMO =
484 MF->getMachineMemOperand(&FirstStore->getMMO(), 0, WideValueTy);
485 if (ConstantVals.empty()) {
486 // Mimic the SDAG behaviour here and don't try to do anything for unknown
487 // values. In future, we should also support the cases of loads and
488 // extracted vector elements.
489 return false;
490 }
491
492 assert(ConstantVals.size() == NumStores);
493 // Check if our wide constant is legal.
494 if (!isLegalOrBeforeLegalizer({TargetOpcode::G_CONSTANT, {WideValueTy}}, *MF))
495 return false;
496 APInt WideConst(WideValueTy.getSizeInBits(), 0);
497 for (unsigned Idx = 0; Idx < ConstantVals.size(); ++Idx) {
498 // Insert the smaller constant into the corresponding position in the
499 // wider one.
500 WideConst.insertBits(ConstantVals[Idx], Idx * SmallTy.getSizeInBits());
501 }
502 WideReg = Builder.buildConstant(WideValueTy, WideConst).getReg(0);
503 auto NewStore =
504 Builder.buildStore(WideReg, FirstStore->getPointerReg(), *WideMMO);
505 (void) NewStore;
506 LLVM_DEBUG(dbgs() << "Merged " << Stores.size()
507 << " stores into merged store: " << *NewStore);
508 LLVM_DEBUG(for (auto *MI : Stores) dbgs() << " " << *MI;);
509 NumStoresMerged += Stores.size();
510
511 MachineOptimizationRemarkEmitter MORE(*MF, nullptr);
512 MORE.emit([&]() {
513 MachineOptimizationRemark R(DEBUG_TYPE, "MergedStore",
514 FirstStore->getDebugLoc(),
515 FirstStore->getParent());
516 R << "Merged " << NV("NumMerged", Stores.size()) << " stores of "
517 << NV("OrigWidth", SmallTy.getSizeInBytes())
518 << " bytes into a single store of "
519 << NV("NewWidth", WideValueTy.getSizeInBytes()) << " bytes";
520 return R;
521 });
522
523 InstsToErase.insert_range(Stores);
524 return true;
525}
526
527bool LoadStoreOptImpl::processMergeCandidate(StoreMergeCandidate &C) {
528 if (C.Stores.size() < 2) {
529 C.reset();
530 return false;
531 }
532
533 LLVM_DEBUG(dbgs() << "Checking store merge candidate with " << C.Stores.size()
534 << " stores, starting with " << *C.Stores[0]);
535 // We know that the stores in the candidate are adjacent.
536 // Now we need to check if any potential aliasing instructions recorded
537 // during the search alias with load/stores added to the candidate after.
538 // For example, if we have the candidate:
539 // C.Stores = [ST1, ST2, ST3, ST4]
540 // and after seeing ST2 we saw a load LD1, which did not alias with ST1 or
541 // ST2, then we would have recorded it into the PotentialAliases structure
542 // with the associated index value of "1". Then we see ST3 and ST4 and add
543 // them to the candidate group. We know that LD1 does not alias with ST1 or
544 // ST2, since we already did that check. However we don't yet know if it
545 // may alias ST3 and ST4, so we perform those checks now.
546 SmallVector<GStore *> StoresToMerge;
547
548 auto DoesStoreAliasWithPotential = [&](unsigned Idx, GStore &CheckStore) {
549 for (auto AliasInfo : reverse(C.PotentialAliases)) {
550 MachineInstr *PotentialAliasOp = AliasInfo.first;
551 unsigned PreCheckedIdx = AliasInfo.second;
552 if (Idx < PreCheckedIdx) {
553 // Once our store index is lower than the index associated with the
554 // potential alias, we know that we've already checked for this alias
555 // and all of the earlier potential aliases too.
556 return false;
557 }
558 // Need to check this alias.
559 if (GISelAddressing::instMayAlias(CheckStore, *PotentialAliasOp, *MRI,
560 AA)) {
561 LLVM_DEBUG(dbgs() << "Potential alias " << *PotentialAliasOp
562 << " detected\n");
563 return true;
564 }
565 }
566 return false;
567 };
568 // Start from the last store in the group, and check if it aliases with any
569 // of the potential aliasing operations in the list. A store that does has to
570 // stay where it is, so the stores on either side of it are no longer adjacent
571 // and are merged separately.
572 bool AnyMerged = false;
573 for (int StoreIdx = C.Stores.size() - 1; StoreIdx >= 0; --StoreIdx) {
574 auto *CheckStore = C.Stores[StoreIdx];
575 if (DoesStoreAliasWithPotential(StoreIdx, *CheckStore)) {
576 if (StoresToMerge.size() > 1)
577 AnyMerged |= mergeStores(StoresToMerge);
578 StoresToMerge.clear();
579 continue;
580 }
581 StoresToMerge.emplace_back(CheckStore);
582 }
583
584 LLVM_DEBUG(dbgs() << StoresToMerge.size()
585 << " stores remaining after alias checks. Merging...\n");
586
587 // Now we've checked for aliasing hazards, merge any stores left.
588 C.reset();
589 if (StoresToMerge.size() > 1)
590 AnyMerged |= mergeStores(StoresToMerge);
591 return AnyMerged;
592}
593
594bool LoadStoreOptImpl::operationAliasesWithCandidate(MachineInstr &MI,
595 StoreMergeCandidate &C) {
596 if (C.Stores.empty())
597 return false;
598 return llvm::any_of(C.Stores, [&](MachineInstr *OtherMI) {
599 return instMayAlias(MI, *OtherMI, *MRI, AA);
600 });
601}
602
603void LoadStoreOptImpl::StoreMergeCandidate::addPotentialAlias(
604 MachineInstr &MI) {
605 PotentialAliases.emplace_back(std::make_pair(&MI, Stores.size() - 1));
606}
607
608bool LoadStoreOptImpl::addStoreToCandidate(GStore &StoreMI,
609 StoreMergeCandidate &C) {
610 // Check if the given store writes to an adjacent address, and other
611 // requirements.
612 LLT ValueTy = MRI->getType(StoreMI.getValueReg());
613 LLT PtrTy = MRI->getType(StoreMI.getPointerReg());
614
615 // Only handle scalars.
616 if (!ValueTy.isScalar())
617 return false;
618
619 // Don't allow truncating stores for now.
620 if (StoreMI.getMemSizeInBits() != ValueTy.getSizeInBits())
621 return false;
622
623 // Avoid adding volatile or ordered stores to the candidate. We already have a
624 // check for this in instMayAlias() but that only get's called later between
625 // potential aliasing hazards.
626 if (!StoreMI.isSimple())
627 return false;
628
629 Register StoreAddr = StoreMI.getPointerReg();
630 auto BIO = getPointerInfo(StoreAddr, *MRI);
631 Register StoreBase = BIO.getBase();
632 if (C.Stores.empty()) {
633 C.BasePtr = StoreBase;
634 if (!BIO.hasValidOffset()) {
635 C.CurrentLowestOffset = 0;
636 } else {
637 C.CurrentLowestOffset = BIO.getOffset();
638 }
639 // This is the first store of the candidate.
640 // If the offset can't possibly allow for a lower addressed store with the
641 // same base, don't bother adding it.
642 if (BIO.hasValidOffset() &&
643 BIO.getOffset() < static_cast<int64_t>(ValueTy.getSizeInBytes()))
644 return false;
645 C.Stores.emplace_back(&StoreMI);
646 LLVM_DEBUG(dbgs() << "Starting a new merge candidate group with: "
647 << StoreMI);
648 return true;
649 }
650
651 // Check the store is the same size as the existing ones in the candidate.
652 if (MRI->getType(C.Stores[0]->getValueReg()).getSizeInBits() !=
653 ValueTy.getSizeInBits())
654 return false;
655
656 if (MRI->getType(C.Stores[0]->getPointerReg()).getAddressSpace() !=
657 PtrTy.getAddressSpace())
658 return false;
659
660 // There are other stores in the candidate. Check that the store address
661 // writes to the next lowest adjacent address.
662 if (C.BasePtr != StoreBase)
663 return false;
664 // If we don't have a valid offset, we can't guarantee to be an adjacent
665 // offset.
666 if (!BIO.hasValidOffset())
667 return false;
668 if ((C.CurrentLowestOffset -
669 static_cast<int64_t>(ValueTy.getSizeInBytes())) != BIO.getOffset())
670 return false;
671
672 // This writes to an adjacent address. Allow it.
673 C.Stores.emplace_back(&StoreMI);
674 C.CurrentLowestOffset = C.CurrentLowestOffset - ValueTy.getSizeInBytes();
675 LLVM_DEBUG(dbgs() << "Candidate added store: " << StoreMI);
676 return true;
677}
678
679bool LoadStoreOptImpl::mergeBlockStores(MachineBasicBlock &MBB) {
680 bool Changed = false;
681 // Walk through the block bottom-up, looking for merging candidates.
682 StoreMergeCandidate Candidate;
683 for (MachineInstr &MI : llvm::reverse(MBB)) {
684 if (InstsToErase.contains(&MI))
685 continue;
686
687 if (auto *StoreMI = dyn_cast<GStore>(&MI)) {
688 // We have a G_STORE. Add it to the candidate if it writes to an adjacent
689 // address.
690 if (!addStoreToCandidate(*StoreMI, Candidate)) {
691 // Store wasn't eligible to be added. May need to record it as a
692 // potential alias.
693 if (operationAliasesWithCandidate(*StoreMI, Candidate)) {
694 Changed |= processMergeCandidate(Candidate);
695 continue;
696 }
697 Candidate.addPotentialAlias(*StoreMI);
698 }
699 continue;
700 }
701
702 // If we don't have any stores yet, this instruction can't pose a problem.
703 if (Candidate.Stores.empty())
704 continue;
705
706 // We're dealing with some other kind of instruction.
708 Changed |= processMergeCandidate(Candidate);
709 Candidate.Stores.clear();
710 continue;
711 }
712
713 if (!MI.mayLoadOrStore())
714 continue;
715
716 if (operationAliasesWithCandidate(MI, Candidate)) {
717 // We have a potential alias, so process the current candidate if we can
718 // and then continue looking for a new candidate.
719 Changed |= processMergeCandidate(Candidate);
720 continue;
721 }
722
723 // Record this instruction as a potential alias for future stores that are
724 // added to the candidate.
725 Candidate.addPotentialAlias(MI);
726 }
727
728 // Process any candidate left after finishing searching the entire block.
729 Changed |= processMergeCandidate(Candidate);
730
731 // Erase instructions now that we're no longer iterating over the block.
732 for (auto *MI : InstsToErase)
733 MI->eraseFromParent();
734 InstsToErase.clear();
735 return Changed;
736}
737
738/// Check if the store \p Store is a truncstore that can be merged. That is,
739/// it's a store of a shifted value of \p SrcVal. If \p SrcVal is an empty
740/// Register then it does not need to match and SrcVal is set to the source
741/// value found.
742/// On match, returns the start byte offset of the \p SrcVal that is being
743/// stored.
744static std::optional<int64_t>
747 Register TruncVal;
748 if (!mi_match(Store.getValueReg(), MRI, m_GTrunc(m_Reg(TruncVal))))
749 return std::nullopt;
750
751 // The shift amount must be a constant multiple of the narrow type.
752 // It is translated to the offset address in the wide source value "y".
753 //
754 // x = G_LSHR y, ShiftAmtC
755 // s8 z = G_TRUNC x
756 // store z, ...
757 Register FoundSrcVal;
758 int64_t ShiftAmt;
759 if (!mi_match(TruncVal, MRI,
760 m_any_of(m_GLShr(m_Reg(FoundSrcVal), m_ICst(ShiftAmt)),
761 m_GAShr(m_Reg(FoundSrcVal), m_ICst(ShiftAmt))))) {
762 if (!SrcVal.isValid() || TruncVal == SrcVal) {
763 if (!SrcVal.isValid())
764 SrcVal = TruncVal;
765 return 0; // If it's the lowest index store.
766 }
767 return std::nullopt;
768 }
769
770 unsigned NarrowBits = Store.getMMO().getMemoryType().getScalarSizeInBits();
771 if (ShiftAmt % NarrowBits != 0)
772 return std::nullopt;
773 const unsigned Offset = ShiftAmt / NarrowBits;
774
775 if (SrcVal.isValid() && FoundSrcVal != SrcVal)
776 return std::nullopt;
777
778 if (!SrcVal.isValid())
779 SrcVal = FoundSrcVal;
780 else if (MRI.getType(SrcVal) != MRI.getType(FoundSrcVal))
781 return std::nullopt;
782 return Offset;
783}
784
785/// Match a pattern where a wide type scalar value is stored by several narrow
786/// stores. Fold it into a single store or a BSWAP and a store if the targets
787/// supports it.
788///
789/// Assuming little endian target:
790/// i8 *p = ...
791/// i32 val = ...
792/// p[0] = (val >> 0) & 0xFF;
793/// p[1] = (val >> 8) & 0xFF;
794/// p[2] = (val >> 16) & 0xFF;
795/// p[3] = (val >> 24) & 0xFF;
796/// =>
797/// *((i32)p) = val;
798///
799/// i8 *p = ...
800/// i32 val = ...
801/// p[0] = (val >> 24) & 0xFF;
802/// p[1] = (val >> 16) & 0xFF;
803/// p[2] = (val >> 8) & 0xFF;
804/// p[3] = (val >> 0) & 0xFF;
805/// =>
806/// *((i32)p) = BSWAP(val);
807bool LoadStoreOptImpl::mergeTruncStore(
808 GStore &StoreMI, SmallPtrSetImpl<GStore *> &DeletedStores) {
809 LLT MemTy = StoreMI.getMMO().getMemoryType();
810
811 // We only handle merging simple stores of 1-4 bytes.
812 if (!MemTy.isScalar())
813 return false;
814 switch (MemTy.getSizeInBits()) {
815 case 8:
816 case 16:
817 case 32:
818 break;
819 default:
820 return false;
821 }
822 if (!StoreMI.isSimple())
823 return false;
824
825 // We do a simple search for mergeable stores prior to this one.
826 // Any potential alias hazard along the way terminates the search.
827 SmallVector<GStore *> FoundStores;
828
829 // We're looking for:
830 // 1) a (store(trunc(...)))
831 // 2) of an LSHR/ASHR of a single wide value, by the appropriate shift to get
832 // the partial value stored.
833 // 3) where the offsets form either a little or big-endian sequence.
834
835 auto &LastStore = StoreMI;
836
837 // The single base pointer that all stores must use.
839 int64_t LastOffset;
840 if (!mi_match(LastStore.getPointerReg(), *MRI,
841 m_GPtrAdd(m_Reg(BaseReg), m_ICst(LastOffset)))) {
842 BaseReg = LastStore.getPointerReg();
843 LastOffset = 0;
844 }
845
846 GStore *LowestIdxStore = &LastStore;
847 int64_t LowestIdxOffset = LastOffset;
848
849 Register WideSrcVal;
850 auto LowestShiftAmt = getTruncStoreByteOffset(LastStore, WideSrcVal, *MRI);
851 if (!LowestShiftAmt)
852 return false; // Didn't match a trunc.
853 assert(WideSrcVal.isValid());
854
855 LLT WideStoreTy = MRI->getType(WideSrcVal);
856 // The wide type might not be a multiple of the memory type, e.g. s48 and s32.
857 if (WideStoreTy.getSizeInBits() % MemTy.getSizeInBits() != 0)
858 return false;
859 const unsigned NumStoresRequired =
860 WideStoreTy.getSizeInBits() / MemTy.getSizeInBits();
861
862 SmallVector<int64_t, 8> OffsetMap(NumStoresRequired, INT64_MAX);
863 OffsetMap[*LowestShiftAmt] = LastOffset;
864 FoundStores.emplace_back(&LastStore);
865
866 const int MaxInstsToCheck = 10;
867 int NumInstsChecked = 0;
868 for (auto II = ++LastStore.getReverseIterator();
869 II != LastStore.getParent()->rend() && NumInstsChecked < MaxInstsToCheck;
870 ++II) {
871 NumInstsChecked++;
872 GStore *NewStore;
873 if ((NewStore = dyn_cast<GStore>(&*II))) {
874 if (NewStore->getMMO().getMemoryType() != MemTy || !NewStore->isSimple())
875 break;
876 } else if (II->isLoadFoldBarrier() || II->mayLoad()) {
877 break;
878 } else {
879 continue; // This is a safe instruction we can look past.
880 }
881
882 Register NewBaseReg;
883 int64_t MemOffset;
884 // Check we're storing to the same base + some offset.
885 if (!mi_match(NewStore->getPointerReg(), *MRI,
886 m_GPtrAdd(m_Reg(NewBaseReg), m_ICst(MemOffset)))) {
887 NewBaseReg = NewStore->getPointerReg();
888 MemOffset = 0;
889 }
890 if (BaseReg != NewBaseReg)
891 break;
892
893 auto ShiftByteOffset = getTruncStoreByteOffset(*NewStore, WideSrcVal, *MRI);
894 if (!ShiftByteOffset)
895 break;
896 if (MemOffset < LowestIdxOffset) {
897 LowestIdxOffset = MemOffset;
898 LowestIdxStore = NewStore;
899 }
900
901 // Map the offset in the store and the offset in the combined value, and
902 // early return if it has been set before.
903 if (*ShiftByteOffset < 0 || *ShiftByteOffset >= NumStoresRequired ||
904 OffsetMap[*ShiftByteOffset] != INT64_MAX)
905 break;
906 OffsetMap[*ShiftByteOffset] = MemOffset;
907
908 FoundStores.emplace_back(NewStore);
909 // Reset counter since we've found a matching inst.
910 NumInstsChecked = 0;
911 if (FoundStores.size() == NumStoresRequired)
912 break;
913 }
914
915 if (FoundStores.size() != NumStoresRequired) {
916 if (FoundStores.size() == 1)
917 return false;
918 // We didn't find enough stores to merge into the size of the original
919 // source value, but we may be able to generate a smaller store if we
920 // truncate the source value.
921 WideStoreTy =
922 LLT::integer(FoundStores.size() * MemTy.getScalarSizeInBits());
923 }
924
925 unsigned NumStoresFound = FoundStores.size();
926
927 const auto &DL = LastStore.getMF()->getDataLayout();
928 auto &C = LastStore.getMF()->getFunction().getContext();
929 // Check that a store of the wide type is both allowed and fast on the target
930 unsigned Fast = 0;
932 C, DL, WideStoreTy, LowestIdxStore->getMMO(), &Fast);
933 if (!Allowed || !Fast)
934 return false;
935
936 // Check if the pieces of the value are going to the expected places in memory
937 // to merge the stores.
938 unsigned NarrowBits = MemTy.getScalarSizeInBits();
939 auto checkOffsets = [&](bool MatchLittleEndian) {
940 if (MatchLittleEndian) {
941 for (unsigned i = 0; i != NumStoresFound; ++i)
942 if (OffsetMap[i] != i * (NarrowBits / 8) + LowestIdxOffset)
943 return false;
944 } else { // MatchBigEndian by reversing loop counter.
945 for (unsigned i = 0, j = NumStoresFound - 1; i != NumStoresFound;
946 ++i, --j)
947 if (OffsetMap[j] != i * (NarrowBits / 8) + LowestIdxOffset)
948 return false;
949 }
950 return true;
951 };
952
953 // Check if the offsets line up for the native data layout of this target.
954 bool NeedBswap = false;
955 bool NeedRotate = false;
956 if (!checkOffsets(DL.isLittleEndian())) {
957 // Special-case: check if byte offsets line up for the opposite endian.
958 if (NarrowBits == 8 && checkOffsets(DL.isBigEndian()))
959 NeedBswap = true;
960 else if (NumStoresFound == 2 && checkOffsets(DL.isBigEndian()))
961 NeedRotate = true;
962 else
963 return false;
964 }
965
966 if (NeedBswap &&
967 !isLegalOrBeforeLegalizer({TargetOpcode::G_BSWAP, {WideStoreTy}}, *MF))
968 return false;
969 if (NeedRotate &&
970 !isLegalOrBeforeLegalizer(
971 {TargetOpcode::G_ROTR, {WideStoreTy, WideStoreTy}}, *MF))
972 return false;
973
974 Builder.setInstrAndDebugLoc(StoreMI);
975
976 if (WideStoreTy != MRI->getType(WideSrcVal))
977 WideSrcVal = Builder.buildTrunc(WideStoreTy, WideSrcVal).getReg(0);
978
979 if (NeedBswap) {
980 WideSrcVal = Builder.buildBSwap(WideStoreTy, WideSrcVal).getReg(0);
981 } else if (NeedRotate) {
982 assert(WideStoreTy.getSizeInBits() % 2 == 0 &&
983 "Unexpected type for rotate");
984 auto RotAmt =
985 Builder.buildConstant(WideStoreTy, WideStoreTy.getSizeInBits() / 2);
986 WideSrcVal =
987 Builder.buildRotateRight(WideStoreTy, WideSrcVal, RotAmt).getReg(0);
988 }
989
990 Builder.buildStore(WideSrcVal, LowestIdxStore->getPointerReg(),
991 LowestIdxStore->getMMO().getPointerInfo(),
992 LowestIdxStore->getMMO().getAlign());
993
994 // Erase the old stores.
995 for (auto *ST : FoundStores) {
996 ST->eraseFromParent();
997 DeletedStores.insert(ST);
998 }
999 return true;
1000}
1001
1002bool LoadStoreOptImpl::mergeTruncStoresBlock(MachineBasicBlock &BB) {
1003 bool Changed = false;
1005 SmallPtrSet<GStore *, 8> DeletedStores;
1006 // Walk up the block so we can see the most eligible stores.
1007 for (MachineInstr &MI : llvm::reverse(BB))
1008 if (auto *StoreMI = dyn_cast<GStore>(&MI))
1009 Stores.emplace_back(StoreMI);
1010
1011 for (auto *StoreMI : Stores) {
1012 if (DeletedStores.count(StoreMI))
1013 continue;
1014 if (mergeTruncStore(*StoreMI, DeletedStores))
1015 Changed = true;
1016 }
1017 return Changed;
1018}
1019
1020bool LoadStoreOptImpl::mergeFunctionStores(MachineFunction &MF) {
1021 bool Changed = false;
1022 for (auto &BB : MF){
1023 Changed |= mergeBlockStores(BB);
1024 Changed |= mergeTruncStoresBlock(BB);
1025 }
1026
1027 // Erase all dead instructions left over by the merging.
1028 if (Changed) {
1029 for (auto &BB : MF) {
1030 for (auto &I : make_early_inc_range(reverse(BB))) {
1031 if (isTriviallyDead(I, *MRI))
1032 I.eraseFromParent();
1033 }
1034 }
1035 }
1036
1037 return Changed;
1038}
1039
1040void LoadStoreOptImpl::initializeStoreMergeTargetInfo(unsigned AddrSpace) {
1041 // Query the legalizer info to record what store types are legal.
1042 // We record this because we don't want to bother trying to merge stores into
1043 // illegal ones, which would just result in being split again.
1044
1045 if (LegalStoreSizes.count(AddrSpace)) {
1046 assert(LegalStoreSizes[AddrSpace].any());
1047 return; // Already cached sizes for this address space.
1048 }
1049
1050 // Need to reserve at least MaxStoreSizeToForm + 1 bits.
1051 BitVector LegalSizes(MaxStoreSizeToForm * 2);
1052 const auto &LI = *MF->getSubtarget().getLegalizerInfo();
1053 const auto &DL = MF->getFunction().getDataLayout();
1054 Type *IRPtrTy = PointerType::get(MF->getFunction().getContext(), AddrSpace);
1055 LLT PtrTy = getLLTForType(*IRPtrTy, DL);
1056 // We assume that we're not going to be generating any stores wider than
1057 // MaxStoreSizeToForm bits for now.
1058 for (unsigned Size = 2; Size <= MaxStoreSizeToForm; Size *= 2) {
1059 LLT Ty = LLT::scalar(Size);
1061 {{Ty, Ty.getSizeInBits(), AtomicOrdering::NotAtomic,
1062 AtomicOrdering::NotAtomic}});
1063 SmallVector<LLT> StoreTys({Ty, PtrTy});
1064 LegalityQuery Q(TargetOpcode::G_STORE, StoreTys, MemDescrs);
1065 LegalizeActionStep ActionStep = LI.getAction(Q);
1066 if (ActionStep.Action == LegalizeActions::Legal)
1067 LegalSizes.set(Size);
1068 }
1069 assert(LegalSizes.any() && "Expected some store sizes to be legal!");
1070 LegalStoreSizes[AddrSpace] = std::move(LegalSizes);
1071}
1072
1073bool LoadStoreOptImpl::runOnMachineFunction(
1074 MachineFunction &MF, function_ref<AliasAnalysis *()> GetAA) {
1075 // If the ISel pipeline failed, do not bother running that pass.
1076 if (MF.getProperties().hasFailedISel())
1077 return false;
1078
1079 LLVM_DEBUG(dbgs() << "Begin memory optimizations for: " << MF.getName()
1080 << '\n');
1081
1082 init(MF, GetAA);
1083 bool Changed = false;
1084 Changed |= mergeFunctionStores(MF);
1085
1086 LegalStoreSizes.clear();
1087 return Changed;
1088}
1089
1091 LoadStoreOptImpl Impl;
1092 return Impl.runOnMachineFunction(MF, [&]() {
1093 return &getAnalysis<AAResultsWrapperPass>().getAAResults();
1094 });
1095}
1096
1100 LoadStoreOptImpl Impl;
1101 Impl.runOnMachineFunction(MF, [&]() {
1104 .getManager();
1105 return &FAM.getResult<AAManager>(MF.getFunction());
1106 });
1107 return PreservedAnalyses::all();
1108}
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Atomic ordering constants.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
#define _
IRTranslator LLVM IR MI
Interface for Targets to specify which operations they can successfully select and how the others sho...
const unsigned MaxStoreSizeToForm
static std::optional< int64_t > getTruncStoreByteOffset(GStore &Store, Register &SrcVal, MachineRegisterInfo &MRI)
Check if the store Store is a truncstore that can be merged.
static bool isInstHardMergeHazard(MachineInstr &MI)
Returns true if the instruction creates an unavoidable hazard that forces a boundary between store me...
Implement a low-level type suitable for MachineInstr level instruction selection.
#define I(x, y, z)
Definition MD5.cpp:57
Contains matchers for matching SSA Machine Instructions.
===- MachineOptimizationRemarkEmitter.h - Opt Diagnostics -*- C++ -*-—===//
Promote Memory to Register
Definition Mem2Reg.cpp:110
This file provides utility analysis objects describing memory locations.
uint64_t IntrinsicInst * II
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
MachineRegisterInfo & MRI
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallPtrSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:119
This file describes how to lower LLVM code to machine code.
A manager for alias analyses.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
static LLVM_ABI DebugLoc getMergedLocation(DebugLoc LocA, DebugLoc LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
Definition DebugLoc.cpp:186
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
Register getValueReg() const
Get the stored value register.
Helper struct to store a base, index and offset that forms an address.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
Represents a G_STORE.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr unsigned getAddressSpace() const
static LLT integer(unsigned SizeInBits)
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
LLVM_ABI PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
bool hasValue() const
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
bool isScalable() const
TypeSize getValue() const
An RAII based helper class to modify MachineFunctionProperties when running pass.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineFunctionProperties & getProperties() const
Get the function properties.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Helper class to build MachineInstr.
MachineInstrBuilder buildRotateRight(const DstOp &Dst, const SrcOp &Src, const SrcOp &Amt)
Build and insert Dst = G_ROTR Src, Amt.
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
MachineInstrBuilder buildBSwap(const DstOp &Dst, const SrcOp &Src0)
Build and insert Dst = G_BSWAP Src0.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_TRUNC Op.
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Register getReg(unsigned Idx) const
Get the register for the operand index.
Representation of each machine instruction.
const MachineBasicBlock * getParent() const
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
A description of a memory reference used in the backend.
LLT getMemoryType() const
Return the memory type of the memory reference.
const MachinePointerInfo & getPointerInfo() const
LLVM_ABI Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Representation for a specific memory location.
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
void insert_range(Range &&R)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const
Returns if it's reasonable to merge stores to MemVT size.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual const LegalizerInfo * getLegalizerInfo() const
virtual const TargetLowering * getTargetLowering() const
constexpr ScalarTy getFixedValue() const
Definition TypeSize.h:201
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
An efficient, type-erasing, non-owning reference to a callable.
#define INT64_MAX
Definition DataTypes.h:71
Pass manager infrastructure for declaring and invalidating analyses.
Abstract Attribute helper functions.
Definition Attributor.h:166
constexpr bool any(E Val)
LLVM_ABI bool aliasIsKnownForLoadStore(const MachineInstr &MI1, const MachineInstr &MI2, bool &IsAlias, MachineRegisterInfo &MRI)
Compute whether or not a memory access at MI1 aliases with an access at MI2.
LLVM_ABI BaseIndexOffset getPointerInfo(Register Ptr, MachineRegisterInfo &MRI)
Returns a BaseIndexOffset which describes the pointer in Ptr.
LLVM_ABI bool instMayAlias(const MachineInstr &MI, const MachineInstr &Other, MachineRegisterInfo &MRI, AliasAnalysis *AA)
Returns true if the instruction MI may alias Other.
@ Legal
The operation is expected to be selectable directly by the target, and no transformation is necessary...
@ Unsupported
This operation is completely unsupported on the target.
operand_type_match m_Reg()
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ASHR, false > m_GAShr(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
initializer< Ty > init(const Ty &Val)
Add a small namespace to avoid name clashes with the classes used in the streaming interface.
DiagnosticInfoOptimizationBase::Argument NV
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Store
The extracted value is stored (ExtractElement only).
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
LLVM_ABI MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
Definition Utils.cpp:520
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI EVT getApproximateEVTForLLT(LLT Ty, LLVMContext &Ctx)
@ Other
Any other memory.
Definition ModRef.h:68
LLVM_ABI void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition Utils.cpp:1296
@ Fast
Assign the register banks as fast as possible (default).
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:440
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
Definition bit.h:347
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
LLVM_ABI LLT getLLTForType(Type &Ty, const DataLayout &DL)
Construct a low-level type based on an LLVM type.
LLVM_ABI bool isTriviallyDead(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Check whether an instruction MI is dead: it only defines dead virtual registers, and doesn't have oth...
Definition Utils.cpp:228
#define MORE()
Definition regcomp.c:247
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
LegalizeAction Action
The action to take or the final answer.