LLVM 18.0.0git
AArch64LoadStoreOptimizer.cpp
Go to the documentation of this file.
1//===- AArch64LoadStoreOptimizer.cpp - AArch64 load/store opt. pass -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that performs load / store related peephole
10// optimizations. This pass should be run after register allocation.
11//
12// The pass runs after the PrologEpilogInserter where we emit the CFI
13// instructions. In order to preserve the correctness of the unwind informaiton,
14// the pass should not change the order of any two instructions, one of which
15// has the FrameSetup/FrameDestroy flag or, alternatively, apply an add-hoc fix
16// to unwind information.
17//
18//===----------------------------------------------------------------------===//
19
20#include "AArch64InstrInfo.h"
22#include "AArch64Subtarget.h"
24#include "llvm/ADT/BitVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringRef.h"
38#include "llvm/IR/DebugLoc.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCDwarf.h"
42#include "llvm/Pass.h"
44#include "llvm/Support/Debug.h"
48#include <cassert>
49#include <cstdint>
50#include <functional>
51#include <iterator>
52#include <limits>
53#include <optional>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "aarch64-ldst-opt"
58
59STATISTIC(NumPairCreated, "Number of load/store pair instructions generated");
60STATISTIC(NumPostFolded, "Number of post-index updates folded");
61STATISTIC(NumPreFolded, "Number of pre-index updates folded");
62STATISTIC(NumUnscaledPairCreated,
63 "Number of load/store from unscaled generated");
64STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
65STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
66
67DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming",
68 "Controls which pairs are considered for renaming");
69
70// The LdStLimit limits how far we search for load/store pairs.
71static cl::opt<unsigned> LdStLimit("aarch64-load-store-scan-limit",
72 cl::init(20), cl::Hidden);
73
74// The UpdateLimit limits how far we search for update instructions when we form
75// pre-/post-index instructions.
76static cl::opt<unsigned> UpdateLimit("aarch64-update-scan-limit", cl::init(100),
78
79// Enable register renaming to find additional store pairing opportunities.
80static cl::opt<bool> EnableRenaming("aarch64-load-store-renaming",
81 cl::init(true), cl::Hidden);
82
83#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"
84
85namespace {
86
87using LdStPairFlags = struct LdStPairFlags {
88 // If a matching instruction is found, MergeForward is set to true if the
89 // merge is to remove the first instruction and replace the second with
90 // a pair-wise insn, and false if the reverse is true.
91 bool MergeForward = false;
92
93 // SExtIdx gives the index of the result of the load pair that must be
94 // extended. The value of SExtIdx assumes that the paired load produces the
95 // value in this order: (I, returned iterator), i.e., -1 means no value has
96 // to be extended, 0 means I, and 1 means the returned iterator.
97 int SExtIdx = -1;
98
99 // If not none, RenameReg can be used to rename the result register of the
100 // first store in a pair. Currently this only works when merging stores
101 // forward.
102 std::optional<MCPhysReg> RenameReg;
103
104 LdStPairFlags() = default;
105
106 void setMergeForward(bool V = true) { MergeForward = V; }
107 bool getMergeForward() const { return MergeForward; }
108
109 void setSExtIdx(int V) { SExtIdx = V; }
110 int getSExtIdx() const { return SExtIdx; }
111
112 void setRenameReg(MCPhysReg R) { RenameReg = R; }
113 void clearRenameReg() { RenameReg = std::nullopt; }
114 std::optional<MCPhysReg> getRenameReg() const { return RenameReg; }
115};
116
117struct AArch64LoadStoreOpt : public MachineFunctionPass {
118 static char ID;
119
120 AArch64LoadStoreOpt() : MachineFunctionPass(ID) {
122 }
123
124 AliasAnalysis *AA;
125 const AArch64InstrInfo *TII;
126 const TargetRegisterInfo *TRI;
127 const AArch64Subtarget *Subtarget;
128
129 // Track which register units have been modified and used.
130 LiveRegUnits ModifiedRegUnits, UsedRegUnits;
131 LiveRegUnits DefinedInBB;
132
133 void getAnalysisUsage(AnalysisUsage &AU) const override {
136 }
137
138 // Scan the instructions looking for a load/store that can be combined
139 // with the current instruction into a load/store pair.
140 // Return the matching instruction if one is found, else MBB->end().
142 LdStPairFlags &Flags,
143 unsigned Limit,
144 bool FindNarrowMerge);
145
146 // Scan the instructions looking for a store that writes to the address from
147 // which the current load instruction reads. Return true if one is found.
148 bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit,
150
151 // Merge the two instructions indicated into a wider narrow store instruction.
153 mergeNarrowZeroStores(MachineBasicBlock::iterator I,
155 const LdStPairFlags &Flags);
156
157 // Merge the two instructions indicated into a single pair-wise instruction.
159 mergePairedInsns(MachineBasicBlock::iterator I,
161 const LdStPairFlags &Flags);
162
163 // Promote the load that reads directly from the address stored to.
165 promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
167
168 // Scan the instruction list to find a base register update that can
169 // be combined with the current instruction (a load or store) using
170 // pre or post indexed addressing with writeback. Scan forwards.
172 findMatchingUpdateInsnForward(MachineBasicBlock::iterator I,
173 int UnscaledOffset, unsigned Limit);
174
175 // Scan the instruction list to find a base register update that can
176 // be combined with the current instruction (a load or store) using
177 // pre or post indexed addressing with writeback. Scan backwards.
179 findMatchingUpdateInsnBackward(MachineBasicBlock::iterator I, unsigned Limit);
180
181 // Find an instruction that updates the base register of the ld/st
182 // instruction.
183 bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI,
184 unsigned BaseReg, int Offset);
185
186 // Merge a pre- or post-index base register update into a ld/st instruction.
188 mergeUpdateInsn(MachineBasicBlock::iterator I,
189 MachineBasicBlock::iterator Update, bool IsPreIdx);
190
191 // Find and merge zero store instructions.
192 bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
193
194 // Find and pair ldr/str instructions.
195 bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
196
197 // Find and promote load instructions which read directly from store.
198 bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI);
199
200 // Find and merge a base register updates before or after a ld/st instruction.
201 bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI);
202
203 bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt);
204
205 bool runOnMachineFunction(MachineFunction &Fn) override;
206
209 MachineFunctionProperties::Property::NoVRegs);
210 }
211
212 StringRef getPassName() const override { return AARCH64_LOAD_STORE_OPT_NAME; }
213};
214
215char AArch64LoadStoreOpt::ID = 0;
216
217} // end anonymous namespace
218
219INITIALIZE_PASS(AArch64LoadStoreOpt, "aarch64-ldst-opt",
220 AARCH64_LOAD_STORE_OPT_NAME, false, false)
221
222static bool isNarrowStore(unsigned Opc) {
223 switch (Opc) {
224 default:
225 return false;
226 case AArch64::STRBBui:
227 case AArch64::STURBBi:
228 case AArch64::STRHHui:
229 case AArch64::STURHHi:
230 return true;
231 }
232}
233
234// These instruction set memory tag and either keep memory contents unchanged or
235// set it to zero, ignoring the address part of the source register.
236static bool isTagStore(const MachineInstr &MI) {
237 switch (MI.getOpcode()) {
238 default:
239 return false;
240 case AArch64::STGi:
241 case AArch64::STZGi:
242 case AArch64::ST2Gi:
243 case AArch64::STZ2Gi:
244 return true;
245 }
246}
247
248static unsigned getMatchingNonSExtOpcode(unsigned Opc,
249 bool *IsValidLdStrOpc = nullptr) {
250 if (IsValidLdStrOpc)
251 *IsValidLdStrOpc = true;
252 switch (Opc) {
253 default:
254 if (IsValidLdStrOpc)
255 *IsValidLdStrOpc = false;
256 return std::numeric_limits<unsigned>::max();
257 case AArch64::STRDui:
258 case AArch64::STURDi:
259 case AArch64::STRDpre:
260 case AArch64::STRQui:
261 case AArch64::STURQi:
262 case AArch64::STRQpre:
263 case AArch64::STRBBui:
264 case AArch64::STURBBi:
265 case AArch64::STRHHui:
266 case AArch64::STURHHi:
267 case AArch64::STRWui:
268 case AArch64::STRWpre:
269 case AArch64::STURWi:
270 case AArch64::STRXui:
271 case AArch64::STRXpre:
272 case AArch64::STURXi:
273 case AArch64::LDRDui:
274 case AArch64::LDURDi:
275 case AArch64::LDRDpre:
276 case AArch64::LDRQui:
277 case AArch64::LDURQi:
278 case AArch64::LDRQpre:
279 case AArch64::LDRWui:
280 case AArch64::LDURWi:
281 case AArch64::LDRWpre:
282 case AArch64::LDRXui:
283 case AArch64::LDURXi:
284 case AArch64::LDRXpre:
285 case AArch64::STRSui:
286 case AArch64::STURSi:
287 case AArch64::STRSpre:
288 case AArch64::LDRSui:
289 case AArch64::LDURSi:
290 case AArch64::LDRSpre:
291 return Opc;
292 case AArch64::LDRSWui:
293 return AArch64::LDRWui;
294 case AArch64::LDURSWi:
295 return AArch64::LDURWi;
296 case AArch64::LDRSWpre:
297 return AArch64::LDRWpre;
298 }
299}
300
301static unsigned getMatchingWideOpcode(unsigned Opc) {
302 switch (Opc) {
303 default:
304 llvm_unreachable("Opcode has no wide equivalent!");
305 case AArch64::STRBBui:
306 return AArch64::STRHHui;
307 case AArch64::STRHHui:
308 return AArch64::STRWui;
309 case AArch64::STURBBi:
310 return AArch64::STURHHi;
311 case AArch64::STURHHi:
312 return AArch64::STURWi;
313 case AArch64::STURWi:
314 return AArch64::STURXi;
315 case AArch64::STRWui:
316 return AArch64::STRXui;
317 }
318}
319
320static unsigned getMatchingPairOpcode(unsigned Opc) {
321 switch (Opc) {
322 default:
323 llvm_unreachable("Opcode has no pairwise equivalent!");
324 case AArch64::STRSui:
325 case AArch64::STURSi:
326 return AArch64::STPSi;
327 case AArch64::STRSpre:
328 return AArch64::STPSpre;
329 case AArch64::STRDui:
330 case AArch64::STURDi:
331 return AArch64::STPDi;
332 case AArch64::STRDpre:
333 return AArch64::STPDpre;
334 case AArch64::STRQui:
335 case AArch64::STURQi:
336 return AArch64::STPQi;
337 case AArch64::STRQpre:
338 return AArch64::STPQpre;
339 case AArch64::STRWui:
340 case AArch64::STURWi:
341 return AArch64::STPWi;
342 case AArch64::STRWpre:
343 return AArch64::STPWpre;
344 case AArch64::STRXui:
345 case AArch64::STURXi:
346 return AArch64::STPXi;
347 case AArch64::STRXpre:
348 return AArch64::STPXpre;
349 case AArch64::LDRSui:
350 case AArch64::LDURSi:
351 return AArch64::LDPSi;
352 case AArch64::LDRSpre:
353 return AArch64::LDPSpre;
354 case AArch64::LDRDui:
355 case AArch64::LDURDi:
356 return AArch64::LDPDi;
357 case AArch64::LDRDpre:
358 return AArch64::LDPDpre;
359 case AArch64::LDRQui:
360 case AArch64::LDURQi:
361 return AArch64::LDPQi;
362 case AArch64::LDRQpre:
363 return AArch64::LDPQpre;
364 case AArch64::LDRWui:
365 case AArch64::LDURWi:
366 return AArch64::LDPWi;
367 case AArch64::LDRWpre:
368 return AArch64::LDPWpre;
369 case AArch64::LDRXui:
370 case AArch64::LDURXi:
371 return AArch64::LDPXi;
372 case AArch64::LDRXpre:
373 return AArch64::LDPXpre;
374 case AArch64::LDRSWui:
375 case AArch64::LDURSWi:
376 return AArch64::LDPSWi;
377 case AArch64::LDRSWpre:
378 return AArch64::LDPSWpre;
379 }
380}
381
384 unsigned LdOpc = LoadInst.getOpcode();
385 unsigned StOpc = StoreInst.getOpcode();
386 switch (LdOpc) {
387 default:
388 llvm_unreachable("Unsupported load instruction!");
389 case AArch64::LDRBBui:
390 return StOpc == AArch64::STRBBui || StOpc == AArch64::STRHHui ||
391 StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
392 case AArch64::LDURBBi:
393 return StOpc == AArch64::STURBBi || StOpc == AArch64::STURHHi ||
394 StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
395 case AArch64::LDRHHui:
396 return StOpc == AArch64::STRHHui || StOpc == AArch64::STRWui ||
397 StOpc == AArch64::STRXui;
398 case AArch64::LDURHHi:
399 return StOpc == AArch64::STURHHi || StOpc == AArch64::STURWi ||
400 StOpc == AArch64::STURXi;
401 case AArch64::LDRWui:
402 return StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
403 case AArch64::LDURWi:
404 return StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
405 case AArch64::LDRXui:
406 return StOpc == AArch64::STRXui;
407 case AArch64::LDURXi:
408 return StOpc == AArch64::STURXi;
409 }
410}
411
412static unsigned getPreIndexedOpcode(unsigned Opc) {
413 // FIXME: We don't currently support creating pre-indexed loads/stores when
414 // the load or store is the unscaled version. If we decide to perform such an
415 // optimization in the future the cases for the unscaled loads/stores will
416 // need to be added here.
417 switch (Opc) {
418 default:
419 llvm_unreachable("Opcode has no pre-indexed equivalent!");
420 case AArch64::STRSui:
421 return AArch64::STRSpre;
422 case AArch64::STRDui:
423 return AArch64::STRDpre;
424 case AArch64::STRQui:
425 return AArch64::STRQpre;
426 case AArch64::STRBBui:
427 return AArch64::STRBBpre;
428 case AArch64::STRHHui:
429 return AArch64::STRHHpre;
430 case AArch64::STRWui:
431 return AArch64::STRWpre;
432 case AArch64::STRXui:
433 return AArch64::STRXpre;
434 case AArch64::LDRSui:
435 return AArch64::LDRSpre;
436 case AArch64::LDRDui:
437 return AArch64::LDRDpre;
438 case AArch64::LDRQui:
439 return AArch64::LDRQpre;
440 case AArch64::LDRBBui:
441 return AArch64::LDRBBpre;
442 case AArch64::LDRHHui:
443 return AArch64::LDRHHpre;
444 case AArch64::LDRWui:
445 return AArch64::LDRWpre;
446 case AArch64::LDRXui:
447 return AArch64::LDRXpre;
448 case AArch64::LDRSWui:
449 return AArch64::LDRSWpre;
450 case AArch64::LDPSi:
451 return AArch64::LDPSpre;
452 case AArch64::LDPSWi:
453 return AArch64::LDPSWpre;
454 case AArch64::LDPDi:
455 return AArch64::LDPDpre;
456 case AArch64::LDPQi:
457 return AArch64::LDPQpre;
458 case AArch64::LDPWi:
459 return AArch64::LDPWpre;
460 case AArch64::LDPXi:
461 return AArch64::LDPXpre;
462 case AArch64::STPSi:
463 return AArch64::STPSpre;
464 case AArch64::STPDi:
465 return AArch64::STPDpre;
466 case AArch64::STPQi:
467 return AArch64::STPQpre;
468 case AArch64::STPWi:
469 return AArch64::STPWpre;
470 case AArch64::STPXi:
471 return AArch64::STPXpre;
472 case AArch64::STGi:
473 return AArch64::STGPreIndex;
474 case AArch64::STZGi:
475 return AArch64::STZGPreIndex;
476 case AArch64::ST2Gi:
477 return AArch64::ST2GPreIndex;
478 case AArch64::STZ2Gi:
479 return AArch64::STZ2GPreIndex;
480 case AArch64::STGPi:
481 return AArch64::STGPpre;
482 }
483}
484
485static unsigned getPostIndexedOpcode(unsigned Opc) {
486 switch (Opc) {
487 default:
488 llvm_unreachable("Opcode has no post-indexed wise equivalent!");
489 case AArch64::STRSui:
490 case AArch64::STURSi:
491 return AArch64::STRSpost;
492 case AArch64::STRDui:
493 case AArch64::STURDi:
494 return AArch64::STRDpost;
495 case AArch64::STRQui:
496 case AArch64::STURQi:
497 return AArch64::STRQpost;
498 case AArch64::STRBBui:
499 return AArch64::STRBBpost;
500 case AArch64::STRHHui:
501 return AArch64::STRHHpost;
502 case AArch64::STRWui:
503 case AArch64::STURWi:
504 return AArch64::STRWpost;
505 case AArch64::STRXui:
506 case AArch64::STURXi:
507 return AArch64::STRXpost;
508 case AArch64::LDRSui:
509 case AArch64::LDURSi:
510 return AArch64::LDRSpost;
511 case AArch64::LDRDui:
512 case AArch64::LDURDi:
513 return AArch64::LDRDpost;
514 case AArch64::LDRQui:
515 case AArch64::LDURQi:
516 return AArch64::LDRQpost;
517 case AArch64::LDRBBui:
518 return AArch64::LDRBBpost;
519 case AArch64::LDRHHui:
520 return AArch64::LDRHHpost;
521 case AArch64::LDRWui:
522 case AArch64::LDURWi:
523 return AArch64::LDRWpost;
524 case AArch64::LDRXui:
525 case AArch64::LDURXi:
526 return AArch64::LDRXpost;
527 case AArch64::LDRSWui:
528 return AArch64::LDRSWpost;
529 case AArch64::LDPSi:
530 return AArch64::LDPSpost;
531 case AArch64::LDPSWi:
532 return AArch64::LDPSWpost;
533 case AArch64::LDPDi:
534 return AArch64::LDPDpost;
535 case AArch64::LDPQi:
536 return AArch64::LDPQpost;
537 case AArch64::LDPWi:
538 return AArch64::LDPWpost;
539 case AArch64::LDPXi:
540 return AArch64::LDPXpost;
541 case AArch64::STPSi:
542 return AArch64::STPSpost;
543 case AArch64::STPDi:
544 return AArch64::STPDpost;
545 case AArch64::STPQi:
546 return AArch64::STPQpost;
547 case AArch64::STPWi:
548 return AArch64::STPWpost;
549 case AArch64::STPXi:
550 return AArch64::STPXpost;
551 case AArch64::STGi:
552 return AArch64::STGPostIndex;
553 case AArch64::STZGi:
554 return AArch64::STZGPostIndex;
555 case AArch64::ST2Gi:
556 return AArch64::ST2GPostIndex;
557 case AArch64::STZ2Gi:
558 return AArch64::STZ2GPostIndex;
559 case AArch64::STGPi:
560 return AArch64::STGPpost;
561 }
562}
563
565
566 unsigned OpcA = FirstMI.getOpcode();
567 unsigned OpcB = MI.getOpcode();
568
569 switch (OpcA) {
570 default:
571 return false;
572 case AArch64::STRSpre:
573 return (OpcB == AArch64::STRSui) || (OpcB == AArch64::STURSi);
574 case AArch64::STRDpre:
575 return (OpcB == AArch64::STRDui) || (OpcB == AArch64::STURDi);
576 case AArch64::STRQpre:
577 return (OpcB == AArch64::STRQui) || (OpcB == AArch64::STURQi);
578 case AArch64::STRWpre:
579 return (OpcB == AArch64::STRWui) || (OpcB == AArch64::STURWi);
580 case AArch64::STRXpre:
581 return (OpcB == AArch64::STRXui) || (OpcB == AArch64::STURXi);
582 case AArch64::LDRSpre:
583 return (OpcB == AArch64::LDRSui) || (OpcB == AArch64::LDURSi);
584 case AArch64::LDRDpre:
585 return (OpcB == AArch64::LDRDui) || (OpcB == AArch64::LDURDi);
586 case AArch64::LDRQpre:
587 return (OpcB == AArch64::LDRQui) || (OpcB == AArch64::LDURQi);
588 case AArch64::LDRWpre:
589 return (OpcB == AArch64::LDRWui) || (OpcB == AArch64::LDURWi);
590 case AArch64::LDRXpre:
591 return (OpcB == AArch64::LDRXui) || (OpcB == AArch64::LDURXi);
592 case AArch64::LDRSWpre:
593 return (OpcB == AArch64::LDRSWui) || (OpcB == AArch64::LDURSWi);
594 }
595}
596
597// Returns the scale and offset range of pre/post indexed variants of MI.
598static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale,
599 int &MinOffset, int &MaxOffset) {
600 bool IsPaired = AArch64InstrInfo::isPairedLdSt(MI);
601 bool IsTagStore = isTagStore(MI);
602 // ST*G and all paired ldst have the same scale in pre/post-indexed variants
603 // as in the "unsigned offset" variant.
604 // All other pre/post indexed ldst instructions are unscaled.
605 Scale = (IsTagStore || IsPaired) ? AArch64InstrInfo::getMemScale(MI) : 1;
606
607 if (IsPaired) {
608 MinOffset = -64;
609 MaxOffset = 63;
610 } else {
611 MinOffset = -256;
612 MaxOffset = 255;
613 }
614}
615
617 unsigned PairedRegOp = 0) {
618 assert(PairedRegOp < 2 && "Unexpected register operand idx.");
619 bool IsPreLdSt = AArch64InstrInfo::isPreLdSt(MI);
620 if (IsPreLdSt)
621 PairedRegOp += 1;
622 unsigned Idx =
623 AArch64InstrInfo::isPairedLdSt(MI) || IsPreLdSt ? PairedRegOp : 0;
624 return MI.getOperand(Idx);
625}
626
629 const AArch64InstrInfo *TII) {
630 assert(isMatchingStore(LoadInst, StoreInst) && "Expect only matched ld/st.");
631 int LoadSize = TII->getMemScale(LoadInst);
632 int StoreSize = TII->getMemScale(StoreInst);
633 int UnscaledStOffset =
634 TII->hasUnscaledLdStOffset(StoreInst)
637 int UnscaledLdOffset =
638 TII->hasUnscaledLdStOffset(LoadInst)
641 return (UnscaledStOffset <= UnscaledLdOffset) &&
642 (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize));
643}
644
646 unsigned Opc = MI.getOpcode();
647 return (Opc == AArch64::STRWui || Opc == AArch64::STURWi ||
648 isNarrowStore(Opc)) &&
649 getLdStRegOp(MI).getReg() == AArch64::WZR;
650}
651
653 switch (MI.getOpcode()) {
654 default:
655 return false;
656 // Scaled instructions.
657 case AArch64::LDRBBui:
658 case AArch64::LDRHHui:
659 case AArch64::LDRWui:
660 case AArch64::LDRXui:
661 // Unscaled instructions.
662 case AArch64::LDURBBi:
663 case AArch64::LDURHHi:
664 case AArch64::LDURWi:
665 case AArch64::LDURXi:
666 return true;
667 }
668}
669
671 unsigned Opc = MI.getOpcode();
672 switch (Opc) {
673 default:
674 return false;
675 // Scaled instructions.
676 case AArch64::STRSui:
677 case AArch64::STRDui:
678 case AArch64::STRQui:
679 case AArch64::STRXui:
680 case AArch64::STRWui:
681 case AArch64::STRHHui:
682 case AArch64::STRBBui:
683 case AArch64::LDRSui:
684 case AArch64::LDRDui:
685 case AArch64::LDRQui:
686 case AArch64::LDRXui:
687 case AArch64::LDRWui:
688 case AArch64::LDRHHui:
689 case AArch64::LDRBBui:
690 case AArch64::STGi:
691 case AArch64::STZGi:
692 case AArch64::ST2Gi:
693 case AArch64::STZ2Gi:
694 case AArch64::STGPi:
695 // Unscaled instructions.
696 case AArch64::STURSi:
697 case AArch64::STURDi:
698 case AArch64::STURQi:
699 case AArch64::STURWi:
700 case AArch64::STURXi:
701 case AArch64::LDURSi:
702 case AArch64::LDURDi:
703 case AArch64::LDURQi:
704 case AArch64::LDURWi:
705 case AArch64::LDURXi:
706 // Paired instructions.
707 case AArch64::LDPSi:
708 case AArch64::LDPSWi:
709 case AArch64::LDPDi:
710 case AArch64::LDPQi:
711 case AArch64::LDPWi:
712 case AArch64::LDPXi:
713 case AArch64::STPSi:
714 case AArch64::STPDi:
715 case AArch64::STPQi:
716 case AArch64::STPWi:
717 case AArch64::STPXi:
718 // Make sure this is a reg+imm (as opposed to an address reloc).
720 return false;
721
722 return true;
723 }
724}
725
727AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
729 const LdStPairFlags &Flags) {
731 "Expected promotable zero stores.");
732
733 MachineBasicBlock::iterator E = I->getParent()->end();
735 // If NextI is the second of the two instructions to be merged, we need
736 // to skip one further. Either way we merge will invalidate the iterator,
737 // and we don't need to scan the new instruction, as it's a pairwise
738 // instruction, which we're not considering for further action anyway.
739 if (NextI == MergeMI)
740 NextI = next_nodbg(NextI, E);
741
742 unsigned Opc = I->getOpcode();
743 unsigned MergeMIOpc = MergeMI->getOpcode();
744 bool IsScaled = !TII->hasUnscaledLdStOffset(Opc);
745 bool IsMergedMIScaled = !TII->hasUnscaledLdStOffset(MergeMIOpc);
746 int OffsetStride = IsScaled ? TII->getMemScale(*I) : 1;
747 int MergeMIOffsetStride = IsMergedMIScaled ? TII->getMemScale(*MergeMI) : 1;
748
749 bool MergeForward = Flags.getMergeForward();
750 // Insert our new paired instruction after whichever of the paired
751 // instructions MergeForward indicates.
752 MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I;
753 // Also based on MergeForward is from where we copy the base register operand
754 // so we get the flags compatible with the input code.
755 const MachineOperand &BaseRegOp =
756 MergeForward ? AArch64InstrInfo::getLdStBaseOp(*MergeMI)
757 : AArch64InstrInfo::getLdStBaseOp(*I);
758
759 // Which register is Rt and which is Rt2 depends on the offset order.
760 int64_t IOffsetInBytes =
761 AArch64InstrInfo::getLdStOffsetOp(*I).getImm() * OffsetStride;
762 int64_t MIOffsetInBytes =
764 MergeMIOffsetStride;
765 // Select final offset based on the offset order.
766 int64_t OffsetImm;
767 if (IOffsetInBytes > MIOffsetInBytes)
768 OffsetImm = MIOffsetInBytes;
769 else
770 OffsetImm = IOffsetInBytes;
771
772 int NewOpcode = getMatchingWideOpcode(Opc);
773 bool FinalIsScaled = !TII->hasUnscaledLdStOffset(NewOpcode);
774
775 // Adjust final offset if the result opcode is a scaled store.
776 if (FinalIsScaled) {
777 int NewOffsetStride = FinalIsScaled ? TII->getMemScale(NewOpcode) : 1;
778 assert(((OffsetImm % NewOffsetStride) == 0) &&
779 "Offset should be a multiple of the store memory scale");
780 OffsetImm = OffsetImm / NewOffsetStride;
781 }
782
783 // Construct the new instruction.
784 DebugLoc DL = I->getDebugLoc();
785 MachineBasicBlock *MBB = I->getParent();
787 MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(getMatchingWideOpcode(Opc)))
788 .addReg(isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR)
789 .add(BaseRegOp)
790 .addImm(OffsetImm)
791 .cloneMergedMemRefs({&*I, &*MergeMI})
792 .setMIFlags(I->mergeFlagsWith(*MergeMI));
793 (void)MIB;
794
795 LLVM_DEBUG(dbgs() << "Creating wider store. Replacing instructions:\n ");
796 LLVM_DEBUG(I->print(dbgs()));
797 LLVM_DEBUG(dbgs() << " ");
798 LLVM_DEBUG(MergeMI->print(dbgs()));
799 LLVM_DEBUG(dbgs() << " with instruction:\n ");
800 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
801 LLVM_DEBUG(dbgs() << "\n");
802
803 // Erase the old instructions.
804 I->eraseFromParent();
805 MergeMI->eraseFromParent();
806 return NextI;
807}
808
809// Apply Fn to all instructions between MI and the beginning of the block, until
810// a def for DefReg is reached. Returns true, iff Fn returns true for all
811// visited instructions. Stop after visiting Limit iterations.
813 const TargetRegisterInfo *TRI, unsigned Limit,
814 std::function<bool(MachineInstr &, bool)> &Fn) {
815 auto MBB = MI.getParent();
816 for (MachineInstr &I :
817 instructionsWithoutDebug(MI.getReverseIterator(), MBB->instr_rend())) {
818 if (!Limit)
819 return false;
820 --Limit;
821
822 bool isDef = any_of(I.operands(), [DefReg, TRI](MachineOperand &MOP) {
823 return MOP.isReg() && MOP.isDef() && !MOP.isDebug() && MOP.getReg() &&
824 TRI->regsOverlap(MOP.getReg(), DefReg);
825 });
826 if (!Fn(I, isDef))
827 return false;
828 if (isDef)
829 break;
830 }
831 return true;
832}
833
835 const TargetRegisterInfo *TRI) {
836
837 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
838 if (MOP.isReg() && MOP.isKill())
839 Units.removeReg(MOP.getReg());
840
841 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
842 if (MOP.isReg() && !MOP.isKill())
843 Units.addReg(MOP.getReg());
844}
845
847AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
849 const LdStPairFlags &Flags) {
850 MachineBasicBlock::iterator E = I->getParent()->end();
852 // If NextI is the second of the two instructions to be merged, we need
853 // to skip one further. Either way we merge will invalidate the iterator,
854 // and we don't need to scan the new instruction, as it's a pairwise
855 // instruction, which we're not considering for further action anyway.
856 if (NextI == Paired)
857 NextI = next_nodbg(NextI, E);
858
859 int SExtIdx = Flags.getSExtIdx();
860 unsigned Opc =
861 SExtIdx == -1 ? I->getOpcode() : getMatchingNonSExtOpcode(I->getOpcode());
862 bool IsUnscaled = TII->hasUnscaledLdStOffset(Opc);
863 int OffsetStride = IsUnscaled ? TII->getMemScale(*I) : 1;
864
865 bool MergeForward = Flags.getMergeForward();
866
867 std::optional<MCPhysReg> RenameReg = Flags.getRenameReg();
868 if (MergeForward && RenameReg) {
869 MCRegister RegToRename = getLdStRegOp(*I).getReg();
870 DefinedInBB.addReg(*RenameReg);
871
872 // Return the sub/super register for RenameReg, matching the size of
873 // OriginalReg.
874 auto GetMatchingSubReg = [this,
875 RenameReg](MCPhysReg OriginalReg) -> MCPhysReg {
876 for (MCPhysReg SubOrSuper : TRI->sub_and_superregs_inclusive(*RenameReg))
877 if (TRI->getMinimalPhysRegClass(OriginalReg) ==
878 TRI->getMinimalPhysRegClass(SubOrSuper))
879 return SubOrSuper;
880 llvm_unreachable("Should have found matching sub or super register!");
881 };
882
883 std::function<bool(MachineInstr &, bool)> UpdateMIs =
884 [this, RegToRename, GetMatchingSubReg](MachineInstr &MI, bool IsDef) {
885 if (IsDef) {
886 bool SeenDef = false;
887 for (auto &MOP : MI.operands()) {
888 // Rename the first explicit definition and all implicit
889 // definitions matching RegToRename.
890 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
891 (!SeenDef || (MOP.isDef() && MOP.isImplicit())) &&
892 TRI->regsOverlap(MOP.getReg(), RegToRename)) {
893 assert((MOP.isImplicit() ||
894 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
895 "Need renamable operands");
896 MOP.setReg(GetMatchingSubReg(MOP.getReg()));
897 SeenDef = true;
898 }
899 }
900 } else {
901 for (auto &MOP : MI.operands()) {
902 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
903 TRI->regsOverlap(MOP.getReg(), RegToRename)) {
904 assert((MOP.isImplicit() ||
905 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
906 "Need renamable operands");
907 MOP.setReg(GetMatchingSubReg(MOP.getReg()));
908 }
909 }
910 }
911 LLVM_DEBUG(dbgs() << "Renamed " << MI << "\n");
912 return true;
913 };
914 forAllMIsUntilDef(*I, RegToRename, TRI, LdStLimit, UpdateMIs);
915
916#if !defined(NDEBUG)
917 // Make sure the register used for renaming is not used between the paired
918 // instructions. That would trash the content before the new paired
919 // instruction.
920 for (auto &MI :
922 std::next(I), std::next(Paired)))
923 assert(all_of(MI.operands(),
924 [this, &RenameReg](const MachineOperand &MOP) {
925 return !MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
926 MOP.isUndef() ||
927 !TRI->regsOverlap(MOP.getReg(), *RenameReg);
928 }) &&
929 "Rename register used between paired instruction, trashing the "
930 "content");
931#endif
932 }
933
934 // Insert our new paired instruction after whichever of the paired
935 // instructions MergeForward indicates.
936 MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
937 // Also based on MergeForward is from where we copy the base register operand
938 // so we get the flags compatible with the input code.
939 const MachineOperand &BaseRegOp =
940 MergeForward ? AArch64InstrInfo::getLdStBaseOp(*Paired)
941 : AArch64InstrInfo::getLdStBaseOp(*I);
942
944 int PairedOffset = AArch64InstrInfo::getLdStOffsetOp(*Paired).getImm();
945 bool PairedIsUnscaled = TII->hasUnscaledLdStOffset(Paired->getOpcode());
946 if (IsUnscaled != PairedIsUnscaled) {
947 // We're trying to pair instructions that differ in how they are scaled. If
948 // I is scaled then scale the offset of Paired accordingly. Otherwise, do
949 // the opposite (i.e., make Paired's offset unscaled).
950 int MemSize = TII->getMemScale(*Paired);
951 if (PairedIsUnscaled) {
952 // If the unscaled offset isn't a multiple of the MemSize, we can't
953 // pair the operations together.
954 assert(!(PairedOffset % TII->getMemScale(*Paired)) &&
955 "Offset should be a multiple of the stride!");
956 PairedOffset /= MemSize;
957 } else {
958 PairedOffset *= MemSize;
959 }
960 }
961
962 // Which register is Rt and which is Rt2 depends on the offset order.
963 // However, for pre load/stores the Rt should be the one of the pre
964 // load/store.
965 MachineInstr *RtMI, *Rt2MI;
966 if (Offset == PairedOffset + OffsetStride &&
968 RtMI = &*Paired;
969 Rt2MI = &*I;
970 // Here we swapped the assumption made for SExtIdx.
971 // I.e., we turn ldp I, Paired into ldp Paired, I.
972 // Update the index accordingly.
973 if (SExtIdx != -1)
974 SExtIdx = (SExtIdx + 1) % 2;
975 } else {
976 RtMI = &*I;
977 Rt2MI = &*Paired;
978 }
979 int OffsetImm = AArch64InstrInfo::getLdStOffsetOp(*RtMI).getImm();
980 // Scale the immediate offset, if necessary.
981 if (TII->hasUnscaledLdStOffset(RtMI->getOpcode())) {
982 assert(!(OffsetImm % TII->getMemScale(*RtMI)) &&
983 "Unscaled offset cannot be scaled.");
984 OffsetImm /= TII->getMemScale(*RtMI);
985 }
986
987 // Construct the new instruction.
989 DebugLoc DL = I->getDebugLoc();
990 MachineBasicBlock *MBB = I->getParent();
991 MachineOperand RegOp0 = getLdStRegOp(*RtMI);
992 MachineOperand RegOp1 = getLdStRegOp(*Rt2MI);
993 // Kill flags may become invalid when moving stores for pairing.
994 if (RegOp0.isUse()) {
995 if (!MergeForward) {
996 // Clear kill flags on store if moving upwards. Example:
997 // STRWui %w0, ...
998 // USE %w1
999 // STRWui kill %w1 ; need to clear kill flag when moving STRWui upwards
1000 RegOp0.setIsKill(false);
1001 RegOp1.setIsKill(false);
1002 } else {
1003 // Clear kill flags of the first stores register. Example:
1004 // STRWui %w1, ...
1005 // USE kill %w1 ; need to clear kill flag when moving STRWui downwards
1006 // STRW %w0
1008 for (MachineInstr &MI : make_range(std::next(I), Paired))
1009 MI.clearRegisterKills(Reg, TRI);
1010 }
1011 }
1012
1013 unsigned int MatchPairOpcode = getMatchingPairOpcode(Opc);
1014 MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(MatchPairOpcode));
1015
1016 // Adds the pre-index operand for pre-indexed ld/st pairs.
1017 if (AArch64InstrInfo::isPreLdSt(*RtMI))
1018 MIB.addReg(BaseRegOp.getReg(), RegState::Define);
1019
1020 MIB.add(RegOp0)
1021 .add(RegOp1)
1022 .add(BaseRegOp)
1023 .addImm(OffsetImm)
1024 .cloneMergedMemRefs({&*I, &*Paired})
1025 .setMIFlags(I->mergeFlagsWith(*Paired));
1026
1027 (void)MIB;
1028
1029 LLVM_DEBUG(
1030 dbgs() << "Creating pair load/store. Replacing instructions:\n ");
1031 LLVM_DEBUG(I->print(dbgs()));
1032 LLVM_DEBUG(dbgs() << " ");
1033 LLVM_DEBUG(Paired->print(dbgs()));
1034 LLVM_DEBUG(dbgs() << " with instruction:\n ");
1035 if (SExtIdx != -1) {
1036 // Generate the sign extension for the proper result of the ldp.
1037 // I.e., with X1, that would be:
1038 // %w1 = KILL %w1, implicit-def %x1
1039 // %x1 = SBFMXri killed %x1, 0, 31
1040 MachineOperand &DstMO = MIB->getOperand(SExtIdx);
1041 // Right now, DstMO has the extended register, since it comes from an
1042 // extended opcode.
1043 Register DstRegX = DstMO.getReg();
1044 // Get the W variant of that register.
1045 Register DstRegW = TRI->getSubReg(DstRegX, AArch64::sub_32);
1046 // Update the result of LDP to use the W instead of the X variant.
1047 DstMO.setReg(DstRegW);
1048 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1049 LLVM_DEBUG(dbgs() << "\n");
1050 // Make the machine verifier happy by providing a definition for
1051 // the X register.
1052 // Insert this definition right after the generated LDP, i.e., before
1053 // InsertionPoint.
1054 MachineInstrBuilder MIBKill =
1055 BuildMI(*MBB, InsertionPoint, DL, TII->get(TargetOpcode::KILL), DstRegW)
1056 .addReg(DstRegW)
1057 .addReg(DstRegX, RegState::Define);
1058 MIBKill->getOperand(2).setImplicit();
1059 // Create the sign extension.
1060 MachineInstrBuilder MIBSXTW =
1061 BuildMI(*MBB, InsertionPoint, DL, TII->get(AArch64::SBFMXri), DstRegX)
1062 .addReg(DstRegX)
1063 .addImm(0)
1064 .addImm(31);
1065 (void)MIBSXTW;
1066 LLVM_DEBUG(dbgs() << " Extend operand:\n ");
1067 LLVM_DEBUG(((MachineInstr *)MIBSXTW)->print(dbgs()));
1068 } else {
1069 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1070 }
1071 LLVM_DEBUG(dbgs() << "\n");
1072
1073 if (MergeForward)
1074 for (const MachineOperand &MOP : phys_regs_and_masks(*I))
1075 if (MOP.isReg() && MOP.isKill())
1076 DefinedInBB.addReg(MOP.getReg());
1077
1078 // Erase the old instructions.
1079 I->eraseFromParent();
1080 Paired->eraseFromParent();
1081
1082 return NextI;
1083}
1084
1086AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
1089 next_nodbg(LoadI, LoadI->getParent()->end());
1090
1091 int LoadSize = TII->getMemScale(*LoadI);
1092 int StoreSize = TII->getMemScale(*StoreI);
1093 Register LdRt = getLdStRegOp(*LoadI).getReg();
1094 const MachineOperand &StMO = getLdStRegOp(*StoreI);
1095 Register StRt = getLdStRegOp(*StoreI).getReg();
1096 bool IsStoreXReg = TRI->getRegClass(AArch64::GPR64RegClassID)->contains(StRt);
1097
1098 assert((IsStoreXReg ||
1099 TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) &&
1100 "Unexpected RegClass");
1101
1102 MachineInstr *BitExtMI;
1103 if (LoadSize == StoreSize && (LoadSize == 4 || LoadSize == 8)) {
1104 // Remove the load, if the destination register of the loads is the same
1105 // register for stored value.
1106 if (StRt == LdRt && LoadSize == 8) {
1107 for (MachineInstr &MI : make_range(StoreI->getIterator(),
1108 LoadI->getIterator())) {
1109 if (MI.killsRegister(StRt, TRI)) {
1110 MI.clearRegisterKills(StRt, TRI);
1111 break;
1112 }
1113 }
1114 LLVM_DEBUG(dbgs() << "Remove load instruction:\n ");
1115 LLVM_DEBUG(LoadI->print(dbgs()));
1116 LLVM_DEBUG(dbgs() << "\n");
1117 LoadI->eraseFromParent();
1118 return NextI;
1119 }
1120 // Replace the load with a mov if the load and store are in the same size.
1121 BitExtMI =
1122 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1123 TII->get(IsStoreXReg ? AArch64::ORRXrs : AArch64::ORRWrs), LdRt)
1124 .addReg(IsStoreXReg ? AArch64::XZR : AArch64::WZR)
1125 .add(StMO)
1127 .setMIFlags(LoadI->getFlags());
1128 } else {
1129 // FIXME: Currently we disable this transformation in big-endian targets as
1130 // performance and correctness are verified only in little-endian.
1131 if (!Subtarget->isLittleEndian())
1132 return NextI;
1133 bool IsUnscaled = TII->hasUnscaledLdStOffset(*LoadI);
1134 assert(IsUnscaled == TII->hasUnscaledLdStOffset(*StoreI) &&
1135 "Unsupported ld/st match");
1136 assert(LoadSize <= StoreSize && "Invalid load size");
1137 int UnscaledLdOffset =
1138 IsUnscaled
1140 : AArch64InstrInfo::getLdStOffsetOp(*LoadI).getImm() * LoadSize;
1141 int UnscaledStOffset =
1142 IsUnscaled
1144 : AArch64InstrInfo::getLdStOffsetOp(*StoreI).getImm() * StoreSize;
1145 int Width = LoadSize * 8;
1146 Register DestReg =
1147 IsStoreXReg ? Register(TRI->getMatchingSuperReg(
1148 LdRt, AArch64::sub_32, &AArch64::GPR64RegClass))
1149 : LdRt;
1150
1151 assert((UnscaledLdOffset >= UnscaledStOffset &&
1152 (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) &&
1153 "Invalid offset");
1154
1155 int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset);
1156 int Imms = Immr + Width - 1;
1157 if (UnscaledLdOffset == UnscaledStOffset) {
1158 uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12) // N
1159 | ((Immr) << 6) // immr
1160 | ((Imms) << 0) // imms
1161 ;
1162
1163 BitExtMI =
1164 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1165 TII->get(IsStoreXReg ? AArch64::ANDXri : AArch64::ANDWri),
1166 DestReg)
1167 .add(StMO)
1168 .addImm(AndMaskEncoded)
1169 .setMIFlags(LoadI->getFlags());
1170 } else {
1171 BitExtMI =
1172 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1173 TII->get(IsStoreXReg ? AArch64::UBFMXri : AArch64::UBFMWri),
1174 DestReg)
1175 .add(StMO)
1176 .addImm(Immr)
1177 .addImm(Imms)
1178 .setMIFlags(LoadI->getFlags());
1179 }
1180 }
1181
1182 // Clear kill flags between store and load.
1183 for (MachineInstr &MI : make_range(StoreI->getIterator(),
1184 BitExtMI->getIterator()))
1185 if (MI.killsRegister(StRt, TRI)) {
1186 MI.clearRegisterKills(StRt, TRI);
1187 break;
1188 }
1189
1190 LLVM_DEBUG(dbgs() << "Promoting load by replacing :\n ");
1191 LLVM_DEBUG(StoreI->print(dbgs()));
1192 LLVM_DEBUG(dbgs() << " ");
1193 LLVM_DEBUG(LoadI->print(dbgs()));
1194 LLVM_DEBUG(dbgs() << " with instructions:\n ");
1195 LLVM_DEBUG(StoreI->print(dbgs()));
1196 LLVM_DEBUG(dbgs() << " ");
1197 LLVM_DEBUG((BitExtMI)->print(dbgs()));
1198 LLVM_DEBUG(dbgs() << "\n");
1199
1200 // Erase the old instructions.
1201 LoadI->eraseFromParent();
1202 return NextI;
1203}
1204
1205static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride) {
1206 // Convert the byte-offset used by unscaled into an "element" offset used
1207 // by the scaled pair load/store instructions.
1208 if (IsUnscaled) {
1209 // If the byte-offset isn't a multiple of the stride, there's no point
1210 // trying to match it.
1211 if (Offset % OffsetStride)
1212 return false;
1213 Offset /= OffsetStride;
1214 }
1215 return Offset <= 63 && Offset >= -64;
1216}
1217
1218// Do alignment, specialized to power of 2 and for signed ints,
1219// avoiding having to do a C-style cast from uint_64t to int when
1220// using alignTo from include/llvm/Support/MathExtras.h.
1221// FIXME: Move this function to include/MathExtras.h?
1222static int alignTo(int Num, int PowOf2) {
1223 return (Num + PowOf2 - 1) & ~(PowOf2 - 1);
1224}
1225
1226static bool mayAlias(MachineInstr &MIa,
1228 AliasAnalysis *AA) {
1229 for (MachineInstr *MIb : MemInsns)
1230 if (MIa.mayAlias(AA, *MIb, /*UseTBAA*/ false))
1231 return true;
1232
1233 return false;
1234}
1235
1236bool AArch64LoadStoreOpt::findMatchingStore(
1237 MachineBasicBlock::iterator I, unsigned Limit,
1239 MachineBasicBlock::iterator B = I->getParent()->begin();
1241 MachineInstr &LoadMI = *I;
1243
1244 // If the load is the first instruction in the block, there's obviously
1245 // not any matching store.
1246 if (MBBI == B)
1247 return false;
1248
1249 // Track which register units have been modified and used between the first
1250 // insn and the second insn.
1251 ModifiedRegUnits.clear();
1252 UsedRegUnits.clear();
1253
1254 unsigned Count = 0;
1255 do {
1256 MBBI = prev_nodbg(MBBI, B);
1257 MachineInstr &MI = *MBBI;
1258
1259 // Don't count transient instructions towards the search limit since there
1260 // may be different numbers of them if e.g. debug information is present.
1261 if (!MI.isTransient())
1262 ++Count;
1263
1264 // If the load instruction reads directly from the address to which the
1265 // store instruction writes and the stored value is not modified, we can
1266 // promote the load. Since we do not handle stores with pre-/post-index,
1267 // it's unnecessary to check if BaseReg is modified by the store itself.
1268 // Also we can't handle stores without an immediate offset operand,
1269 // while the operand might be the address for a global variable.
1270 if (MI.mayStore() && isMatchingStore(LoadMI, MI) &&
1273 isLdOffsetInRangeOfSt(LoadMI, MI, TII) &&
1274 ModifiedRegUnits.available(getLdStRegOp(MI).getReg())) {
1275 StoreI = MBBI;
1276 return true;
1277 }
1278
1279 if (MI.isCall())
1280 return false;
1281
1282 // Update modified / uses register units.
1283 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
1284
1285 // Otherwise, if the base register is modified, we have no match, so
1286 // return early.
1287 if (!ModifiedRegUnits.available(BaseReg))
1288 return false;
1289
1290 // If we encounter a store aliased with the load, return early.
1291 if (MI.mayStore() && LoadMI.mayAlias(AA, MI, /*UseTBAA*/ false))
1292 return false;
1293 } while (MBBI != B && Count < Limit);
1294 return false;
1295}
1296
1297static bool needsWinCFI(const MachineFunction *MF) {
1298 return MF->getTarget().getMCAsmInfo()->usesWindowsCFI() &&
1300}
1301
1302// Returns true if FirstMI and MI are candidates for merging or pairing.
1303// Otherwise, returns false.
1305 LdStPairFlags &Flags,
1306 const AArch64InstrInfo *TII) {
1307 // If this is volatile or if pairing is suppressed, not a candidate.
1308 if (MI.hasOrderedMemoryRef() || TII->isLdStPairSuppressed(MI))
1309 return false;
1310
1311 // We should have already checked FirstMI for pair suppression and volatility.
1312 assert(!FirstMI.hasOrderedMemoryRef() &&
1313 !TII->isLdStPairSuppressed(FirstMI) &&
1314 "FirstMI shouldn't get here if either of these checks are true.");
1315
1316 if (needsWinCFI(MI.getMF()) && (MI.getFlag(MachineInstr::FrameSetup) ||
1318 return false;
1319
1320 unsigned OpcA = FirstMI.getOpcode();
1321 unsigned OpcB = MI.getOpcode();
1322
1323 // Opcodes match: If the opcodes are pre ld/st there is nothing more to check.
1324 if (OpcA == OpcB)
1325 return !AArch64InstrInfo::isPreLdSt(FirstMI);
1326
1327 // Two pre ld/st of different opcodes cannot be merged either
1329 return false;
1330
1331 // Try to match a sign-extended load/store with a zero-extended load/store.
1332 bool IsValidLdStrOpc, PairIsValidLdStrOpc;
1333 unsigned NonSExtOpc = getMatchingNonSExtOpcode(OpcA, &IsValidLdStrOpc);
1334 assert(IsValidLdStrOpc &&
1335 "Given Opc should be a Load or Store with an immediate");
1336 // OpcA will be the first instruction in the pair.
1337 if (NonSExtOpc == getMatchingNonSExtOpcode(OpcB, &PairIsValidLdStrOpc)) {
1338 Flags.setSExtIdx(NonSExtOpc == (unsigned)OpcA ? 1 : 0);
1339 return true;
1340 }
1341
1342 // If the second instruction isn't even a mergable/pairable load/store, bail
1343 // out.
1344 if (!PairIsValidLdStrOpc)
1345 return false;
1346
1347 // FIXME: We don't support merging narrow stores with mixed scaled/unscaled
1348 // offsets.
1349 if (isNarrowStore(OpcA) || isNarrowStore(OpcB))
1350 return false;
1351
1352 // The STR<S,D,Q,W,X>pre - STR<S,D,Q,W,X>ui and
1353 // LDR<S,D,Q,W,X,SW>pre-LDR<S,D,Q,W,X,SW>ui
1354 // are candidate pairs that can be merged.
1355 if (isPreLdStPairCandidate(FirstMI, MI))
1356 return true;
1357
1358 // Try to match an unscaled load/store with a scaled load/store.
1359 return TII->hasUnscaledLdStOffset(OpcA) != TII->hasUnscaledLdStOffset(OpcB) &&
1361
1362 // FIXME: Can we also match a mixed sext/zext unscaled/scaled pair?
1363}
1364
1365static bool
1368 const TargetRegisterInfo *TRI) {
1369 if (!FirstMI.mayStore())
1370 return false;
1371
1372 // Check if we can find an unused register which we can use to rename
1373 // the register used by the first load/store.
1374 auto *RegClass = TRI->getMinimalPhysRegClass(getLdStRegOp(FirstMI).getReg());
1375 MachineFunction &MF = *FirstMI.getParent()->getParent();
1376 if (!RegClass || !MF.getRegInfo().tracksLiveness())
1377 return false;
1378
1379 auto RegToRename = getLdStRegOp(FirstMI).getReg();
1380 // For now, we only rename if the store operand gets killed at the store.
1381 if (!getLdStRegOp(FirstMI).isKill() &&
1382 !any_of(FirstMI.operands(),
1383 [TRI, RegToRename](const MachineOperand &MOP) {
1384 return MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1385 MOP.isImplicit() && MOP.isKill() &&
1386 TRI->regsOverlap(RegToRename, MOP.getReg());
1387 })) {
1388 LLVM_DEBUG(dbgs() << " Operand not killed at " << FirstMI << "\n");
1389 return false;
1390 }
1391 auto canRenameMOP = [TRI](const MachineOperand &MOP) {
1392 if (MOP.isReg()) {
1393 auto *RegClass = TRI->getMinimalPhysRegClass(MOP.getReg());
1394 // Renaming registers with multiple disjunct sub-registers (e.g. the
1395 // result of a LD3) means that all sub-registers are renamed, potentially
1396 // impacting other instructions we did not check. Bail out.
1397 // Note that this relies on the structure of the AArch64 register file. In
1398 // particular, a subregister cannot be written without overwriting the
1399 // whole register.
1400 if (RegClass->HasDisjunctSubRegs) {
1401 LLVM_DEBUG(
1402 dbgs()
1403 << " Cannot rename operands with multiple disjunct subregisters ("
1404 << MOP << ")\n");
1405 return false;
1406 }
1407 }
1408 return MOP.isImplicit() ||
1409 (MOP.isRenamable() && !MOP.isEarlyClobber() && !MOP.isTied());
1410 };
1411
1412 bool FoundDef = false;
1413
1414 // For each instruction between FirstMI and the previous def for RegToRename,
1415 // we
1416 // * check if we can rename RegToRename in this instruction
1417 // * collect the registers used and required register classes for RegToRename.
1418 std::function<bool(MachineInstr &, bool)> CheckMIs = [&](MachineInstr &MI,
1419 bool IsDef) {
1420 LLVM_DEBUG(dbgs() << "Checking " << MI << "\n");
1421 // Currently we do not try to rename across frame-setup instructions.
1422 if (MI.getFlag(MachineInstr::FrameSetup)) {
1423 LLVM_DEBUG(dbgs() << " Cannot rename framesetup instructions currently ("
1424 << MI << ")\n");
1425 return false;
1426 }
1427
1428 UsedInBetween.accumulate(MI);
1429
1430 // For a definition, check that we can rename the definition and exit the
1431 // loop.
1432 FoundDef = IsDef;
1433
1434 // For defs, check if we can rename the first def of RegToRename.
1435 if (FoundDef) {
1436 // For some pseudo instructions, we might not generate code in the end
1437 // (e.g. KILL) and we would end up without a correct def for the rename
1438 // register.
1439 // TODO: This might be overly conservative and we could handle those cases
1440 // in multiple ways:
1441 // 1. Insert an extra copy, to materialize the def.
1442 // 2. Skip pseudo-defs until we find an non-pseudo def.
1443 if (MI.isPseudo()) {
1444 LLVM_DEBUG(dbgs() << " Cannot rename pseudo instruction " << MI
1445 << "\n");
1446 return false;
1447 }
1448
1449 for (auto &MOP : MI.operands()) {
1450 if (!MOP.isReg() || !MOP.isDef() || MOP.isDebug() || !MOP.getReg() ||
1451 !TRI->regsOverlap(MOP.getReg(), RegToRename))
1452 continue;
1453 if (!canRenameMOP(MOP)) {
1455 << " Cannot rename " << MOP << " in " << MI << "\n");
1456 return false;
1457 }
1458 RequiredClasses.insert(TRI->getMinimalPhysRegClass(MOP.getReg()));
1459 }
1460 return true;
1461 } else {
1462 for (auto &MOP : MI.operands()) {
1463 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1464 !TRI->regsOverlap(MOP.getReg(), RegToRename))
1465 continue;
1466
1467 if (!canRenameMOP(MOP)) {
1469 << " Cannot rename " << MOP << " in " << MI << "\n");
1470 return false;
1471 }
1472 RequiredClasses.insert(TRI->getMinimalPhysRegClass(MOP.getReg()));
1473 }
1474 }
1475 return true;
1476 };
1477
1478 if (!forAllMIsUntilDef(FirstMI, RegToRename, TRI, LdStLimit, CheckMIs))
1479 return false;
1480
1481 if (!FoundDef) {
1482 LLVM_DEBUG(dbgs() << " Did not find definition for register in BB\n");
1483 return false;
1484 }
1485 return true;
1486}
1487
1488// Check if we can find a physical register for renaming \p Reg. This register
1489// must:
1490// * not be defined already in \p DefinedInBB; DefinedInBB must contain all
1491// defined registers up to the point where the renamed register will be used,
1492// * not used in \p UsedInBetween; UsedInBetween must contain all accessed
1493// registers in the range the rename register will be used,
1494// * is available in all used register classes (checked using RequiredClasses).
1495static std::optional<MCPhysReg> tryToFindRegisterToRename(
1496 const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB,
1497 LiveRegUnits &UsedInBetween,
1499 const TargetRegisterInfo *TRI) {
1501
1502 // Checks if any sub- or super-register of PR is callee saved.
1503 auto AnySubOrSuperRegCalleePreserved = [&MF, TRI](MCPhysReg PR) {
1504 return any_of(TRI->sub_and_superregs_inclusive(PR),
1505 [&MF, TRI](MCPhysReg SubOrSuper) {
1506 return TRI->isCalleeSavedPhysReg(SubOrSuper, MF);
1507 });
1508 };
1509
1510 // Check if PR or one of its sub- or super-registers can be used for all
1511 // required register classes.
1512 auto CanBeUsedForAllClasses = [&RequiredClasses, TRI](MCPhysReg PR) {
1513 return all_of(RequiredClasses, [PR, TRI](const TargetRegisterClass *C) {
1514 return any_of(TRI->sub_and_superregs_inclusive(PR),
1515 [C, TRI](MCPhysReg SubOrSuper) {
1516 return C == TRI->getMinimalPhysRegClass(SubOrSuper);
1517 });
1518 });
1519 };
1520
1521 auto *RegClass = TRI->getMinimalPhysRegClass(Reg);
1522 for (const MCPhysReg &PR : *RegClass) {
1523 if (DefinedInBB.available(PR) && UsedInBetween.available(PR) &&
1524 !RegInfo.isReserved(PR) && !AnySubOrSuperRegCalleePreserved(PR) &&
1525 CanBeUsedForAllClasses(PR)) {
1526 DefinedInBB.addReg(PR);
1527 LLVM_DEBUG(dbgs() << "Found rename register " << printReg(PR, TRI)
1528 << "\n");
1529 return {PR};
1530 }
1531 }
1532 LLVM_DEBUG(dbgs() << "No rename register found from "
1533 << TRI->getRegClassName(RegClass) << "\n");
1534 return std::nullopt;
1535}
1536
1537/// Scan the instructions looking for a load/store that can be combined with the
1538/// current instruction into a wider equivalent or a load/store pair.
1540AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
1541 LdStPairFlags &Flags, unsigned Limit,
1542 bool FindNarrowMerge) {
1543 MachineBasicBlock::iterator E = I->getParent()->end();
1545 MachineBasicBlock::iterator MBBIWithRenameReg;
1546 MachineInstr &FirstMI = *I;
1547 MBBI = next_nodbg(MBBI, E);
1548
1549 bool MayLoad = FirstMI.mayLoad();
1550 bool IsUnscaled = TII->hasUnscaledLdStOffset(FirstMI);
1551 Register Reg = getLdStRegOp(FirstMI).getReg();
1552 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(FirstMI).getReg();
1554 int OffsetStride = IsUnscaled ? TII->getMemScale(FirstMI) : 1;
1555 bool IsPromotableZeroStore = isPromotableZeroStoreInst(FirstMI);
1556
1557 std::optional<bool> MaybeCanRename;
1558 if (!EnableRenaming)
1559 MaybeCanRename = {false};
1560
1562 LiveRegUnits UsedInBetween;
1563 UsedInBetween.init(*TRI);
1564
1565 Flags.clearRenameReg();
1566
1567 // Track which register units have been modified and used between the first
1568 // insn (inclusive) and the second insn.
1569 ModifiedRegUnits.clear();
1570 UsedRegUnits.clear();
1571
1572 // Remember any instructions that read/write memory between FirstMI and MI.
1574
1575 for (unsigned Count = 0; MBBI != E && Count < Limit;
1576 MBBI = next_nodbg(MBBI, E)) {
1577 MachineInstr &MI = *MBBI;
1578
1579 UsedInBetween.accumulate(MI);
1580
1581 // Don't count transient instructions towards the search limit since there
1582 // may be different numbers of them if e.g. debug information is present.
1583 if (!MI.isTransient())
1584 ++Count;
1585
1586 Flags.setSExtIdx(-1);
1587 if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) &&
1589 assert(MI.mayLoadOrStore() && "Expected memory operation.");
1590 // If we've found another instruction with the same opcode, check to see
1591 // if the base and offset are compatible with our starting instruction.
1592 // These instructions all have scaled immediate operands, so we just
1593 // check for +1/-1. Make sure to check the new instruction offset is
1594 // actually an immediate and not a symbolic reference destined for
1595 // a relocation.
1598 bool MIIsUnscaled = TII->hasUnscaledLdStOffset(MI);
1599 if (IsUnscaled != MIIsUnscaled) {
1600 // We're trying to pair instructions that differ in how they are scaled.
1601 // If FirstMI is scaled then scale the offset of MI accordingly.
1602 // Otherwise, do the opposite (i.e., make MI's offset unscaled).
1603 int MemSize = TII->getMemScale(MI);
1604 if (MIIsUnscaled) {
1605 // If the unscaled offset isn't a multiple of the MemSize, we can't
1606 // pair the operations together: bail and keep looking.
1607 if (MIOffset % MemSize) {
1608 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
1609 UsedRegUnits, TRI);
1610 MemInsns.push_back(&MI);
1611 continue;
1612 }
1613 MIOffset /= MemSize;
1614 } else {
1615 MIOffset *= MemSize;
1616 }
1617 }
1618
1619 bool IsPreLdSt = isPreLdStPairCandidate(FirstMI, MI);
1620
1621 if (BaseReg == MIBaseReg) {
1622 // If the offset of the second ld/st is not equal to the size of the
1623 // destination register it can’t be paired with a pre-index ld/st
1624 // pair. Additionally if the base reg is used or modified the operations
1625 // can't be paired: bail and keep looking.
1626 if (IsPreLdSt) {
1627 bool IsOutOfBounds = MIOffset != TII->getMemScale(MI);
1628 bool IsBaseRegUsed = !UsedRegUnits.available(
1630 bool IsBaseRegModified = !ModifiedRegUnits.available(
1632 // If the stored value and the address of the second instruction is
1633 // the same, it needs to be using the updated register and therefore
1634 // it must not be folded.
1635 bool IsMIRegTheSame =
1636 TRI->regsOverlap(getLdStRegOp(MI).getReg(),
1638 if (IsOutOfBounds || IsBaseRegUsed || IsBaseRegModified ||
1639 IsMIRegTheSame) {
1640 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
1641 UsedRegUnits, TRI);
1642 MemInsns.push_back(&MI);
1643 continue;
1644 }
1645 } else {
1646 if ((Offset != MIOffset + OffsetStride) &&
1647 (Offset + OffsetStride != MIOffset)) {
1648 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
1649 UsedRegUnits, TRI);
1650 MemInsns.push_back(&MI);
1651 continue;
1652 }
1653 }
1654
1655 int MinOffset = Offset < MIOffset ? Offset : MIOffset;
1656 if (FindNarrowMerge) {
1657 // If the alignment requirements of the scaled wide load/store
1658 // instruction can't express the offset of the scaled narrow input,
1659 // bail and keep looking. For promotable zero stores, allow only when
1660 // the stored value is the same (i.e., WZR).
1661 if ((!IsUnscaled && alignTo(MinOffset, 2) != MinOffset) ||
1662 (IsPromotableZeroStore && Reg != getLdStRegOp(MI).getReg())) {
1663 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
1664 UsedRegUnits, TRI);
1665 MemInsns.push_back(&MI);
1666 continue;
1667 }
1668 } else {
1669 // Pairwise instructions have a 7-bit signed offset field. Single
1670 // insns have a 12-bit unsigned offset field. If the resultant
1671 // immediate offset of merging these instructions is out of range for
1672 // a pairwise instruction, bail and keep looking.
1673 if (!inBoundsForPair(IsUnscaled, MinOffset, OffsetStride)) {
1674 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
1675 UsedRegUnits, TRI);
1676 MemInsns.push_back(&MI);
1677 continue;
1678 }
1679 // If the alignment requirements of the paired (scaled) instruction
1680 // can't express the offset of the unscaled input, bail and keep
1681 // looking.
1682 if (IsUnscaled && (alignTo(MinOffset, OffsetStride) != MinOffset)) {
1683 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
1684 UsedRegUnits, TRI);
1685 MemInsns.push_back(&MI);
1686 continue;
1687 }
1688 }
1689 // If the destination register of one load is the same register or a
1690 // sub/super register of the other load, bail and keep looking. A
1691 // load-pair instruction with both destination registers the same is
1692 // UNPREDICTABLE and will result in an exception.
1693 if (MayLoad &&
1694 TRI->isSuperOrSubRegisterEq(Reg, getLdStRegOp(MI).getReg())) {
1695 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
1696 TRI);
1697 MemInsns.push_back(&MI);
1698 continue;
1699 }
1700
1701 // If the BaseReg has been modified, then we cannot do the optimization.
1702 // For example, in the following pattern
1703 // ldr x1 [x2]
1704 // ldr x2 [x3]
1705 // ldr x4 [x2, #8],
1706 // the first and third ldr cannot be converted to ldp x1, x4, [x2]
1707 if (!ModifiedRegUnits.available(BaseReg))
1708 return E;
1709
1710 // If the Rt of the second instruction was not modified or used between
1711 // the two instructions and none of the instructions between the second
1712 // and first alias with the second, we can combine the second into the
1713 // first.
1714 if (ModifiedRegUnits.available(getLdStRegOp(MI).getReg()) &&
1715 !(MI.mayLoad() &&
1716 !UsedRegUnits.available(getLdStRegOp(MI).getReg())) &&
1717 !mayAlias(MI, MemInsns, AA)) {
1718
1719 Flags.setMergeForward(false);
1720 Flags.clearRenameReg();
1721 return MBBI;
1722 }
1723
1724 // Likewise, if the Rt of the first instruction is not modified or used
1725 // between the two instructions and none of the instructions between the
1726 // first and the second alias with the first, we can combine the first
1727 // into the second.
1728 if (!(MayLoad &&
1729 !UsedRegUnits.available(getLdStRegOp(FirstMI).getReg())) &&
1730 !mayAlias(FirstMI, MemInsns, AA)) {
1731
1732 if (ModifiedRegUnits.available(getLdStRegOp(FirstMI).getReg())) {
1733 Flags.setMergeForward(true);
1734 Flags.clearRenameReg();
1735 return MBBI;
1736 }
1737
1738 if (DebugCounter::shouldExecute(RegRenamingCounter)) {
1739 if (!MaybeCanRename)
1740 MaybeCanRename = {canRenameUpToDef(FirstMI, UsedInBetween,
1741 RequiredClasses, TRI)};
1742
1743 if (*MaybeCanRename) {
1744 std::optional<MCPhysReg> MaybeRenameReg =
1746 Reg, DefinedInBB, UsedInBetween,
1747 RequiredClasses, TRI);
1748 if (MaybeRenameReg) {
1749 Flags.setRenameReg(*MaybeRenameReg);
1750 Flags.setMergeForward(true);
1751 MBBIWithRenameReg = MBBI;
1752 }
1753 }
1754 }
1755 }
1756 // Unable to combine these instructions due to interference in between.
1757 // Keep looking.
1758 }
1759 }
1760
1761 if (Flags.getRenameReg())
1762 return MBBIWithRenameReg;
1763
1764 // If the instruction wasn't a matching load or store. Stop searching if we
1765 // encounter a call instruction that might modify memory.
1766 if (MI.isCall())
1767 return E;
1768
1769 // Update modified / uses register units.
1770 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
1771
1772 // Otherwise, if the base register is modified, we have no match, so
1773 // return early.
1774 if (!ModifiedRegUnits.available(BaseReg))
1775 return E;
1776
1777 // Update list of instructions that read/write memory.
1778 if (MI.mayLoadOrStore())
1779 MemInsns.push_back(&MI);
1780 }
1781 return E;
1782}
1783
1786 auto End = MI.getParent()->end();
1787 if (MaybeCFI == End ||
1788 MaybeCFI->getOpcode() != TargetOpcode::CFI_INSTRUCTION ||
1789 !(MI.getFlag(MachineInstr::FrameSetup) ||
1790 MI.getFlag(MachineInstr::FrameDestroy)) ||
1791 AArch64InstrInfo::getLdStBaseOp(MI).getReg() != AArch64::SP)
1792 return End;
1793
1794 const MachineFunction &MF = *MI.getParent()->getParent();
1795 unsigned CFIIndex = MaybeCFI->getOperand(0).getCFIIndex();
1796 const MCCFIInstruction &CFI = MF.getFrameInstructions()[CFIIndex];
1797 switch (CFI.getOperation()) {
1800 return MaybeCFI;
1801 default:
1802 return End;
1803 }
1804}
1805
1807AArch64LoadStoreOpt::mergeUpdateInsn(MachineBasicBlock::iterator I,
1809 bool IsPreIdx) {
1810 assert((Update->getOpcode() == AArch64::ADDXri ||
1811 Update->getOpcode() == AArch64::SUBXri) &&
1812 "Unexpected base register update instruction to merge!");
1813 MachineBasicBlock::iterator E = I->getParent()->end();
1815
1816 // If updating the SP and the following instruction is CFA offset related CFI
1817 // instruction move it after the merged instruction.
1819 IsPreIdx ? maybeMoveCFI(*Update, next_nodbg(Update, E)) : E;
1820
1821 // Return the instruction following the merged instruction, which is
1822 // the instruction following our unmerged load. Unless that's the add/sub
1823 // instruction we're merging, in which case it's the one after that.
1824 if (NextI == Update)
1825 NextI = next_nodbg(NextI, E);
1826
1827 int Value = Update->getOperand(2).getImm();
1828 assert(AArch64_AM::getShiftValue(Update->getOperand(3).getImm()) == 0 &&
1829 "Can't merge 1 << 12 offset into pre-/post-indexed load / store");
1830 if (Update->getOpcode() == AArch64::SUBXri)
1831 Value = -Value;
1832
1833 unsigned NewOpc = IsPreIdx ? getPreIndexedOpcode(I->getOpcode())
1836 int Scale, MinOffset, MaxOffset;
1837 getPrePostIndexedMemOpInfo(*I, Scale, MinOffset, MaxOffset);
1839 // Non-paired instruction.
1840 MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
1841 .add(getLdStRegOp(*Update))
1842 .add(getLdStRegOp(*I))
1844 .addImm(Value / Scale)
1845 .setMemRefs(I->memoperands())
1846 .setMIFlags(I->mergeFlagsWith(*Update));
1847 } else {
1848 // Paired instruction.
1849 MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
1850 .add(getLdStRegOp(*Update))
1851 .add(getLdStRegOp(*I, 0))
1852 .add(getLdStRegOp(*I, 1))
1854 .addImm(Value / Scale)
1855 .setMemRefs(I->memoperands())
1856 .setMIFlags(I->mergeFlagsWith(*Update));
1857 }
1858 if (CFI != E) {
1859 MachineBasicBlock *MBB = I->getParent();
1860 MBB->splice(std::next(MIB.getInstr()->getIterator()), MBB, CFI);
1861 }
1862
1863 if (IsPreIdx) {
1864 ++NumPreFolded;
1865 LLVM_DEBUG(dbgs() << "Creating pre-indexed load/store.");
1866 } else {
1867 ++NumPostFolded;
1868 LLVM_DEBUG(dbgs() << "Creating post-indexed load/store.");
1869 }
1870 LLVM_DEBUG(dbgs() << " Replacing instructions:\n ");
1871 LLVM_DEBUG(I->print(dbgs()));
1872 LLVM_DEBUG(dbgs() << " ");
1873 LLVM_DEBUG(Update->print(dbgs()));
1874 LLVM_DEBUG(dbgs() << " with instruction:\n ");
1875 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1876 LLVM_DEBUG(dbgs() << "\n");
1877
1878 // Erase the old instructions for the block.
1879 I->eraseFromParent();
1880 Update->eraseFromParent();
1881
1882 return NextI;
1883}
1884
1885bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
1887 unsigned BaseReg, int Offset) {
1888 switch (MI.getOpcode()) {
1889 default:
1890 break;
1891 case AArch64::SUBXri:
1892 case AArch64::ADDXri:
1893 // Make sure it's a vanilla immediate operand, not a relocation or
1894 // anything else we can't handle.
1895 if (!MI.getOperand(2).isImm())
1896 break;
1897 // Watch out for 1 << 12 shifted value.
1898 if (AArch64_AM::getShiftValue(MI.getOperand(3).getImm()))
1899 break;
1900
1901 // The update instruction source and destination register must be the
1902 // same as the load/store base register.
1903 if (MI.getOperand(0).getReg() != BaseReg ||
1904 MI.getOperand(1).getReg() != BaseReg)
1905 break;
1906
1907 int UpdateOffset = MI.getOperand(2).getImm();
1908 if (MI.getOpcode() == AArch64::SUBXri)
1909 UpdateOffset = -UpdateOffset;
1910
1911 // The immediate must be a multiple of the scaling factor of the pre/post
1912 // indexed instruction.
1913 int Scale, MinOffset, MaxOffset;
1914 getPrePostIndexedMemOpInfo(MemMI, Scale, MinOffset, MaxOffset);
1915 if (UpdateOffset % Scale != 0)
1916 break;
1917
1918 // Scaled offset must fit in the instruction immediate.
1919 int ScaledOffset = UpdateOffset / Scale;
1920 if (ScaledOffset > MaxOffset || ScaledOffset < MinOffset)
1921 break;
1922
1923 // If we have a non-zero Offset, we check that it matches the amount
1924 // we're adding to the register.
1925 if (!Offset || Offset == UpdateOffset)
1926 return true;
1927 break;
1928 }
1929 return false;
1930}
1931
1932MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
1933 MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
1934 MachineBasicBlock::iterator E = I->getParent()->end();
1935 MachineInstr &MemMI = *I;
1937
1939 int MIUnscaledOffset = AArch64InstrInfo::getLdStOffsetOp(MemMI).getImm() *
1940 TII->getMemScale(MemMI);
1941
1942 // Scan forward looking for post-index opportunities. Updating instructions
1943 // can't be formed if the memory instruction doesn't have the offset we're
1944 // looking for.
1945 if (MIUnscaledOffset != UnscaledOffset)
1946 return E;
1947
1948 // If the base register overlaps a source/destination register, we can't
1949 // merge the update. This does not apply to tag store instructions which
1950 // ignore the address part of the source register.
1951 // This does not apply to STGPi as well, which does not have unpredictable
1952 // behavior in this case unlike normal stores, and always performs writeback
1953 // after reading the source register value.
1954 if (!isTagStore(MemMI) && MemMI.getOpcode() != AArch64::STGPi) {
1955 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MemMI);
1956 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
1957 Register DestReg = getLdStRegOp(MemMI, i).getReg();
1958 if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg))
1959 return E;
1960 }
1961 }
1962
1963 // Track which register units have been modified and used between the first
1964 // insn (inclusive) and the second insn.
1965 ModifiedRegUnits.clear();
1966 UsedRegUnits.clear();
1967 MBBI = next_nodbg(MBBI, E);
1968
1969 // We can't post-increment the stack pointer if any instruction between
1970 // the memory access (I) and the increment (MBBI) can access the memory
1971 // region defined by [SP, MBBI].
1972 const bool BaseRegSP = BaseReg == AArch64::SP;
1973 if (BaseRegSP && needsWinCFI(I->getMF())) {
1974 // FIXME: For now, we always block the optimization over SP in windows
1975 // targets as it requires to adjust the unwind/debug info, messing up
1976 // the unwind info can actually cause a miscompile.
1977 return E;
1978 }
1979
1980 for (unsigned Count = 0; MBBI != E && Count < Limit;
1981 MBBI = next_nodbg(MBBI, E)) {
1982 MachineInstr &MI = *MBBI;
1983
1984 // Don't count transient instructions towards the search limit since there
1985 // may be different numbers of them if e.g. debug information is present.
1986 if (!MI.isTransient())
1987 ++Count;
1988
1989 // If we found a match, return it.
1990 if (isMatchingUpdateInsn(*I, MI, BaseReg, UnscaledOffset))
1991 return MBBI;
1992
1993 // Update the status of what the instruction clobbered and used.
1994 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
1995
1996 // Otherwise, if the base register is used or modified, we have no match, so
1997 // return early.
1998 // If we are optimizing SP, do not allow instructions that may load or store
1999 // in between the load and the optimized value update.
2000 if (!ModifiedRegUnits.available(BaseReg) ||
2001 !UsedRegUnits.available(BaseReg) ||
2002 (BaseRegSP && MBBI->mayLoadOrStore()))
2003 return E;
2004 }
2005 return E;
2006}
2007
2008MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
2009 MachineBasicBlock::iterator I, unsigned Limit) {
2010 MachineBasicBlock::iterator B = I->getParent()->begin();
2011 MachineBasicBlock::iterator E = I->getParent()->end();
2012 MachineInstr &MemMI = *I;
2014 MachineFunction &MF = *MemMI.getMF();
2015
2018
2019 // If the load/store is the first instruction in the block, there's obviously
2020 // not any matching update. Ditto if the memory offset isn't zero.
2021 if (MBBI == B || Offset != 0)
2022 return E;
2023 // If the base register overlaps a destination register, we can't
2024 // merge the update.
2025 if (!isTagStore(MemMI)) {
2026 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MemMI);
2027 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
2028 Register DestReg = getLdStRegOp(MemMI, i).getReg();
2029 if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg))
2030 return E;
2031 }
2032 }
2033
2034 const bool BaseRegSP = BaseReg == AArch64::SP;
2035 if (BaseRegSP && needsWinCFI(I->getMF())) {
2036 // FIXME: For now, we always block the optimization over SP in windows
2037 // targets as it requires to adjust the unwind/debug info, messing up
2038 // the unwind info can actually cause a miscompile.
2039 return E;
2040 }
2041
2042 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2043 unsigned RedZoneSize =
2044 Subtarget.getTargetLowering()->getRedZoneSize(MF.getFunction());
2045
2046 // Track which register units have been modified and used between the first
2047 // insn (inclusive) and the second insn.
2048 ModifiedRegUnits.clear();
2049 UsedRegUnits.clear();
2050 unsigned Count = 0;
2051 bool MemAcessBeforeSPPreInc = false;
2052 do {
2053 MBBI = prev_nodbg(MBBI, B);
2054 MachineInstr &MI = *MBBI;
2055
2056 // Don't count transient instructions towards the search limit since there
2057 // may be different numbers of them if e.g. debug information is present.
2058 if (!MI.isTransient())
2059 ++Count;
2060
2061 // If we found a match, return it.
2062 if (isMatchingUpdateInsn(*I, MI, BaseReg, Offset)) {
2063 // Check that the update value is within our red zone limit (which may be
2064 // zero).
2065 if (MemAcessBeforeSPPreInc && MBBI->getOperand(2).getImm() > RedZoneSize)
2066 return E;
2067 return MBBI;
2068 }
2069
2070 // Update the status of what the instruction clobbered and used.
2071 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2072
2073 // Otherwise, if the base register is used or modified, we have no match, so
2074 // return early.
2075 if (!ModifiedRegUnits.available(BaseReg) ||
2076 !UsedRegUnits.available(BaseReg))
2077 return E;
2078 // Keep track if we have a memory access before an SP pre-increment, in this
2079 // case we need to validate later that the update amount respects the red
2080 // zone.
2081 if (BaseRegSP && MBBI->mayLoadOrStore())
2082 MemAcessBeforeSPPreInc = true;
2083 } while (MBBI != B && Count < Limit);
2084 return E;
2085}
2086
2087bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
2089 MachineInstr &MI = *MBBI;
2090 // If this is a volatile load, don't mess with it.
2091 if (MI.hasOrderedMemoryRef())
2092 return false;
2093
2094 if (needsWinCFI(MI.getMF()) && MI.getFlag(MachineInstr::FrameDestroy))
2095 return false;
2096
2097 // Make sure this is a reg+imm.
2098 // FIXME: It is possible to extend it to handle reg+reg cases.
2100 return false;
2101
2102 // Look backward up to LdStLimit instructions.
2104 if (findMatchingStore(MBBI, LdStLimit, StoreI)) {
2105 ++NumLoadsFromStoresPromoted;
2106 // Promote the load. Keeping the iterator straight is a
2107 // pain, so we let the merge routine tell us what the next instruction
2108 // is after it's done mucking about.
2109 MBBI = promoteLoadFromStore(MBBI, StoreI);
2110 return true;
2111 }
2112 return false;
2113}
2114
2115// Merge adjacent zero stores into a wider store.
2116bool AArch64LoadStoreOpt::tryToMergeZeroStInst(
2118 assert(isPromotableZeroStoreInst(*MBBI) && "Expected narrow store.");
2119 MachineInstr &MI = *MBBI;
2120 MachineBasicBlock::iterator E = MI.getParent()->end();
2121
2122 if (!TII->isCandidateToMergeOrPair(MI))
2123 return false;
2124
2125 // Look ahead up to LdStLimit instructions for a mergable instruction.
2126 LdStPairFlags Flags;
2128 findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ true);
2129 if (MergeMI != E) {
2130 ++NumZeroStoresPromoted;
2131
2132 // Keeping the iterator straight is a pain, so we let the merge routine tell
2133 // us what the next instruction is after it's done mucking about.
2134 MBBI = mergeNarrowZeroStores(MBBI, MergeMI, Flags);
2135 return true;
2136 }
2137 return false;
2138}
2139
2140// Find loads and stores that can be merged into a single load or store pair
2141// instruction.
2142bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
2143 MachineInstr &MI = *MBBI;
2144 MachineBasicBlock::iterator E = MI.getParent()->end();
2145
2146 if (!TII->isCandidateToMergeOrPair(MI))
2147 return false;
2148
2149 // If disable-ldp feature is opted, do not emit ldp.
2150 if (MI.mayLoad() && Subtarget->hasDisableLdp())
2151 return false;
2152
2153 // If disable-stp feature is opted, do not emit stp.
2154 if (MI.mayStore() && Subtarget->hasDisableStp())
2155 return false;
2156
2157 // Early exit if the offset is not possible to match. (6 bits of positive
2158 // range, plus allow an extra one in case we find a later insn that matches
2159 // with Offset-1)
2160 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI);
2162 int OffsetStride = IsUnscaled ? TII->getMemScale(MI) : 1;
2163 // Allow one more for offset.
2164 if (Offset > 0)
2165 Offset -= OffsetStride;
2166 if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride))
2167 return false;
2168
2169 // Look ahead up to LdStLimit instructions for a pairable instruction.
2170 LdStPairFlags Flags;
2172 findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ false);
2173 if (Paired != E) {
2174 ++NumPairCreated;
2175 if (TII->hasUnscaledLdStOffset(MI))
2176 ++NumUnscaledPairCreated;
2177 // Keeping the iterator straight is a pain, so we let the merge routine tell
2178 // us what the next instruction is after it's done mucking about.
2179 auto Prev = std::prev(MBBI);
2180
2181 // Fetch the memoperand of the load/store that is a candidate for
2182 // combination.
2184 MI.memoperands_empty() ? nullptr : MI.memoperands().front();
2185
2186 // Get the needed alignments to check them if
2187 // ldp-aligned-only/stp-aligned-only features are opted.
2188 uint64_t MemAlignment = MemOp ? MemOp->getAlign().value() : -1;
2189 uint64_t TypeAlignment = MemOp ? Align(MemOp->getSize()).value() : -1;
2190
2191 // If a load arrives and ldp-aligned-only feature is opted, check that the
2192 // alignment of the source pointer is at least double the alignment of the
2193 // type.
2194 if (MI.mayLoad() && Subtarget->hasLdpAlignedOnly() && MemOp &&
2195 MemAlignment < 2 * TypeAlignment)
2196 return false;
2197
2198 // If a store arrives and stp-aligned-only feature is opted, check that the
2199 // alignment of the source pointer is at least double the alignment of the
2200 // type.
2201 if (MI.mayStore() && Subtarget->hasStpAlignedOnly() && MemOp &&
2202 MemAlignment < 2 * TypeAlignment)
2203 return false;
2204
2205 MBBI = mergePairedInsns(MBBI, Paired, Flags);
2206 // Collect liveness info for instructions between Prev and the new position
2207 // MBBI.
2208 for (auto I = std::next(Prev); I != MBBI; I++)
2209 updateDefinedRegisters(*I, DefinedInBB, TRI);
2210
2211 return true;
2212 }
2213 return false;
2214}
2215
2216bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
2218 MachineInstr &MI = *MBBI;
2219 MachineBasicBlock::iterator E = MI.getParent()->end();
2221
2222 // Look forward to try to form a post-index instruction. For example,
2223 // ldr x0, [x20]
2224 // add x20, x20, #32
2225 // merged into:
2226 // ldr x0, [x20], #32
2227 Update = findMatchingUpdateInsnForward(MBBI, 0, UpdateLimit);
2228 if (Update != E) {
2229 // Merge the update into the ld/st.
2230 MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/false);
2231 return true;
2232 }
2233
2234 // Don't know how to handle unscaled pre/post-index versions below, so bail.
2235 if (TII->hasUnscaledLdStOffset(MI.getOpcode()))
2236 return false;
2237
2238 // Look back to try to find a pre-index instruction. For example,
2239 // add x0, x0, #8
2240 // ldr x1, [x0]
2241 // merged into:
2242 // ldr x1, [x0, #8]!
2243 Update = findMatchingUpdateInsnBackward(MBBI, UpdateLimit);
2244 if (Update != E) {
2245 // Merge the update into the ld/st.
2246 MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/true);
2247 return true;
2248 }
2249
2250 // The immediate in the load/store is scaled by the size of the memory
2251 // operation. The immediate in the add we're looking for,
2252 // however, is not, so adjust here.
2253 int UnscaledOffset =
2255
2256 // Look forward to try to find a pre-index instruction. For example,
2257 // ldr x1, [x0, #64]
2258 // add x0, x0, #64
2259 // merged into:
2260 // ldr x1, [x0, #64]!
2261 Update = findMatchingUpdateInsnForward(MBBI, UnscaledOffset, UpdateLimit);
2262 if (Update != E) {
2263 // Merge the update into the ld/st.
2264 MBBI = mergeUpdateInsn(MBBI, Update, /*IsPreIdx=*/true);
2265 return true;
2266 }
2267
2268 return false;
2269}
2270
2271bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
2272 bool EnableNarrowZeroStOpt) {
2273
2274 bool Modified = false;
2275 // Four tranformations to do here:
2276 // 1) Find loads that directly read from stores and promote them by
2277 // replacing with mov instructions. If the store is wider than the load,
2278 // the load will be replaced with a bitfield extract.
2279 // e.g.,
2280 // str w1, [x0, #4]
2281 // ldrh w2, [x0, #6]
2282 // ; becomes
2283 // str w1, [x0, #4]
2284 // lsr w2, w1, #16
2286 MBBI != E;) {
2287 if (isPromotableLoadFromStore(*MBBI) && tryToPromoteLoadFromStore(MBBI))
2288 Modified = true;
2289 else
2290 ++MBBI;
2291 }
2292 // 2) Merge adjacent zero stores into a wider store.
2293 // e.g.,
2294 // strh wzr, [x0]
2295 // strh wzr, [x0, #2]
2296 // ; becomes
2297 // str wzr, [x0]
2298 // e.g.,
2299 // str wzr, [x0]
2300 // str wzr, [x0, #4]
2301 // ; becomes
2302 // str xzr, [x0]
2303 if (EnableNarrowZeroStOpt)
2305 MBBI != E;) {
2306 if (isPromotableZeroStoreInst(*MBBI) && tryToMergeZeroStInst(MBBI))
2307 Modified = true;
2308 else
2309 ++MBBI;
2310 }
2311 // 3) Find loads and stores that can be merged into a single load or store
2312 // pair instruction.
2313 // e.g.,
2314 // ldr x0, [x2]
2315 // ldr x1, [x2, #8]
2316 // ; becomes
2317 // ldp x0, x1, [x2]
2318
2320 DefinedInBB.clear();
2321 DefinedInBB.addLiveIns(MBB);
2322 }
2323
2325 MBBI != E;) {
2326 // Track currently live registers up to this point, to help with
2327 // searching for a rename register on demand.
2328 updateDefinedRegisters(*MBBI, DefinedInBB, TRI);
2329 if (TII->isPairableLdStInst(*MBBI) && tryToPairLdStInst(MBBI))
2330 Modified = true;
2331 else
2332 ++MBBI;
2333 }
2334 // 4) Find base register updates that can be merged into the load or store
2335 // as a base-reg writeback.
2336 // e.g.,
2337 // ldr x0, [x2]
2338 // add x2, x2, #4
2339 // ; becomes
2340 // ldr x0, [x2], #4
2342 MBBI != E;) {
2343 if (isMergeableLdStUpdate(*MBBI) && tryToMergeLdStUpdate(MBBI))
2344 Modified = true;
2345 else
2346 ++MBBI;
2347 }
2348
2349 return Modified;
2350}
2351
2352bool AArch64LoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
2353 if (skipFunction(Fn.getFunction()))
2354 return false;
2355
2356 Subtarget = &Fn.getSubtarget<AArch64Subtarget>();
2357 TII = static_cast<const AArch64InstrInfo *>(Subtarget->getInstrInfo());
2358 TRI = Subtarget->getRegisterInfo();
2359 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
2360
2361 // Resize the modified and used register unit trackers. We do this once
2362 // per function and then clear the register units each time we optimize a load
2363 // or store.
2364 ModifiedRegUnits.init(*TRI);
2365 UsedRegUnits.init(*TRI);
2366 DefinedInBB.init(*TRI);
2367
2368 bool Modified = false;
2369 bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign();
2370 for (auto &MBB : Fn) {
2371 auto M = optimizeBlock(MBB, enableNarrowZeroStOpt);
2372 Modified |= M;
2373 }
2374
2375 return Modified;
2376}
2377
2378// FIXME: Do we need/want a pre-alloc pass like ARM has to try to keep loads and
2379// stores near one another? Note: The pre-RA instruction scheduler already has
2380// hooks to try and schedule pairable loads/stores together to improve pairing
2381// opportunities. Thus, pre-RA pairing pass may not be worth the effort.
2382
2383// FIXME: When pairing store instructions it's very possible for this pass to
2384// hoist a store with a KILL marker above another use (without a KILL marker).
2385// The resulting IR is invalid, but nothing uses the KILL markers after this
2386// pass, so it's never caused a problem in practice.
2387
2388/// createAArch64LoadStoreOptimizationPass - returns an instance of the
2389/// load / store optimization pass.
2391 return new AArch64LoadStoreOpt();
2392}
static cl::opt< bool > EnableRenaming("aarch64-load-store-renaming", cl::init(true), cl::Hidden)
static MachineOperand & getLdStRegOp(MachineInstr &MI, unsigned PairedRegOp=0)
static bool isPromotableLoadFromStore(MachineInstr &MI)
static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale, int &MinOffset, int &MaxOffset)
static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride)
static unsigned getMatchingPairOpcode(unsigned Opc)
static bool isMergeableLdStUpdate(MachineInstr &MI)
static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI, LdStPairFlags &Flags, const AArch64InstrInfo *TII)
static std::optional< MCPhysReg > tryToFindRegisterToRename(const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static bool needsWinCFI(const MachineFunction *MF)
static bool mayAlias(MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
static cl::opt< unsigned > LdStLimit("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden)
static unsigned getPreIndexedOpcode(unsigned Opc)
#define AARCH64_LOAD_STORE_OPT_NAME
static cl::opt< unsigned > UpdateLimit("aarch64-update-scan-limit", cl::init(100), cl::Hidden)
static bool isPromotableZeroStoreInst(MachineInstr &MI)
static unsigned getMatchingWideOpcode(unsigned Opc)
static unsigned getMatchingNonSExtOpcode(unsigned Opc, bool *IsValidLdStrOpc=nullptr)
static MachineBasicBlock::iterator maybeMoveCFI(MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI)
static int alignTo(int Num, int PowOf2)
static bool isTagStore(const MachineInstr &MI)
static unsigned isMatchingStore(MachineInstr &LoadInst, MachineInstr &StoreInst)
static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg, const TargetRegisterInfo *TRI, unsigned Limit, std::function< bool(MachineInstr &, bool)> &Fn)
static unsigned getPostIndexedOpcode(unsigned Opc)
#define DEBUG_TYPE
static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst, MachineInstr &StoreInst, const AArch64InstrInfo *TII)
static bool isPreLdStPairCandidate(MachineInstr &FirstMI, MachineInstr &MI)
static void updateDefinedRegisters(MachineInstr &MI, LiveRegUnits &Units, const TargetRegisterInfo *TRI)
static bool canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
This file implements the BitVector class.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
Definition: DebugCounter.h:182
#define LLVM_DEBUG(X)
Definition: Debug.h:101
bool End
Definition: ELF_riscv.cpp:469
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, DomTreeUpdater *DTU)
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:167
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:191
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
const AArch64RegisterInfo * getRegisterInfo() const override
const AArch64InstrInfo * getInstrInfo() const override
const AArch64TargetLowering * getTargetLowering() const override
unsigned getRedZoneSize(const Function &F) const
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
static bool shouldExecute(unsigned CounterName)
Definition: DebugCounter.h:72
A debug info location.
Definition: DebugLoc.h:33
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
bool needsUnwindTableEntry() const
True if this function needs an unwind table.
Definition: Function.h:622
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:195
A set of register units used to track register liveness.
Definition: LiveRegUnits.h:30
static void accumulateUsedDefed(const MachineInstr &MI, LiveRegUnits &ModifiedRegUnits, LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI)
For a machine instruction MI, adds all register units used in UsedRegUnits and defined or clobbered i...
Definition: LiveRegUnits.h:47
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
Definition: LiveRegUnits.h:116
void init(const TargetRegisterInfo &TRI)
Initialize and clear the set.
Definition: LiveRegUnits.h:73
void addReg(MCPhysReg Reg)
Adds register units covered by physical register Reg.
Definition: LiveRegUnits.h:86
void removeReg(MCPhysReg Reg)
Removes all register units covered by physical register Reg.
Definition: LiveRegUnits.h:102
void accumulate(const MachineInstr &MI)
Adds all register units used, defined or clobbered in MI.
An instruction for reading from memory.
Definition: Instructions.h:177
bool usesWindowsCFI() const
Definition: MCAsmInfo.h:799
OpType getOperation() const
Definition: MCDwarf.h:657
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:33
reverse_instr_iterator instr_rend()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const std::vector< MCCFIInstruction > & getFrameInstructions() const
Returns a reference to a list of cfi instructions in the function's prologue.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & cloneMergedMemRefs(ArrayRef< const MachineInstr * > OtherMIs) const
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:543
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:326
bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const
Returns true if this instruction's memory access aliases the memory access of Other.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:659
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:553
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
void setImplicit(bool Val=true)
int64_t getImm() const
bool isImplicit() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
bool isRenamable() const
isRenamable - Returns true if this register may be renamed, i.e.
bool isEarlyClobber() const
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:345
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:366
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:451
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
An instruction for storing to memory.
Definition: Instructions.h:301
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
LLVM Value Representation.
Definition: Value.h:74
self_iterator getIterator()
Definition: ilist_node.h:82
A range adaptor for a pair of iterators.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ Define
Register definition.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
constexpr double e
Definition: MathExtras.h:31
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
@ Offset
Definition: DWP.cpp:440
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1727
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< filter_iterator< ConstMIBundleOperands, bool(*)(const MachineOperand &)> > phys_regs_and_masks(const MachineInstr &MI)
Returns an iterator range over all physical register and mask operands for MI and bundled instruction...
Definition: LiveRegUnits.h:166
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1734
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
FunctionPass * createAArch64LoadStoreOptimizationPass()
createAArch64LoadStoreOptimizationPass - returns an instance of the load / store optimization pass.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
void initializeAArch64LoadStoreOptPass(PassRegistry &)
IterT prev_nodbg(IterT It, IterT Begin, bool SkipPseudoOp=true)
Decrement It, then continue decrementing it while it points to a debug instruction.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85