56#define DEBUG_TYPE "aarch64-ldst-opt"
58STATISTIC(NumPairCreated,
"Number of load/store pair instructions generated");
59STATISTIC(NumPostFolded,
"Number of post-index updates folded");
60STATISTIC(NumPreFolded,
"Number of pre-index updates folded");
62 "Number of load/store from unscaled generated");
63STATISTIC(NumZeroStoresPromoted,
"Number of narrow zero stores promoted");
64STATISTIC(NumLoadsFromStoresPromoted,
"Number of loads from stores promoted");
65STATISTIC(NumFailedAlignmentCheck,
"Number of load/store pair transformation "
66 "not passed the alignment check");
69 "Controls which pairs are considered for renaming");
84#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"
88using LdStPairFlags =
struct LdStPairFlags {
92 bool MergeForward =
false;
103 std::optional<MCPhysReg> RenameReg;
105 LdStPairFlags() =
default;
107 void setMergeForward(
bool V =
true) { MergeForward = V; }
108 bool getMergeForward()
const {
return MergeForward; }
110 void setSExtIdx(
int V) { SExtIdx = V; }
111 int getSExtIdx()
const {
return SExtIdx; }
113 void setRenameReg(
MCPhysReg R) { RenameReg = R; }
114 void clearRenameReg() { RenameReg = std::nullopt; }
115 std::optional<MCPhysReg> getRenameReg()
const {
return RenameReg; }
143 LdStPairFlags &Flags,
145 bool FindNarrowMerge);
156 const LdStPairFlags &Flags);
162 const LdStPairFlags &Flags);
174 int UnscaledOffset,
unsigned Limit);
185 unsigned BaseReg,
int Offset);
210 MachineFunctionProperties::Property::NoVRegs);
216char AArch64LoadStoreOpt::ID = 0;
223static
bool isNarrowStore(
unsigned Opc) {
227 case AArch64::STRBBui:
228 case AArch64::STURBBi:
229 case AArch64::STRHHui:
230 case AArch64::STURHHi:
238 switch (
MI.getOpcode()) {
244 case AArch64::STZ2Gi:
250 bool *IsValidLdStrOpc =
nullptr) {
252 *IsValidLdStrOpc =
true;
256 *IsValidLdStrOpc =
false;
257 return std::numeric_limits<unsigned>::max();
258 case AArch64::STRDui:
259 case AArch64::STURDi:
260 case AArch64::STRDpre:
261 case AArch64::STRQui:
262 case AArch64::STURQi:
263 case AArch64::STRQpre:
264 case AArch64::STRBBui:
265 case AArch64::STURBBi:
266 case AArch64::STRHHui:
267 case AArch64::STURHHi:
268 case AArch64::STRWui:
269 case AArch64::STRWpre:
270 case AArch64::STURWi:
271 case AArch64::STRXui:
272 case AArch64::STRXpre:
273 case AArch64::STURXi:
274 case AArch64::LDRDui:
275 case AArch64::LDURDi:
276 case AArch64::LDRDpre:
277 case AArch64::LDRQui:
278 case AArch64::LDURQi:
279 case AArch64::LDRQpre:
280 case AArch64::LDRWui:
281 case AArch64::LDURWi:
282 case AArch64::LDRWpre:
283 case AArch64::LDRXui:
284 case AArch64::LDURXi:
285 case AArch64::LDRXpre:
286 case AArch64::STRSui:
287 case AArch64::STURSi:
288 case AArch64::STRSpre:
289 case AArch64::LDRSui:
290 case AArch64::LDURSi:
291 case AArch64::LDRSpre:
293 case AArch64::LDRSWui:
294 return AArch64::LDRWui;
295 case AArch64::LDURSWi:
296 return AArch64::LDURWi;
297 case AArch64::LDRSWpre:
298 return AArch64::LDRWpre;
306 case AArch64::STRBBui:
307 return AArch64::STRHHui;
308 case AArch64::STRHHui:
309 return AArch64::STRWui;
310 case AArch64::STURBBi:
311 return AArch64::STURHHi;
312 case AArch64::STURHHi:
313 return AArch64::STURWi;
314 case AArch64::STURWi:
315 return AArch64::STURXi;
316 case AArch64::STRWui:
317 return AArch64::STRXui;
325 case AArch64::STRSui:
326 case AArch64::STURSi:
327 return AArch64::STPSi;
328 case AArch64::STRSpre:
329 return AArch64::STPSpre;
330 case AArch64::STRDui:
331 case AArch64::STURDi:
332 return AArch64::STPDi;
333 case AArch64::STRDpre:
334 return AArch64::STPDpre;
335 case AArch64::STRQui:
336 case AArch64::STURQi:
337 return AArch64::STPQi;
338 case AArch64::STRQpre:
339 return AArch64::STPQpre;
340 case AArch64::STRWui:
341 case AArch64::STURWi:
342 return AArch64::STPWi;
343 case AArch64::STRWpre:
344 return AArch64::STPWpre;
345 case AArch64::STRXui:
346 case AArch64::STURXi:
347 return AArch64::STPXi;
348 case AArch64::STRXpre:
349 return AArch64::STPXpre;
350 case AArch64::LDRSui:
351 case AArch64::LDURSi:
352 return AArch64::LDPSi;
353 case AArch64::LDRSpre:
354 return AArch64::LDPSpre;
355 case AArch64::LDRDui:
356 case AArch64::LDURDi:
357 return AArch64::LDPDi;
358 case AArch64::LDRDpre:
359 return AArch64::LDPDpre;
360 case AArch64::LDRQui:
361 case AArch64::LDURQi:
362 return AArch64::LDPQi;
363 case AArch64::LDRQpre:
364 return AArch64::LDPQpre;
365 case AArch64::LDRWui:
366 case AArch64::LDURWi:
367 return AArch64::LDPWi;
368 case AArch64::LDRWpre:
369 return AArch64::LDPWpre;
370 case AArch64::LDRXui:
371 case AArch64::LDURXi:
372 return AArch64::LDPXi;
373 case AArch64::LDRXpre:
374 return AArch64::LDPXpre;
375 case AArch64::LDRSWui:
376 case AArch64::LDURSWi:
377 return AArch64::LDPSWi;
378 case AArch64::LDRSWpre:
379 return AArch64::LDPSWpre;
390 case AArch64::LDRBBui:
391 return StOpc == AArch64::STRBBui || StOpc == AArch64::STRHHui ||
392 StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
393 case AArch64::LDURBBi:
394 return StOpc == AArch64::STURBBi || StOpc == AArch64::STURHHi ||
395 StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
396 case AArch64::LDRHHui:
397 return StOpc == AArch64::STRHHui || StOpc == AArch64::STRWui ||
398 StOpc == AArch64::STRXui;
399 case AArch64::LDURHHi:
400 return StOpc == AArch64::STURHHi || StOpc == AArch64::STURWi ||
401 StOpc == AArch64::STURXi;
402 case AArch64::LDRWui:
403 return StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
404 case AArch64::LDURWi:
405 return StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
406 case AArch64::LDRXui:
407 return StOpc == AArch64::STRXui;
408 case AArch64::LDURXi:
409 return StOpc == AArch64::STURXi;
421 case AArch64::STRSui:
422 return AArch64::STRSpre;
423 case AArch64::STRDui:
424 return AArch64::STRDpre;
425 case AArch64::STRQui:
426 return AArch64::STRQpre;
427 case AArch64::STRBBui:
428 return AArch64::STRBBpre;
429 case AArch64::STRHHui:
430 return AArch64::STRHHpre;
431 case AArch64::STRWui:
432 return AArch64::STRWpre;
433 case AArch64::STRXui:
434 return AArch64::STRXpre;
435 case AArch64::LDRSui:
436 return AArch64::LDRSpre;
437 case AArch64::LDRDui:
438 return AArch64::LDRDpre;
439 case AArch64::LDRQui:
440 return AArch64::LDRQpre;
441 case AArch64::LDRBBui:
442 return AArch64::LDRBBpre;
443 case AArch64::LDRHHui:
444 return AArch64::LDRHHpre;
445 case AArch64::LDRWui:
446 return AArch64::LDRWpre;
447 case AArch64::LDRXui:
448 return AArch64::LDRXpre;
449 case AArch64::LDRSWui:
450 return AArch64::LDRSWpre;
452 return AArch64::LDPSpre;
453 case AArch64::LDPSWi:
454 return AArch64::LDPSWpre;
456 return AArch64::LDPDpre;
458 return AArch64::LDPQpre;
460 return AArch64::LDPWpre;
462 return AArch64::LDPXpre;
464 return AArch64::STPSpre;
466 return AArch64::STPDpre;
468 return AArch64::STPQpre;
470 return AArch64::STPWpre;
472 return AArch64::STPXpre;
474 return AArch64::STGPreIndex;
476 return AArch64::STZGPreIndex;
478 return AArch64::ST2GPreIndex;
479 case AArch64::STZ2Gi:
480 return AArch64::STZ2GPreIndex;
482 return AArch64::STGPpre;
490 case AArch64::STRSui:
491 case AArch64::STURSi:
492 return AArch64::STRSpost;
493 case AArch64::STRDui:
494 case AArch64::STURDi:
495 return AArch64::STRDpost;
496 case AArch64::STRQui:
497 case AArch64::STURQi:
498 return AArch64::STRQpost;
499 case AArch64::STRBBui:
500 return AArch64::STRBBpost;
501 case AArch64::STRHHui:
502 return AArch64::STRHHpost;
503 case AArch64::STRWui:
504 case AArch64::STURWi:
505 return AArch64::STRWpost;
506 case AArch64::STRXui:
507 case AArch64::STURXi:
508 return AArch64::STRXpost;
509 case AArch64::LDRSui:
510 case AArch64::LDURSi:
511 return AArch64::LDRSpost;
512 case AArch64::LDRDui:
513 case AArch64::LDURDi:
514 return AArch64::LDRDpost;
515 case AArch64::LDRQui:
516 case AArch64::LDURQi:
517 return AArch64::LDRQpost;
518 case AArch64::LDRBBui:
519 return AArch64::LDRBBpost;
520 case AArch64::LDRHHui:
521 return AArch64::LDRHHpost;
522 case AArch64::LDRWui:
523 case AArch64::LDURWi:
524 return AArch64::LDRWpost;
525 case AArch64::LDRXui:
526 case AArch64::LDURXi:
527 return AArch64::LDRXpost;
528 case AArch64::LDRSWui:
529 return AArch64::LDRSWpost;
531 return AArch64::LDPSpost;
532 case AArch64::LDPSWi:
533 return AArch64::LDPSWpost;
535 return AArch64::LDPDpost;
537 return AArch64::LDPQpost;
539 return AArch64::LDPWpost;
541 return AArch64::LDPXpost;
543 return AArch64::STPSpost;
545 return AArch64::STPDpost;
547 return AArch64::STPQpost;
549 return AArch64::STPWpost;
551 return AArch64::STPXpost;
553 return AArch64::STGPostIndex;
555 return AArch64::STZGPostIndex;
557 return AArch64::ST2GPostIndex;
558 case AArch64::STZ2Gi:
559 return AArch64::STZ2GPostIndex;
561 return AArch64::STGPpost;
568 unsigned OpcB =
MI.getOpcode();
573 case AArch64::STRSpre:
574 return (OpcB == AArch64::STRSui) || (OpcB == AArch64::STURSi);
575 case AArch64::STRDpre:
576 return (OpcB == AArch64::STRDui) || (OpcB == AArch64::STURDi);
577 case AArch64::STRQpre:
578 return (OpcB == AArch64::STRQui) || (OpcB == AArch64::STURQi);
579 case AArch64::STRWpre:
580 return (OpcB == AArch64::STRWui) || (OpcB == AArch64::STURWi);
581 case AArch64::STRXpre:
582 return (OpcB == AArch64::STRXui) || (OpcB == AArch64::STURXi);
583 case AArch64::LDRSpre:
584 return (OpcB == AArch64::LDRSui) || (OpcB == AArch64::LDURSi);
585 case AArch64::LDRDpre:
586 return (OpcB == AArch64::LDRDui) || (OpcB == AArch64::LDURDi);
587 case AArch64::LDRQpre:
588 return (OpcB == AArch64::LDRQui) || (OpcB == AArch64::LDURQi);
589 case AArch64::LDRWpre:
590 return (OpcB == AArch64::LDRWui) || (OpcB == AArch64::LDURWi);
591 case AArch64::LDRXpre:
592 return (OpcB == AArch64::LDRXui) || (OpcB == AArch64::LDURXi);
593 case AArch64::LDRSWpre:
594 return (OpcB == AArch64::LDRSWui) || (OpcB == AArch64::LDURSWi);
600 int &MinOffset,
int &MaxOffset) {
618 unsigned PairedRegOp = 0) {
619 assert(PairedRegOp < 2 &&
"Unexpected register operand idx.");
625 return MI.getOperand(
Idx);
634 int UnscaledStOffset =
638 int UnscaledLdOffset =
642 return (UnscaledStOffset <= UnscaledLdOffset) &&
643 (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize));
647 unsigned Opc =
MI.getOpcode();
648 return (Opc == AArch64::STRWui || Opc == AArch64::STURWi ||
649 isNarrowStore(Opc)) &&
654 switch (
MI.getOpcode()) {
658 case AArch64::LDRBBui:
659 case AArch64::LDRHHui:
660 case AArch64::LDRWui:
661 case AArch64::LDRXui:
663 case AArch64::LDURBBi:
664 case AArch64::LDURHHi:
665 case AArch64::LDURWi:
666 case AArch64::LDURXi:
672 unsigned Opc =
MI.getOpcode();
677 case AArch64::STRSui:
678 case AArch64::STRDui:
679 case AArch64::STRQui:
680 case AArch64::STRXui:
681 case AArch64::STRWui:
682 case AArch64::STRHHui:
683 case AArch64::STRBBui:
684 case AArch64::LDRSui:
685 case AArch64::LDRDui:
686 case AArch64::LDRQui:
687 case AArch64::LDRXui:
688 case AArch64::LDRWui:
689 case AArch64::LDRHHui:
690 case AArch64::LDRBBui:
694 case AArch64::STZ2Gi:
697 case AArch64::STURSi:
698 case AArch64::STURDi:
699 case AArch64::STURQi:
700 case AArch64::STURWi:
701 case AArch64::STURXi:
702 case AArch64::LDURSi:
703 case AArch64::LDURDi:
704 case AArch64::LDURQi:
705 case AArch64::LDURWi:
706 case AArch64::LDURXi:
709 case AArch64::LDPSWi:
731 case AArch64::ORRWrs:
732 case AArch64::ADDWri:
740 const LdStPairFlags &Flags) {
742 "Expected promotable zero stores.");
750 if (NextI == MergeMI)
753 unsigned Opc =
I->getOpcode();
754 unsigned MergeMIOpc = MergeMI->getOpcode();
755 bool IsScaled = !
TII->hasUnscaledLdStOffset(Opc);
756 bool IsMergedMIScaled = !
TII->hasUnscaledLdStOffset(MergeMIOpc);
757 int OffsetStride = IsScaled ?
TII->getMemScale(*
I) : 1;
758 int MergeMIOffsetStride = IsMergedMIScaled ?
TII->getMemScale(*MergeMI) : 1;
760 bool MergeForward =
Flags.getMergeForward();
771 int64_t IOffsetInBytes =
773 int64_t MIOffsetInBytes =
778 if (IOffsetInBytes > MIOffsetInBytes)
779 OffsetImm = MIOffsetInBytes;
781 OffsetImm = IOffsetInBytes;
784 bool FinalIsScaled = !
TII->hasUnscaledLdStOffset(NewOpcode);
788 int NewOffsetStride = FinalIsScaled ?
TII->getMemScale(NewOpcode) : 1;
789 assert(((OffsetImm % NewOffsetStride) == 0) &&
790 "Offset should be a multiple of the store memory scale");
791 OffsetImm = OffsetImm / NewOffsetStride;
799 .
addReg(isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR)
803 .setMIFlags(
I->mergeFlagsWith(*MergeMI));
806 LLVM_DEBUG(
dbgs() <<
"Creating wider store. Replacing instructions:\n ");
815 I->eraseFromParent();
816 MergeMI->eraseFromParent();
826 auto MBB =
MI.getParent();
834 return MOP.isReg() && MOP.isDef() && !MOP.isDebug() && MOP.getReg() &&
835 TRI->regsOverlap(MOP.getReg(), DefReg);
849 if (MOP.isReg() && MOP.isKill())
853 if (MOP.isReg() && !MOP.isKill())
854 Units.
addReg(MOP.getReg());
860 const LdStPairFlags &Flags) {
870 int SExtIdx =
Flags.getSExtIdx();
873 bool IsUnscaled =
TII->hasUnscaledLdStOffset(Opc);
874 int OffsetStride = IsUnscaled ?
TII->getMemScale(*
I) : 1;
876 bool MergeForward =
Flags.getMergeForward();
878 std::optional<MCPhysReg> RenameReg =
Flags.getRenameReg();
881 DefinedInBB.addReg(*RenameReg);
885 auto GetMatchingSubReg =
888 TRI->sub_and_superregs_inclusive(*RenameReg)) {
889 if (
C->contains(SubOrSuper))
896 [
this, RegToRename, GetMatchingSubReg, MergeForward](
MachineInstr &
MI,
899 bool SeenDef =
false;
900 for (
unsigned OpIdx = 0; OpIdx <
MI.getNumOperands(); ++OpIdx) {
905 (!MergeForward || !SeenDef ||
907 TRI->regsOverlap(MOP.
getReg(), RegToRename)) {
910 "Need renamable operands");
913 MI.getRegClassConstraint(OpIdx,
TII,
TRI))
914 MatchingReg = GetMatchingSubReg(RC);
918 MatchingReg = GetMatchingSubReg(
919 TRI->getMinimalPhysRegClass(MOP.
getReg()));
926 for (
unsigned OpIdx = 0; OpIdx <
MI.getNumOperands(); ++OpIdx) {
929 TRI->regsOverlap(MOP.
getReg(), RegToRename)) {
932 "Need renamable operands");
935 MI.getRegClassConstraint(OpIdx,
TII,
TRI))
936 MatchingReg = GetMatchingSubReg(RC);
938 MatchingReg = GetMatchingSubReg(
939 TRI->getMinimalPhysRegClass(MOP.
getReg()));
940 assert(MatchingReg != AArch64::NoRegister &&
941 "Cannot find matching regs for renaming");
950 UINT32_MAX, UpdateMIs);
963 RegToCheck = RegToRename;
966 MergeForward ? std::next(
I) :
I,
967 MergeForward ? std::next(Paired) : Paired))
970 return !MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
972 !TRI->regsOverlap(MOP.getReg(), RegToCheck);
974 "Rename register used between paired instruction, trashing the "
990 bool PairedIsUnscaled =
TII->hasUnscaledLdStOffset(Paired->getOpcode());
991 if (IsUnscaled != PairedIsUnscaled) {
995 int MemSize =
TII->getMemScale(*Paired);
996 if (PairedIsUnscaled) {
999 assert(!(PairedOffset %
TII->getMemScale(*Paired)) &&
1000 "Offset should be a multiple of the stride!");
1001 PairedOffset /= MemSize;
1003 PairedOffset *= MemSize;
1011 if (
Offset == PairedOffset + OffsetStride &&
1019 SExtIdx = (SExtIdx + 1) % 2;
1027 assert(!(OffsetImm %
TII->getMemScale(*RtMI)) &&
1028 "Unscaled offset cannot be scaled.");
1029 OffsetImm /=
TII->getMemScale(*RtMI);
1038 MachineOperand &PairedRegOp = RtMI == &*Paired ? RegOp0 : RegOp1;
1040 if (RegOp0.
isUse()) {
1041 if (!MergeForward) {
1052 for (
auto It = std::next(
I); It != Paired && PairedRegOp.
isKill(); ++It)
1053 if (It->readsRegister(PairedRegOp.
getReg(),
TRI))
1062 MI.clearRegisterKills(Reg,
TRI);
1078 .setMIFlags(
I->mergeFlagsWith(*Paired));
1083 dbgs() <<
"Creating pair load/store. Replacing instructions:\n ");
1088 if (SExtIdx != -1) {
1098 Register DstRegW =
TRI->getSubReg(DstRegX, AArch64::sub_32);
1108 BuildMI(*
MBB, InsertionPoint,
DL,
TII->get(TargetOpcode::KILL), DstRegW)
1114 BuildMI(*
MBB, InsertionPoint,
DL,
TII->get(AArch64::SBFMXri), DstRegX)
1129 DefinedInBB.addReg(MOP.
getReg());
1132 I->eraseFromParent();
1133 Paired->eraseFromParent();
1142 next_nodbg(LoadI, LoadI->getParent()->end());
1144 int LoadSize =
TII->getMemScale(*LoadI);
1145 int StoreSize =
TII->getMemScale(*StoreI);
1149 bool IsStoreXReg =
TRI->getRegClass(AArch64::GPR64RegClassID)->contains(StRt);
1152 TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) &&
1153 "Unexpected RegClass");
1156 if (LoadSize == StoreSize && (LoadSize == 4 || LoadSize == 8)) {
1159 if (StRt == LdRt && LoadSize == 8) {
1161 LoadI->getIterator())) {
1162 if (
MI.killsRegister(StRt,
TRI)) {
1163 MI.clearRegisterKills(StRt,
TRI);
1170 LoadI->eraseFromParent();
1175 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1176 TII->get(IsStoreXReg ? AArch64::ORRXrs : AArch64::ORRWrs), LdRt)
1177 .
addReg(IsStoreXReg ? AArch64::XZR : AArch64::WZR)
1184 if (!Subtarget->isLittleEndian())
1186 bool IsUnscaled =
TII->hasUnscaledLdStOffset(*LoadI);
1187 assert(IsUnscaled ==
TII->hasUnscaledLdStOffset(*StoreI) &&
1188 "Unsupported ld/st match");
1189 assert(LoadSize <= StoreSize &&
"Invalid load size");
1190 int UnscaledLdOffset =
1194 int UnscaledStOffset =
1198 int Width = LoadSize * 8;
1201 LdRt, AArch64::sub_32, &AArch64::GPR64RegClass))
1204 assert((UnscaledLdOffset >= UnscaledStOffset &&
1205 (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) &&
1208 int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset);
1209 int Imms = Immr + Width - 1;
1210 if (UnscaledLdOffset == UnscaledStOffset) {
1211 uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12)
1217 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1218 TII->get(IsStoreXReg ? AArch64::ANDXri : AArch64::ANDWri),
1225 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1226 TII->get(IsStoreXReg ? AArch64::UBFMXri : AArch64::UBFMWri),
1238 if (
MI.killsRegister(StRt,
TRI)) {
1239 MI.clearRegisterKills(StRt,
TRI);
1254 LoadI->eraseFromParent();
1264 if (
Offset % OffsetStride)
1268 return Offset <= 63 && Offset >= -64;
1276 return (Num + PowOf2 - 1) & ~(PowOf2 - 1);
1283 if (MIa.
mayAlias(AA, *MIb,
false)) {
1293bool AArch64LoadStoreOpt::findMatchingStore(
1308 ModifiedRegUnits.clear();
1309 UsedRegUnits.clear();
1318 if (!
MI.isTransient())
1344 if (!ModifiedRegUnits.available(BaseReg))
1350 }
while (
MBBI !=
B && Count < Limit);
1362 LdStPairFlags &Flags,
1365 if (
MI.hasOrderedMemoryRef() ||
TII->isLdStPairSuppressed(
MI))
1370 !
TII->isLdStPairSuppressed(FirstMI) &&
1371 "FirstMI shouldn't get here if either of these checks are true.");
1378 unsigned OpcB =
MI.getOpcode();
1389 bool IsValidLdStrOpc, PairIsValidLdStrOpc;
1391 assert(IsValidLdStrOpc &&
1392 "Given Opc should be a Load or Store with an immediate");
1395 Flags.setSExtIdx(NonSExtOpc == (
unsigned)OpcA ? 1 : 0);
1401 if (!PairIsValidLdStrOpc)
1406 if (isNarrowStore(OpcA) || isNarrowStore(OpcB))
1416 return TII->hasUnscaledLdStOffset(OpcA) !=
TII->hasUnscaledLdStOffset(OpcB) &&
1425 auto *RegClass =
TRI->getMinimalPhysRegClass(MOP.
getReg());
1432 if (RegClass->HasDisjunctSubRegs) {
1435 <<
" Cannot rename operands with multiple disjunct subregisters ("
1446 return TRI->isSuperOrSubRegisterEq(
1469 return MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1470 MOP.isImplicit() && MOP.isKill() &&
1471 TRI->regsOverlap(RegToRename, MOP.getReg());
1477 bool FoundDef =
false;
1508 if (
MI.isPseudo()) {
1509 LLVM_DEBUG(
dbgs() <<
" Cannot rename pseudo/bundle instruction\n");
1513 for (
auto &MOP :
MI.operands()) {
1515 !
TRI->regsOverlap(MOP.
getReg(), RegToRename))
1525 for (
auto &MOP :
MI.operands()) {
1527 !
TRI->regsOverlap(MOP.
getReg(), RegToRename))
1544 LLVM_DEBUG(
dbgs() <<
" Did not find definition for register in BB\n");
1572 LLVM_DEBUG(dbgs() <<
"Checking " << MI);
1574 if (MI.getFlag(MachineInstr::FrameSetup)) {
1575 LLVM_DEBUG(dbgs() <<
" Cannot rename framesetup instructions "
1580 for (
auto &MOP :
MI.operands()) {
1581 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1582 !TRI->regsOverlap(MOP.getReg(), RegToRename))
1584 if (!canRenameMOP(MOP, TRI)) {
1585 LLVM_DEBUG(dbgs() <<
" Cannot rename " << MOP <<
" in " << MI);
1611 auto AnySubOrSuperRegCalleePreserved = [&MF,
TRI](
MCPhysReg PR) {
1612 return any_of(
TRI->sub_and_superregs_inclusive(PR),
1614 return TRI->isCalleeSavedPhysReg(SubOrSuper, MF);
1620 auto CanBeUsedForAllClasses = [&RequiredClasses,
TRI](
MCPhysReg PR) {
1623 TRI->sub_and_superregs_inclusive(PR),
1624 [
C](
MCPhysReg SubOrSuper) { return C->contains(SubOrSuper); });
1628 auto *RegClass =
TRI->getMinimalPhysRegClass(Reg);
1631 !
RegInfo.isReserved(PR) && !AnySubOrSuperRegCalleePreserved(PR) &&
1632 CanBeUsedForAllClasses(PR)) {
1640 <<
TRI->getRegClassName(RegClass) <<
"\n");
1641 return std::nullopt;
1652 std::optional<MCPhysReg> RenameReg;
1661 const bool IsLoad = FirstMI.
mayLoad();
1663 if (!MaybeCanRename) {
1666 RequiredClasses,
TRI)};
1672 if (*MaybeCanRename) {
1674 RequiredClasses,
TRI);
1683 LdStPairFlags &Flags,
unsigned Limit,
1684 bool FindNarrowMerge) {
1692 bool IsUnscaled =
TII->hasUnscaledLdStOffset(FirstMI);
1696 int OffsetStride = IsUnscaled ?
TII->getMemScale(FirstMI) : 1;
1699 std::optional<bool> MaybeCanRename;
1701 MaybeCanRename = {
false};
1707 Flags.clearRenameReg();
1711 ModifiedRegUnits.clear();
1712 UsedRegUnits.clear();
1718 for (
unsigned Count = 0;
MBBI != E && Count < Limit;
1727 if (!
MI.isTransient())
1730 Flags.setSExtIdx(-1);
1733 assert(
MI.mayLoadOrStore() &&
"Expected memory operation.");
1742 bool MIIsUnscaled =
TII->hasUnscaledLdStOffset(
MI);
1743 if (IsUnscaled != MIIsUnscaled) {
1747 int MemSize =
TII->getMemScale(
MI);
1751 if (MIOffset % MemSize) {
1757 MIOffset /= MemSize;
1759 MIOffset *= MemSize;
1765 if (BaseReg == MIBaseReg) {
1771 bool IsOutOfBounds = MIOffset !=
TII->getMemScale(
MI);
1772 bool IsBaseRegUsed = !UsedRegUnits.available(
1774 bool IsBaseRegModified = !ModifiedRegUnits.available(
1779 bool IsMIRegTheSame =
1782 if (IsOutOfBounds || IsBaseRegUsed || IsBaseRegModified ||
1790 if ((
Offset != MIOffset + OffsetStride) &&
1791 (
Offset + OffsetStride != MIOffset)) {
1800 if (FindNarrowMerge) {
1805 if ((!IsUnscaled &&
alignTo(MinOffset, 2) != MinOffset) ||
1822 <<
"keep looking.\n");
1828 if (IsUnscaled && (
alignTo(MinOffset, OffsetStride) != MinOffset)) {
1833 <<
"Offset doesn't fit due to alignment requirements, "
1834 <<
"keep looking.\n");
1845 if (!ModifiedRegUnits.available(BaseReg))
1848 const bool SameLoadReg =
MayLoad &&
TRI->isSuperOrSubRegisterEq(
1855 bool RtNotModified =
1857 bool RtNotUsed = !(
MI.mayLoad() && !SameLoadReg &&
1860 LLVM_DEBUG(
dbgs() <<
"Checking, can combine 2nd into 1st insn:\n"
1862 << (RtNotModified ?
"true" :
"false") <<
"\n"
1864 << (RtNotUsed ?
"true" :
"false") <<
"\n");
1866 if (RtNotModified && RtNotUsed && !
mayAlias(
MI, MemInsns, AA)) {
1871 std::optional<MCPhysReg> RenameReg =
1873 Reg, DefinedInBB, UsedInBetween,
1874 RequiredClasses,
TRI);
1880 <<
"keep looking.\n");
1883 Flags.setRenameReg(*RenameReg);
1886 Flags.setMergeForward(
false);
1888 Flags.clearRenameReg();
1899 LLVM_DEBUG(
dbgs() <<
"Checking, can combine 1st into 2nd insn:\n"
1901 <<
"' not modified: "
1902 << (RtNotModified ?
"true" :
"false") <<
"\n");
1904 if (RtNotModified && !
mayAlias(FirstMI, MemInsns, AA)) {
1906 Flags.setMergeForward(
true);
1907 Flags.clearRenameReg();
1912 MaybeCanRename, FirstMI,
MI, Reg, DefinedInBB, UsedInBetween,
1913 RequiredClasses,
TRI);
1915 Flags.setMergeForward(
true);
1916 Flags.setRenameReg(*RenameReg);
1917 MBBIWithRenameReg =
MBBI;
1920 LLVM_DEBUG(
dbgs() <<
"Unable to combine these instructions due to "
1921 <<
"interference in between, keep looking.\n");
1925 if (
Flags.getRenameReg())
1926 return MBBIWithRenameReg;
1940 if (!ModifiedRegUnits.available(BaseReg)) {
1946 if (
MI.mayLoadOrStore())
1954 auto End =
MI.getParent()->end();
1955 if (MaybeCFI ==
End ||
1956 MaybeCFI->getOpcode() != TargetOpcode::CFI_INSTRUCTION ||
1963 unsigned CFIIndex = MaybeCFI->getOperand(0).getCFIIndex();
1978 assert((Update->getOpcode() == AArch64::ADDXri ||
1979 Update->getOpcode() == AArch64::SUBXri) &&
1980 "Unexpected base register update instruction to merge!");
1992 if (NextI == Update)
1995 int Value = Update->getOperand(2).getImm();
1997 "Can't merge 1 << 12 offset into pre-/post-indexed load / store");
1998 if (Update->getOpcode() == AArch64::SUBXri)
2004 int Scale, MinOffset, MaxOffset;
2008 MIB =
BuildMI(*
I->getParent(),
I,
I->getDebugLoc(),
TII->get(NewOpc))
2017 MIB =
BuildMI(*
I->getParent(),
I,
I->getDebugLoc(),
TII->get(NewOpc))
2047 I->eraseFromParent();
2048 Update->eraseFromParent();
2053bool AArch64LoadStoreOpt::isMatchingUpdateInsn(
MachineInstr &MemMI,
2055 unsigned BaseReg,
int Offset) {
2056 switch (
MI.getOpcode()) {
2059 case AArch64::SUBXri:
2060 case AArch64::ADDXri:
2063 if (!
MI.getOperand(2).isImm())
2071 if (
MI.getOperand(0).getReg() != BaseReg ||
2072 MI.getOperand(1).getReg() != BaseReg)
2075 int UpdateOffset =
MI.getOperand(2).getImm();
2076 if (
MI.getOpcode() == AArch64::SUBXri)
2077 UpdateOffset = -UpdateOffset;
2081 int Scale, MinOffset, MaxOffset;
2083 if (UpdateOffset % Scale != 0)
2087 int ScaledOffset = UpdateOffset / Scale;
2088 if (ScaledOffset > MaxOffset || ScaledOffset < MinOffset)
2108 TII->getMemScale(MemMI);
2113 if (MIUnscaledOffset != UnscaledOffset)
2124 for (
unsigned i = 0, e = IsPairedInsn ? 2 : 1; i !=
e; ++i) {
2126 if (DestReg == BaseReg ||
TRI->isSubRegister(BaseReg, DestReg))
2133 ModifiedRegUnits.clear();
2134 UsedRegUnits.clear();
2140 const bool BaseRegSP = BaseReg == AArch64::SP;
2148 for (
unsigned Count = 0;
MBBI != E && Count < Limit;
2154 if (!
MI.isTransient())
2158 if (isMatchingUpdateInsn(*
I,
MI, BaseReg, UnscaledOffset))
2168 if (!ModifiedRegUnits.available(BaseReg) ||
2169 !UsedRegUnits.available(BaseReg) ||
2170 (BaseRegSP &&
MBBI->mayLoadOrStore()))
2195 for (
unsigned i = 0, e = IsPairedInsn ? 2 : 1; i !=
e; ++i) {
2197 if (DestReg == BaseReg ||
TRI->isSubRegister(BaseReg, DestReg))
2202 const bool BaseRegSP = BaseReg == AArch64::SP;
2211 unsigned RedZoneSize =
2216 ModifiedRegUnits.clear();
2217 UsedRegUnits.clear();
2219 bool MemAcessBeforeSPPreInc =
false;
2226 if (!
MI.isTransient())
2230 if (isMatchingUpdateInsn(*
I,
MI, BaseReg,
Offset)) {
2233 if (MemAcessBeforeSPPreInc &&
MBBI->getOperand(2).getImm() > RedZoneSize)
2243 if (!ModifiedRegUnits.available(BaseReg) ||
2244 !UsedRegUnits.available(BaseReg))
2249 if (BaseRegSP &&
MBBI->mayLoadOrStore())
2250 MemAcessBeforeSPPreInc =
true;
2251 }
while (
MBBI !=
B && Count < Limit);
2255bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
2259 if (
MI.hasOrderedMemoryRef())
2273 ++NumLoadsFromStoresPromoted;
2277 MBBI = promoteLoadFromStore(
MBBI, StoreI);
2284bool AArch64LoadStoreOpt::tryToMergeZeroStInst(
2290 if (!
TII->isCandidateToMergeOrPair(
MI))
2294 LdStPairFlags
Flags;
2298 ++NumZeroStoresPromoted;
2302 MBBI = mergeNarrowZeroStores(
MBBI, MergeMI, Flags);
2314 if (!
TII->isCandidateToMergeOrPair(
MI))
2318 if (
MI.mayLoad() && Subtarget->hasDisableLdp())
2322 if (
MI.mayStore() && Subtarget->hasDisableStp())
2328 bool IsUnscaled =
TII->hasUnscaledLdStOffset(
MI);
2330 int OffsetStride = IsUnscaled ?
TII->getMemScale(
MI) : 1;
2338 LdStPairFlags
Flags;
2344 auto Prev = std::prev(
MBBI);
2349 MI.memoperands_empty() ? nullptr :
MI.memoperands().front();
2354 if ((
MI.mayLoad() && Subtarget->hasLdpAlignedOnly()) ||
2355 (
MI.mayStore() && Subtarget->hasStpAlignedOnly())) {
2357 if (!
MemOp || !
MemOp->getMemoryType().isValid()) {
2358 NumFailedAlignmentCheck++;
2367 if (MemAlignment < 2 * TypeAlignment) {
2368 NumFailedAlignmentCheck++;
2374 if (
TII->hasUnscaledLdStOffset(
MI))
2375 ++NumUnscaledPairCreated;
2377 MBBI = mergePairedInsns(
MBBI, Paired, Flags);
2380 for (
auto I = std::next(Prev);
I !=
MBBI;
I++)
2388bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
2402 MBBI = mergeUpdateInsn(
MBBI, Update,
false);
2407 if (
TII->hasUnscaledLdStOffset(
MI.getOpcode()))
2418 MBBI = mergeUpdateInsn(
MBBI, Update,
true);
2425 int UnscaledOffset =
2433 Update = findMatchingUpdateInsnForward(
MBBI, UnscaledOffset,
UpdateLimit);
2436 MBBI = mergeUpdateInsn(
MBBI, Update,
true);
2444 bool EnableNarrowZeroStOpt) {
2475 if (EnableNarrowZeroStOpt)
2492 DefinedInBB.
clear();
2493 DefinedInBB.addLiveIns(
MBB);
2501 if (
TII->isPairableLdStInst(*
MBBI) && tryToPairLdStInst(
MBBI))
2531 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
2536 ModifiedRegUnits.init(*
TRI);
2537 UsedRegUnits.init(*
TRI);
2538 DefinedInBB.init(*
TRI);
2541 bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign();
2542 for (
auto &
MBB : Fn) {
2563 return new AArch64LoadStoreOpt();
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
static cl::opt< bool > EnableRenaming("aarch64-load-store-renaming", cl::init(true), cl::Hidden)
static MachineOperand & getLdStRegOp(MachineInstr &MI, unsigned PairedRegOp=0)
static bool isPromotableLoadFromStore(MachineInstr &MI)
static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale, int &MinOffset, int &MaxOffset)
static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride)
static unsigned getMatchingPairOpcode(unsigned Opc)
static bool isMergeableLdStUpdate(MachineInstr &MI)
static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI, LdStPairFlags &Flags, const AArch64InstrInfo *TII)
static std::optional< MCPhysReg > tryToFindRegisterToRename(const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static bool needsWinCFI(const MachineFunction *MF)
static bool canRenameUntilSecondLoad(MachineInstr &FirstLoad, MachineInstr &SecondLoad, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static std::optional< MCPhysReg > findRenameRegForSameLdStRegPair(std::optional< bool > MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static bool mayAlias(MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
static cl::opt< unsigned > LdStLimit("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden)
static bool canRenameMOP(const MachineOperand &MOP, const TargetRegisterInfo *TRI)
static unsigned getPreIndexedOpcode(unsigned Opc)
#define AARCH64_LOAD_STORE_OPT_NAME
static cl::opt< unsigned > UpdateLimit("aarch64-update-scan-limit", cl::init(100), cl::Hidden)
static bool isPromotableZeroStoreInst(MachineInstr &MI)
static unsigned getMatchingWideOpcode(unsigned Opc)
static unsigned getMatchingNonSExtOpcode(unsigned Opc, bool *IsValidLdStrOpc=nullptr)
static MachineBasicBlock::iterator maybeMoveCFI(MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI)
static int alignTo(int Num, int PowOf2)
static bool isTagStore(const MachineInstr &MI)
static unsigned isMatchingStore(MachineInstr &LoadInst, MachineInstr &StoreInst)
static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg, const TargetRegisterInfo *TRI, unsigned Limit, std::function< bool(MachineInstr &, bool)> &Fn)
static bool isRewritableImplicitDef(unsigned Opc)
static unsigned getPostIndexedOpcode(unsigned Opc)
static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst, MachineInstr &StoreInst, const AArch64InstrInfo *TII)
static bool isPreLdStPairCandidate(MachineInstr &FirstMI, MachineInstr &MI)
static void updateDefinedRegisters(MachineInstr &MI, LiveRegUnits &Units, const TargetRegisterInfo *TRI)
static bool canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, DomTreeUpdater *DTU)
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
const AArch64RegisterInfo * getRegisterInfo() const override
const AArch64InstrInfo * getInstrInfo() const override
const AArch64TargetLowering * getTargetLowering() const override
unsigned getRedZoneSize(const Function &F) const
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
static bool shouldExecute(unsigned CounterName)
FunctionPass class - This class is used to implement most global optimizations.
bool needsUnwindTableEntry() const
True if this function needs an unwind table.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
A set of register units used to track register liveness.
static void accumulateUsedDefed(const MachineInstr &MI, LiveRegUnits &ModifiedRegUnits, LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI)
For a machine instruction MI, adds all register units used in UsedRegUnits and defined or clobbered i...
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void init(const TargetRegisterInfo &TRI)
Initialize and clear the set.
void addReg(MCPhysReg Reg)
Adds register units covered by physical register Reg.
void removeReg(MCPhysReg Reg)
Removes all register units covered by physical register Reg.
void accumulate(const MachineInstr &MI)
Adds all register units used, defined or clobbered in MI.
An instruction for reading from memory.
bool usesWindowsCFI() const
OpType getOperation() const
Wrapper class representing physical registers. Should be passed by value.
reverse_instr_iterator instr_rend()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const std::vector< MCCFIInstruction > & getFrameInstructions() const
Returns a reference to a list of cfi instructions in the function's prologue.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & cloneMergedMemRefs(ArrayRef< const MachineInstr * > OtherMIs) const
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const
Returns true if this instruction's memory access aliases the memory access of Other.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
iterator_range< mop_iterator > operands()
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
bool isPseudo(QueryType Type=IgnoreBundle) const
Return true if this is a pseudo instruction that doesn't correspond to a real machine instruction.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
void setImplicit(bool Val=true)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
void setIsKill(bool Val=true)
bool isRenamable() const
isRenamable - Returns true if this register may be renamed, i.e.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
bool isEarlyClobber() const
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
LLVM Value Representation.
self_iterator getIterator()
A range adaptor for a pair of iterators.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ Define
Register definition.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< filter_iterator< ConstMIBundleOperands, bool(*)(const MachineOperand &)> > phys_regs_and_masks(const MachineInstr &MI)
Returns an iterator range over all physical register and mask operands for MI and bundled instruction...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createAArch64LoadStoreOptimizationPass()
createAArch64LoadStoreOptimizationPass - returns an instance of the load / store optimization pass.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
void initializeAArch64LoadStoreOptPass(PassRegistry &)
IterT prev_nodbg(IterT It, IterT Begin, bool SkipPseudoOp=true)
Decrement It, then continue decrementing it while it points to a debug instruction.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.