56#define DEBUG_TYPE "aarch64-ldst-opt"
58STATISTIC(NumPairCreated,
"Number of load/store pair instructions generated");
59STATISTIC(NumPostFolded,
"Number of post-index updates folded");
60STATISTIC(NumPreFolded,
"Number of pre-index updates folded");
62 "Number of load/store from unscaled generated");
63STATISTIC(NumZeroStoresPromoted,
"Number of narrow zero stores promoted");
64STATISTIC(NumLoadsFromStoresPromoted,
"Number of loads from stores promoted");
65STATISTIC(NumFailedAlignmentCheck,
"Number of load/store pair transformation "
66 "not passed the alignment check");
68 "Number of const offset of index address folded");
71 "Controls which pairs are considered for renaming");
91#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"
95using LdStPairFlags =
struct LdStPairFlags {
99 bool MergeForward =
false;
110 std::optional<MCPhysReg> RenameReg;
112 LdStPairFlags() =
default;
114 void setMergeForward(
bool V =
true) { MergeForward = V; }
115 bool getMergeForward()
const {
return MergeForward; }
117 void setSExtIdx(
int V) { SExtIdx = V; }
118 int getSExtIdx()
const {
return SExtIdx; }
120 void setRenameReg(
MCPhysReg R) { RenameReg = R; }
121 void clearRenameReg() { RenameReg = std::nullopt; }
122 std::optional<MCPhysReg> getRenameReg()
const {
return RenameReg; }
150 LdStPairFlags &Flags,
152 bool FindNarrowMerge);
163 const LdStPairFlags &Flags);
169 const LdStPairFlags &Flags);
181 int UnscaledOffset,
unsigned Limit);
199 unsigned BaseReg,
int Offset);
202 unsigned IndexReg,
unsigned &
Offset);
235 MachineFunctionProperties::Property::NoVRegs);
241char AArch64LoadStoreOpt::ID = 0;
248static
bool isNarrowStore(
unsigned Opc) {
252 case AArch64::STRBBui:
253 case AArch64::STURBBi:
254 case AArch64::STRHHui:
255 case AArch64::STURHHi:
263 switch (
MI.getOpcode()) {
269 case AArch64::STZ2Gi:
275 bool *IsValidLdStrOpc =
nullptr) {
277 *IsValidLdStrOpc =
true;
281 *IsValidLdStrOpc =
false;
282 return std::numeric_limits<unsigned>::max();
283 case AArch64::STRDui:
284 case AArch64::STURDi:
285 case AArch64::STRDpre:
286 case AArch64::STRQui:
287 case AArch64::STURQi:
288 case AArch64::STRQpre:
289 case AArch64::STRBBui:
290 case AArch64::STURBBi:
291 case AArch64::STRHHui:
292 case AArch64::STURHHi:
293 case AArch64::STRWui:
294 case AArch64::STRWpre:
295 case AArch64::STURWi:
296 case AArch64::STRXui:
297 case AArch64::STRXpre:
298 case AArch64::STURXi:
299 case AArch64::LDRDui:
300 case AArch64::LDURDi:
301 case AArch64::LDRDpre:
302 case AArch64::LDRQui:
303 case AArch64::LDURQi:
304 case AArch64::LDRQpre:
305 case AArch64::LDRWui:
306 case AArch64::LDURWi:
307 case AArch64::LDRWpre:
308 case AArch64::LDRXui:
309 case AArch64::LDURXi:
310 case AArch64::LDRXpre:
311 case AArch64::STRSui:
312 case AArch64::STURSi:
313 case AArch64::STRSpre:
314 case AArch64::LDRSui:
315 case AArch64::LDURSi:
316 case AArch64::LDRSpre:
318 case AArch64::LDRSWui:
319 return AArch64::LDRWui;
320 case AArch64::LDURSWi:
321 return AArch64::LDURWi;
322 case AArch64::LDRSWpre:
323 return AArch64::LDRWpre;
331 case AArch64::STRBBui:
332 return AArch64::STRHHui;
333 case AArch64::STRHHui:
334 return AArch64::STRWui;
335 case AArch64::STURBBi:
336 return AArch64::STURHHi;
337 case AArch64::STURHHi:
338 return AArch64::STURWi;
339 case AArch64::STURWi:
340 return AArch64::STURXi;
341 case AArch64::STRWui:
342 return AArch64::STRXui;
350 case AArch64::STRSui:
351 case AArch64::STURSi:
352 return AArch64::STPSi;
353 case AArch64::STRSpre:
354 return AArch64::STPSpre;
355 case AArch64::STRDui:
356 case AArch64::STURDi:
357 return AArch64::STPDi;
358 case AArch64::STRDpre:
359 return AArch64::STPDpre;
360 case AArch64::STRQui:
361 case AArch64::STURQi:
362 return AArch64::STPQi;
363 case AArch64::STRQpre:
364 return AArch64::STPQpre;
365 case AArch64::STRWui:
366 case AArch64::STURWi:
367 return AArch64::STPWi;
368 case AArch64::STRWpre:
369 return AArch64::STPWpre;
370 case AArch64::STRXui:
371 case AArch64::STURXi:
372 return AArch64::STPXi;
373 case AArch64::STRXpre:
374 return AArch64::STPXpre;
375 case AArch64::LDRSui:
376 case AArch64::LDURSi:
377 return AArch64::LDPSi;
378 case AArch64::LDRSpre:
379 return AArch64::LDPSpre;
380 case AArch64::LDRDui:
381 case AArch64::LDURDi:
382 return AArch64::LDPDi;
383 case AArch64::LDRDpre:
384 return AArch64::LDPDpre;
385 case AArch64::LDRQui:
386 case AArch64::LDURQi:
387 return AArch64::LDPQi;
388 case AArch64::LDRQpre:
389 return AArch64::LDPQpre;
390 case AArch64::LDRWui:
391 case AArch64::LDURWi:
392 return AArch64::LDPWi;
393 case AArch64::LDRWpre:
394 return AArch64::LDPWpre;
395 case AArch64::LDRXui:
396 case AArch64::LDURXi:
397 return AArch64::LDPXi;
398 case AArch64::LDRXpre:
399 return AArch64::LDPXpre;
400 case AArch64::LDRSWui:
401 case AArch64::LDURSWi:
402 return AArch64::LDPSWi;
403 case AArch64::LDRSWpre:
404 return AArch64::LDPSWpre;
415 case AArch64::LDRBBui:
416 return StOpc == AArch64::STRBBui || StOpc == AArch64::STRHHui ||
417 StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
418 case AArch64::LDURBBi:
419 return StOpc == AArch64::STURBBi || StOpc == AArch64::STURHHi ||
420 StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
421 case AArch64::LDRHHui:
422 return StOpc == AArch64::STRHHui || StOpc == AArch64::STRWui ||
423 StOpc == AArch64::STRXui;
424 case AArch64::LDURHHi:
425 return StOpc == AArch64::STURHHi || StOpc == AArch64::STURWi ||
426 StOpc == AArch64::STURXi;
427 case AArch64::LDRWui:
428 return StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
429 case AArch64::LDURWi:
430 return StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
431 case AArch64::LDRXui:
432 return StOpc == AArch64::STRXui;
433 case AArch64::LDURXi:
434 return StOpc == AArch64::STURXi;
446 case AArch64::STRSui:
447 return AArch64::STRSpre;
448 case AArch64::STRDui:
449 return AArch64::STRDpre;
450 case AArch64::STRQui:
451 return AArch64::STRQpre;
452 case AArch64::STRBBui:
453 return AArch64::STRBBpre;
454 case AArch64::STRHHui:
455 return AArch64::STRHHpre;
456 case AArch64::STRWui:
457 return AArch64::STRWpre;
458 case AArch64::STRXui:
459 return AArch64::STRXpre;
460 case AArch64::LDRSui:
461 return AArch64::LDRSpre;
462 case AArch64::LDRDui:
463 return AArch64::LDRDpre;
464 case AArch64::LDRQui:
465 return AArch64::LDRQpre;
466 case AArch64::LDRBBui:
467 return AArch64::LDRBBpre;
468 case AArch64::LDRHHui:
469 return AArch64::LDRHHpre;
470 case AArch64::LDRWui:
471 return AArch64::LDRWpre;
472 case AArch64::LDRXui:
473 return AArch64::LDRXpre;
474 case AArch64::LDRSWui:
475 return AArch64::LDRSWpre;
477 return AArch64::LDPSpre;
478 case AArch64::LDPSWi:
479 return AArch64::LDPSWpre;
481 return AArch64::LDPDpre;
483 return AArch64::LDPQpre;
485 return AArch64::LDPWpre;
487 return AArch64::LDPXpre;
489 return AArch64::STPSpre;
491 return AArch64::STPDpre;
493 return AArch64::STPQpre;
495 return AArch64::STPWpre;
497 return AArch64::STPXpre;
499 return AArch64::STGPreIndex;
501 return AArch64::STZGPreIndex;
503 return AArch64::ST2GPreIndex;
504 case AArch64::STZ2Gi:
505 return AArch64::STZ2GPreIndex;
507 return AArch64::STGPpre;
516 case AArch64::LDRBroX:
517 return AArch64::LDRBui;
518 case AArch64::LDRBBroX:
519 return AArch64::LDRBBui;
520 case AArch64::LDRSBXroX:
521 return AArch64::LDRSBXui;
522 case AArch64::LDRSBWroX:
523 return AArch64::LDRSBWui;
524 case AArch64::LDRHroX:
525 return AArch64::LDRHui;
526 case AArch64::LDRHHroX:
527 return AArch64::LDRHHui;
528 case AArch64::LDRSHXroX:
529 return AArch64::LDRSHXui;
530 case AArch64::LDRSHWroX:
531 return AArch64::LDRSHWui;
532 case AArch64::LDRWroX:
533 return AArch64::LDRWui;
534 case AArch64::LDRSroX:
535 return AArch64::LDRSui;
536 case AArch64::LDRSWroX:
537 return AArch64::LDRSWui;
538 case AArch64::LDRDroX:
539 return AArch64::LDRDui;
540 case AArch64::LDRXroX:
541 return AArch64::LDRXui;
542 case AArch64::LDRQroX:
543 return AArch64::LDRQui;
551 case AArch64::STRSui:
552 case AArch64::STURSi:
553 return AArch64::STRSpost;
554 case AArch64::STRDui:
555 case AArch64::STURDi:
556 return AArch64::STRDpost;
557 case AArch64::STRQui:
558 case AArch64::STURQi:
559 return AArch64::STRQpost;
560 case AArch64::STRBBui:
561 return AArch64::STRBBpost;
562 case AArch64::STRHHui:
563 return AArch64::STRHHpost;
564 case AArch64::STRWui:
565 case AArch64::STURWi:
566 return AArch64::STRWpost;
567 case AArch64::STRXui:
568 case AArch64::STURXi:
569 return AArch64::STRXpost;
570 case AArch64::LDRSui:
571 case AArch64::LDURSi:
572 return AArch64::LDRSpost;
573 case AArch64::LDRDui:
574 case AArch64::LDURDi:
575 return AArch64::LDRDpost;
576 case AArch64::LDRQui:
577 case AArch64::LDURQi:
578 return AArch64::LDRQpost;
579 case AArch64::LDRBBui:
580 return AArch64::LDRBBpost;
581 case AArch64::LDRHHui:
582 return AArch64::LDRHHpost;
583 case AArch64::LDRWui:
584 case AArch64::LDURWi:
585 return AArch64::LDRWpost;
586 case AArch64::LDRXui:
587 case AArch64::LDURXi:
588 return AArch64::LDRXpost;
589 case AArch64::LDRSWui:
590 return AArch64::LDRSWpost;
592 return AArch64::LDPSpost;
593 case AArch64::LDPSWi:
594 return AArch64::LDPSWpost;
596 return AArch64::LDPDpost;
598 return AArch64::LDPQpost;
600 return AArch64::LDPWpost;
602 return AArch64::LDPXpost;
604 return AArch64::STPSpost;
606 return AArch64::STPDpost;
608 return AArch64::STPQpost;
610 return AArch64::STPWpost;
612 return AArch64::STPXpost;
614 return AArch64::STGPostIndex;
616 return AArch64::STZGPostIndex;
618 return AArch64::ST2GPostIndex;
619 case AArch64::STZ2Gi:
620 return AArch64::STZ2GPostIndex;
622 return AArch64::STGPpost;
629 unsigned OpcB =
MI.getOpcode();
634 case AArch64::STRSpre:
635 return (OpcB == AArch64::STRSui) || (OpcB == AArch64::STURSi);
636 case AArch64::STRDpre:
637 return (OpcB == AArch64::STRDui) || (OpcB == AArch64::STURDi);
638 case AArch64::STRQpre:
639 return (OpcB == AArch64::STRQui) || (OpcB == AArch64::STURQi);
640 case AArch64::STRWpre:
641 return (OpcB == AArch64::STRWui) || (OpcB == AArch64::STURWi);
642 case AArch64::STRXpre:
643 return (OpcB == AArch64::STRXui) || (OpcB == AArch64::STURXi);
644 case AArch64::LDRSpre:
645 return (OpcB == AArch64::LDRSui) || (OpcB == AArch64::LDURSi);
646 case AArch64::LDRDpre:
647 return (OpcB == AArch64::LDRDui) || (OpcB == AArch64::LDURDi);
648 case AArch64::LDRQpre:
649 return (OpcB == AArch64::LDRQui) || (OpcB == AArch64::LDURQi);
650 case AArch64::LDRWpre:
651 return (OpcB == AArch64::LDRWui) || (OpcB == AArch64::LDURWi);
652 case AArch64::LDRXpre:
653 return (OpcB == AArch64::LDRXui) || (OpcB == AArch64::LDURXi);
654 case AArch64::LDRSWpre:
655 return (OpcB == AArch64::LDRSWui) || (OpcB == AArch64::LDURSWi);
661 int &MinOffset,
int &MaxOffset) {
679 unsigned PairedRegOp = 0) {
680 assert(PairedRegOp < 2 &&
"Unexpected register operand idx.");
686 return MI.getOperand(
Idx);
695 int UnscaledStOffset =
699 int UnscaledLdOffset =
703 return (UnscaledStOffset <= UnscaledLdOffset) &&
704 (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize));
708 unsigned Opc =
MI.getOpcode();
709 return (Opc == AArch64::STRWui || Opc == AArch64::STURWi ||
710 isNarrowStore(Opc)) &&
715 switch (
MI.getOpcode()) {
719 case AArch64::LDRBBui:
720 case AArch64::LDRHHui:
721 case AArch64::LDRWui:
722 case AArch64::LDRXui:
724 case AArch64::LDURBBi:
725 case AArch64::LDURHHi:
726 case AArch64::LDURWi:
727 case AArch64::LDURXi:
733 unsigned Opc =
MI.getOpcode();
738 case AArch64::STRSui:
739 case AArch64::STRDui:
740 case AArch64::STRQui:
741 case AArch64::STRXui:
742 case AArch64::STRWui:
743 case AArch64::STRHHui:
744 case AArch64::STRBBui:
745 case AArch64::LDRSui:
746 case AArch64::LDRDui:
747 case AArch64::LDRQui:
748 case AArch64::LDRXui:
749 case AArch64::LDRWui:
750 case AArch64::LDRHHui:
751 case AArch64::LDRBBui:
755 case AArch64::STZ2Gi:
758 case AArch64::STURSi:
759 case AArch64::STURDi:
760 case AArch64::STURQi:
761 case AArch64::STURWi:
762 case AArch64::STURXi:
763 case AArch64::LDURSi:
764 case AArch64::LDURDi:
765 case AArch64::LDURQi:
766 case AArch64::LDURWi:
767 case AArch64::LDURXi:
770 case AArch64::LDPSWi:
790 unsigned Opc =
MI.getOpcode();
796 case AArch64::LDRBroX:
797 case AArch64::LDRBBroX:
798 case AArch64::LDRSBXroX:
799 case AArch64::LDRSBWroX:
802 case AArch64::LDRHroX:
803 case AArch64::LDRHHroX:
804 case AArch64::LDRSHXroX:
805 case AArch64::LDRSHWroX:
808 case AArch64::LDRWroX:
809 case AArch64::LDRSroX:
810 case AArch64::LDRSWroX:
813 case AArch64::LDRDroX:
814 case AArch64::LDRXroX:
817 case AArch64::LDRQroX:
827 case AArch64::ORRWrs:
828 case AArch64::ADDWri:
836 const LdStPairFlags &Flags) {
838 "Expected promotable zero stores.");
846 if (NextI == MergeMI)
849 unsigned Opc =
I->getOpcode();
850 unsigned MergeMIOpc = MergeMI->getOpcode();
851 bool IsScaled = !
TII->hasUnscaledLdStOffset(Opc);
852 bool IsMergedMIScaled = !
TII->hasUnscaledLdStOffset(MergeMIOpc);
853 int OffsetStride = IsScaled ?
TII->getMemScale(*
I) : 1;
854 int MergeMIOffsetStride = IsMergedMIScaled ?
TII->getMemScale(*MergeMI) : 1;
856 bool MergeForward =
Flags.getMergeForward();
867 int64_t IOffsetInBytes =
869 int64_t MIOffsetInBytes =
874 if (IOffsetInBytes > MIOffsetInBytes)
875 OffsetImm = MIOffsetInBytes;
877 OffsetImm = IOffsetInBytes;
880 bool FinalIsScaled = !
TII->hasUnscaledLdStOffset(NewOpcode);
884 int NewOffsetStride = FinalIsScaled ?
TII->getMemScale(NewOpcode) : 1;
885 assert(((OffsetImm % NewOffsetStride) == 0) &&
886 "Offset should be a multiple of the store memory scale");
887 OffsetImm = OffsetImm / NewOffsetStride;
895 .
addReg(isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR)
899 .setMIFlags(
I->mergeFlagsWith(*MergeMI));
902 LLVM_DEBUG(
dbgs() <<
"Creating wider store. Replacing instructions:\n ");
911 I->eraseFromParent();
912 MergeMI->eraseFromParent();
922 auto MBB =
MI.getParent();
930 return MOP.isReg() && MOP.isDef() && !MOP.isDebug() && MOP.getReg() &&
931 TRI->regsOverlap(MOP.getReg(), DefReg);
945 if (MOP.isReg() && MOP.isKill())
949 if (MOP.isReg() && !MOP.isKill())
950 Units.
addReg(MOP.getReg());
956 const LdStPairFlags &Flags) {
966 int SExtIdx =
Flags.getSExtIdx();
969 bool IsUnscaled =
TII->hasUnscaledLdStOffset(Opc);
970 int OffsetStride = IsUnscaled ?
TII->getMemScale(*
I) : 1;
972 bool MergeForward =
Flags.getMergeForward();
974 std::optional<MCPhysReg> RenameReg =
Flags.getRenameReg();
977 DefinedInBB.addReg(*RenameReg);
981 auto GetMatchingSubReg =
984 TRI->sub_and_superregs_inclusive(*RenameReg)) {
985 if (
C->contains(SubOrSuper))
992 [
this, RegToRename, GetMatchingSubReg, MergeForward](
MachineInstr &
MI,
995 bool SeenDef =
false;
996 for (
unsigned OpIdx = 0; OpIdx <
MI.getNumOperands(); ++OpIdx) {
1001 (!MergeForward || !SeenDef ||
1003 TRI->regsOverlap(MOP.
getReg(), RegToRename)) {
1006 "Need renamable operands");
1009 MI.getRegClassConstraint(OpIdx,
TII,
TRI))
1010 MatchingReg = GetMatchingSubReg(RC);
1014 MatchingReg = GetMatchingSubReg(
1015 TRI->getMinimalPhysRegClass(MOP.
getReg()));
1022 for (
unsigned OpIdx = 0; OpIdx <
MI.getNumOperands(); ++OpIdx) {
1025 TRI->regsOverlap(MOP.
getReg(), RegToRename)) {
1028 "Need renamable operands");
1031 MI.getRegClassConstraint(OpIdx,
TII,
TRI))
1032 MatchingReg = GetMatchingSubReg(RC);
1034 MatchingReg = GetMatchingSubReg(
1035 TRI->getMinimalPhysRegClass(MOP.
getReg()));
1036 assert(MatchingReg != AArch64::NoRegister &&
1037 "Cannot find matching regs for renaming");
1046 UINT32_MAX, UpdateMIs);
1059 RegToCheck = RegToRename;
1062 MergeForward ? std::next(
I) :
I,
1063 MergeForward ? std::next(Paired) : Paired))
1066 return !MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1068 !TRI->regsOverlap(MOP.getReg(), RegToCheck);
1070 "Rename register used between paired instruction, trashing the "
1086 bool PairedIsUnscaled =
TII->hasUnscaledLdStOffset(Paired->getOpcode());
1087 if (IsUnscaled != PairedIsUnscaled) {
1091 int MemSize =
TII->getMemScale(*Paired);
1092 if (PairedIsUnscaled) {
1095 assert(!(PairedOffset %
TII->getMemScale(*Paired)) &&
1096 "Offset should be a multiple of the stride!");
1097 PairedOffset /= MemSize;
1099 PairedOffset *= MemSize;
1107 if (
Offset == PairedOffset + OffsetStride &&
1115 SExtIdx = (SExtIdx + 1) % 2;
1123 assert(!(OffsetImm %
TII->getMemScale(*RtMI)) &&
1124 "Unscaled offset cannot be scaled.");
1125 OffsetImm /=
TII->getMemScale(*RtMI);
1134 MachineOperand &PairedRegOp = RtMI == &*Paired ? RegOp0 : RegOp1;
1136 if (RegOp0.
isUse()) {
1137 if (!MergeForward) {
1148 for (
auto It = std::next(
I); It != Paired && PairedRegOp.
isKill(); ++It)
1149 if (It->readsRegister(PairedRegOp.
getReg(),
TRI))
1158 MI.clearRegisterKills(Reg,
TRI);
1174 .setMIFlags(
I->mergeFlagsWith(*Paired));
1179 dbgs() <<
"Creating pair load/store. Replacing instructions:\n ");
1184 if (SExtIdx != -1) {
1194 Register DstRegW =
TRI->getSubReg(DstRegX, AArch64::sub_32);
1204 BuildMI(*
MBB, InsertionPoint,
DL,
TII->get(TargetOpcode::KILL), DstRegW)
1210 BuildMI(*
MBB, InsertionPoint,
DL,
TII->get(AArch64::SBFMXri), DstRegX)
1225 DefinedInBB.addReg(MOP.
getReg());
1228 I->eraseFromParent();
1229 Paired->eraseFromParent();
1238 next_nodbg(LoadI, LoadI->getParent()->end());
1240 int LoadSize =
TII->getMemScale(*LoadI);
1241 int StoreSize =
TII->getMemScale(*StoreI);
1245 bool IsStoreXReg =
TRI->getRegClass(AArch64::GPR64RegClassID)->contains(StRt);
1248 TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) &&
1249 "Unexpected RegClass");
1252 if (LoadSize == StoreSize && (LoadSize == 4 || LoadSize == 8)) {
1255 if (StRt == LdRt && LoadSize == 8) {
1257 LoadI->getIterator())) {
1258 if (
MI.killsRegister(StRt,
TRI)) {
1259 MI.clearRegisterKills(StRt,
TRI);
1266 LoadI->eraseFromParent();
1271 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1272 TII->get(IsStoreXReg ? AArch64::ORRXrs : AArch64::ORRWrs), LdRt)
1273 .
addReg(IsStoreXReg ? AArch64::XZR : AArch64::WZR)
1280 if (!Subtarget->isLittleEndian())
1282 bool IsUnscaled =
TII->hasUnscaledLdStOffset(*LoadI);
1283 assert(IsUnscaled ==
TII->hasUnscaledLdStOffset(*StoreI) &&
1284 "Unsupported ld/st match");
1285 assert(LoadSize <= StoreSize &&
"Invalid load size");
1286 int UnscaledLdOffset =
1290 int UnscaledStOffset =
1294 int Width = LoadSize * 8;
1297 LdRt, AArch64::sub_32, &AArch64::GPR64RegClass))
1300 assert((UnscaledLdOffset >= UnscaledStOffset &&
1301 (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) &&
1304 int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset);
1305 int Imms = Immr + Width - 1;
1306 if (UnscaledLdOffset == UnscaledStOffset) {
1307 uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12)
1313 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1314 TII->get(IsStoreXReg ? AArch64::ANDXri : AArch64::ANDWri),
1321 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1322 TII->get(IsStoreXReg ? AArch64::UBFMXri : AArch64::UBFMWri),
1334 if (
MI.killsRegister(StRt,
TRI)) {
1335 MI.clearRegisterKills(StRt,
TRI);
1350 LoadI->eraseFromParent();
1360 if (
Offset % OffsetStride)
1364 return Offset <= 63 && Offset >= -64;
1372 return (Num + PowOf2 - 1) & ~(PowOf2 - 1);
1379 if (MIa.
mayAlias(AA, *MIb,
false)) {
1389bool AArch64LoadStoreOpt::findMatchingStore(
1404 ModifiedRegUnits.clear();
1405 UsedRegUnits.clear();
1414 if (!
MI.isTransient())
1440 if (!ModifiedRegUnits.available(BaseReg))
1446 }
while (
MBBI !=
B && Count < Limit);
1458 LdStPairFlags &Flags,
1461 if (
MI.hasOrderedMemoryRef() ||
TII->isLdStPairSuppressed(
MI))
1466 !
TII->isLdStPairSuppressed(FirstMI) &&
1467 "FirstMI shouldn't get here if either of these checks are true.");
1474 unsigned OpcB =
MI.getOpcode();
1485 bool IsValidLdStrOpc, PairIsValidLdStrOpc;
1487 assert(IsValidLdStrOpc &&
1488 "Given Opc should be a Load or Store with an immediate");
1491 Flags.setSExtIdx(NonSExtOpc == (
unsigned)OpcA ? 1 : 0);
1497 if (!PairIsValidLdStrOpc)
1502 if (isNarrowStore(OpcA) || isNarrowStore(OpcB))
1512 return TII->hasUnscaledLdStOffset(OpcA) !=
TII->hasUnscaledLdStOffset(OpcB) &&
1521 auto *RegClass =
TRI->getMinimalPhysRegClass(MOP.
getReg());
1528 if (RegClass->HasDisjunctSubRegs) {
1531 <<
" Cannot rename operands with multiple disjunct subregisters ("
1542 return TRI->isSuperOrSubRegisterEq(
1565 return MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1566 MOP.isImplicit() && MOP.isKill() &&
1567 TRI->regsOverlap(RegToRename, MOP.getReg());
1573 bool FoundDef =
false;
1604 if (
MI.isPseudo()) {
1605 LLVM_DEBUG(
dbgs() <<
" Cannot rename pseudo/bundle instruction\n");
1609 for (
auto &MOP :
MI.operands()) {
1611 !
TRI->regsOverlap(MOP.
getReg(), RegToRename))
1621 for (
auto &MOP :
MI.operands()) {
1623 !
TRI->regsOverlap(MOP.
getReg(), RegToRename))
1640 LLVM_DEBUG(
dbgs() <<
" Did not find definition for register in BB\n");
1668 LLVM_DEBUG(dbgs() <<
"Checking " << MI);
1670 if (MI.getFlag(MachineInstr::FrameSetup)) {
1671 LLVM_DEBUG(dbgs() <<
" Cannot rename framesetup instructions "
1676 for (
auto &MOP :
MI.operands()) {
1677 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1678 !TRI->regsOverlap(MOP.getReg(), RegToRename))
1680 if (!canRenameMOP(MOP, TRI)) {
1681 LLVM_DEBUG(dbgs() <<
" Cannot rename " << MOP <<
" in " << MI);
1707 auto AnySubOrSuperRegCalleePreserved = [&MF,
TRI](
MCPhysReg PR) {
1708 return any_of(
TRI->sub_and_superregs_inclusive(PR),
1710 return TRI->isCalleeSavedPhysReg(SubOrSuper, MF);
1716 auto CanBeUsedForAllClasses = [&RequiredClasses,
TRI](
MCPhysReg PR) {
1719 TRI->sub_and_superregs_inclusive(PR),
1720 [
C](
MCPhysReg SubOrSuper) { return C->contains(SubOrSuper); });
1724 auto *RegClass =
TRI->getMinimalPhysRegClass(Reg);
1727 !
RegInfo.isReserved(PR) && !AnySubOrSuperRegCalleePreserved(PR) &&
1728 CanBeUsedForAllClasses(PR)) {
1736 <<
TRI->getRegClassName(RegClass) <<
"\n");
1737 return std::nullopt;
1748 std::optional<MCPhysReg> RenameReg;
1757 const bool IsLoad = FirstMI.
mayLoad();
1759 if (!MaybeCanRename) {
1762 RequiredClasses,
TRI)};
1768 if (*MaybeCanRename) {
1770 RequiredClasses,
TRI);
1779 LdStPairFlags &Flags,
unsigned Limit,
1780 bool FindNarrowMerge) {
1788 bool IsUnscaled =
TII->hasUnscaledLdStOffset(FirstMI);
1792 int OffsetStride = IsUnscaled ?
TII->getMemScale(FirstMI) : 1;
1795 std::optional<bool> MaybeCanRename;
1797 MaybeCanRename = {
false};
1803 Flags.clearRenameReg();
1807 ModifiedRegUnits.clear();
1808 UsedRegUnits.clear();
1814 for (
unsigned Count = 0;
MBBI != E && Count < Limit;
1823 if (!
MI.isTransient())
1826 Flags.setSExtIdx(-1);
1829 assert(
MI.mayLoadOrStore() &&
"Expected memory operation.");
1838 bool MIIsUnscaled =
TII->hasUnscaledLdStOffset(
MI);
1839 if (IsUnscaled != MIIsUnscaled) {
1843 int MemSize =
TII->getMemScale(
MI);
1847 if (MIOffset % MemSize) {
1853 MIOffset /= MemSize;
1855 MIOffset *= MemSize;
1861 if (BaseReg == MIBaseReg) {
1867 bool IsOutOfBounds = MIOffset !=
TII->getMemScale(
MI);
1868 bool IsBaseRegUsed = !UsedRegUnits.available(
1870 bool IsBaseRegModified = !ModifiedRegUnits.available(
1875 bool IsMIRegTheSame =
1878 if (IsOutOfBounds || IsBaseRegUsed || IsBaseRegModified ||
1886 if ((
Offset != MIOffset + OffsetStride) &&
1887 (
Offset + OffsetStride != MIOffset)) {
1896 if (FindNarrowMerge) {
1901 if ((!IsUnscaled &&
alignTo(MinOffset, 2) != MinOffset) ||
1918 <<
"keep looking.\n");
1924 if (IsUnscaled && (
alignTo(MinOffset, OffsetStride) != MinOffset)) {
1929 <<
"Offset doesn't fit due to alignment requirements, "
1930 <<
"keep looking.\n");
1941 if (!ModifiedRegUnits.available(BaseReg))
1944 const bool SameLoadReg =
MayLoad &&
TRI->isSuperOrSubRegisterEq(
1951 bool RtNotModified =
1953 bool RtNotUsed = !(
MI.mayLoad() && !SameLoadReg &&
1956 LLVM_DEBUG(
dbgs() <<
"Checking, can combine 2nd into 1st insn:\n"
1958 << (RtNotModified ?
"true" :
"false") <<
"\n"
1960 << (RtNotUsed ?
"true" :
"false") <<
"\n");
1962 if (RtNotModified && RtNotUsed && !
mayAlias(
MI, MemInsns, AA)) {
1967 std::optional<MCPhysReg> RenameReg =
1969 Reg, DefinedInBB, UsedInBetween,
1970 RequiredClasses,
TRI);
1976 <<
"keep looking.\n");
1979 Flags.setRenameReg(*RenameReg);
1982 Flags.setMergeForward(
false);
1984 Flags.clearRenameReg();
1995 LLVM_DEBUG(
dbgs() <<
"Checking, can combine 1st into 2nd insn:\n"
1997 <<
"' not modified: "
1998 << (RtNotModified ?
"true" :
"false") <<
"\n");
2000 if (RtNotModified && !
mayAlias(FirstMI, MemInsns, AA)) {
2002 Flags.setMergeForward(
true);
2003 Flags.clearRenameReg();
2008 MaybeCanRename, FirstMI,
MI, Reg, DefinedInBB, UsedInBetween,
2009 RequiredClasses,
TRI);
2011 Flags.setMergeForward(
true);
2012 Flags.setRenameReg(*RenameReg);
2013 MBBIWithRenameReg =
MBBI;
2016 LLVM_DEBUG(
dbgs() <<
"Unable to combine these instructions due to "
2017 <<
"interference in between, keep looking.\n");
2021 if (
Flags.getRenameReg())
2022 return MBBIWithRenameReg;
2036 if (!ModifiedRegUnits.available(BaseReg)) {
2042 if (
MI.mayLoadOrStore())
2050 assert((
MI.getOpcode() == AArch64::SUBXri ||
2051 MI.getOpcode() == AArch64::ADDXri) &&
2052 "Expected a register update instruction");
2053 auto End =
MI.getParent()->end();
2054 if (MaybeCFI ==
End ||
2055 MaybeCFI->getOpcode() != TargetOpcode::CFI_INSTRUCTION ||
2058 MI.getOperand(0).getReg() != AArch64::SP)
2062 unsigned CFIIndex = MaybeCFI->getOperand(0).getCFIIndex();
2077 assert((Update->getOpcode() == AArch64::ADDXri ||
2078 Update->getOpcode() == AArch64::SUBXri) &&
2079 "Unexpected base register update instruction to merge!");
2091 if (NextI == Update)
2094 int Value = Update->getOperand(2).getImm();
2096 "Can't merge 1 << 12 offset into pre-/post-indexed load / store");
2097 if (Update->getOpcode() == AArch64::SUBXri)
2103 int Scale, MinOffset, MaxOffset;
2107 MIB =
BuildMI(*
I->getParent(),
I,
I->getDebugLoc(),
TII->get(NewOpc))
2108 .
add(Update->getOperand(0))
2116 MIB =
BuildMI(*
I->getParent(),
I,
I->getDebugLoc(),
TII->get(NewOpc))
2117 .
add(Update->getOperand(0))
2146 I->eraseFromParent();
2147 Update->eraseFromParent();
2155 unsigned Offset,
int Scale) {
2156 assert((Update->getOpcode() == AArch64::MOVKWi) &&
2157 "Unexpected const mov instruction to merge!");
2162 unsigned Mask = (1 << 12) * Scale - 1;
2171 BuildMI(*
I->getParent(),
I,
I->getDebugLoc(),
TII->get(AArch64::ADDXri))
2179 MemMIB =
BuildMI(*
I->getParent(),
I,
I->getDebugLoc(),
TII->get(NewOpc))
2187 ++NumConstOffsetFolded;
2202 I->eraseFromParent();
2203 PrevI->eraseFromParent();
2204 Update->eraseFromParent();
2209bool AArch64LoadStoreOpt::isMatchingUpdateInsn(
MachineInstr &MemMI,
2211 unsigned BaseReg,
int Offset) {
2212 switch (
MI.getOpcode()) {
2215 case AArch64::SUBXri:
2216 case AArch64::ADDXri:
2219 if (!
MI.getOperand(2).isImm())
2227 if (
MI.getOperand(0).getReg() != BaseReg ||
2228 MI.getOperand(1).getReg() != BaseReg)
2231 int UpdateOffset =
MI.getOperand(2).getImm();
2232 if (
MI.getOpcode() == AArch64::SUBXri)
2233 UpdateOffset = -UpdateOffset;
2237 int Scale, MinOffset, MaxOffset;
2239 if (UpdateOffset % Scale != 0)
2243 int ScaledOffset = UpdateOffset / Scale;
2244 if (ScaledOffset > MaxOffset || ScaledOffset < MinOffset)
2256bool AArch64LoadStoreOpt::isMatchingMovConstInsn(
MachineInstr &MemMI,
2262 if (
MI.getOpcode() == AArch64::MOVKWi &&
2263 TRI->isSuperOrSubRegisterEq(IndexReg,
MI.getOperand(1).getReg())) {
2273 if (MovzMI.
getOpcode() == AArch64::MOVZWi) {
2275 unsigned High =
MI.getOperand(2).getImm() <<
MI.getOperand(3).getImm();
2278 return Offset >> 24 == 0;
2292 TII->getMemScale(MemMI);
2297 if (MIUnscaledOffset != UnscaledOffset)
2308 for (
unsigned i = 0, e = IsPairedInsn ? 2 : 1; i !=
e; ++i) {
2310 if (DestReg == BaseReg ||
TRI->isSubRegister(BaseReg, DestReg))
2317 ModifiedRegUnits.clear();
2318 UsedRegUnits.clear();
2324 const bool BaseRegSP = BaseReg == AArch64::SP;
2332 for (
unsigned Count = 0;
MBBI != E && Count < Limit;
2338 if (!
MI.isTransient())
2342 if (isMatchingUpdateInsn(*
I,
MI, BaseReg, UnscaledOffset))
2352 if (!ModifiedRegUnits.available(BaseReg) ||
2353 !UsedRegUnits.available(BaseReg) ||
2354 (BaseRegSP &&
MBBI->mayLoadOrStore()))
2379 for (
unsigned i = 0, e = IsPairedInsn ? 2 : 1; i !=
e; ++i) {
2381 if (DestReg == BaseReg ||
TRI->isSubRegister(BaseReg, DestReg))
2386 const bool BaseRegSP = BaseReg == AArch64::SP;
2395 unsigned RedZoneSize =
2400 ModifiedRegUnits.clear();
2401 UsedRegUnits.clear();
2403 bool MemAcessBeforeSPPreInc =
false;
2410 if (!
MI.isTransient())
2414 if (isMatchingUpdateInsn(*
I,
MI, BaseReg,
Offset)) {
2417 if (MemAcessBeforeSPPreInc &&
MBBI->getOperand(2).getImm() > RedZoneSize)
2427 if (!ModifiedRegUnits.available(BaseReg) ||
2428 !UsedRegUnits.available(BaseReg))
2433 if (BaseRegSP &&
MBBI->mayLoadOrStore())
2434 MemAcessBeforeSPPreInc =
true;
2435 }
while (
MBBI !=
B && Count < Limit);
2440AArch64LoadStoreOpt::findMatchingConstOffsetBackward(
2463 ModifiedRegUnits.clear();
2464 UsedRegUnits.clear();
2472 if (!
MI.isTransient())
2476 if (isMatchingMovConstInsn(*
I,
MI, IndexReg,
Offset)) {
2485 if (!ModifiedRegUnits.available(IndexReg) ||
2486 !UsedRegUnits.available(IndexReg))
2489 }
while (
MBBI !=
B && Count < Limit);
2493bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
2497 if (
MI.hasOrderedMemoryRef())
2511 ++NumLoadsFromStoresPromoted;
2515 MBBI = promoteLoadFromStore(
MBBI, StoreI);
2522bool AArch64LoadStoreOpt::tryToMergeZeroStInst(
2528 if (!
TII->isCandidateToMergeOrPair(
MI))
2532 LdStPairFlags
Flags;
2536 ++NumZeroStoresPromoted;
2540 MBBI = mergeNarrowZeroStores(
MBBI, MergeMI, Flags);
2552 if (!
TII->isCandidateToMergeOrPair(
MI))
2556 if (
MI.mayLoad() && Subtarget->hasDisableLdp())
2560 if (
MI.mayStore() && Subtarget->hasDisableStp())
2566 bool IsUnscaled =
TII->hasUnscaledLdStOffset(
MI);
2568 int OffsetStride = IsUnscaled ?
TII->getMemScale(
MI) : 1;
2576 LdStPairFlags
Flags;
2582 auto Prev = std::prev(
MBBI);
2587 MI.memoperands_empty() ? nullptr :
MI.memoperands().front();
2592 if ((
MI.mayLoad() && Subtarget->hasLdpAlignedOnly()) ||
2593 (
MI.mayStore() && Subtarget->hasStpAlignedOnly())) {
2595 if (!
MemOp || !
MemOp->getMemoryType().isValid()) {
2596 NumFailedAlignmentCheck++;
2605 if (MemAlignment < 2 * TypeAlignment) {
2606 NumFailedAlignmentCheck++;
2612 if (
TII->hasUnscaledLdStOffset(
MI))
2613 ++NumUnscaledPairCreated;
2615 MBBI = mergePairedInsns(
MBBI, Paired, Flags);
2618 for (
auto I = std::next(Prev);
I !=
MBBI;
I++)
2626bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
2640 MBBI = mergeUpdateInsn(
MBBI, Update,
false);
2645 if (
TII->hasUnscaledLdStOffset(
MI.getOpcode()))
2656 MBBI = mergeUpdateInsn(
MBBI, Update,
true);
2663 int UnscaledOffset =
2671 Update = findMatchingUpdateInsnForward(
MBBI, UnscaledOffset,
UpdateLimit);
2674 MBBI = mergeUpdateInsn(
MBBI, Update,
true);
2688 if (
TII->hasUnscaledLdStOffset(
MI.getOpcode()))
2700 if (Update != E && (
Offset & (Scale - 1)) == 0) {
2710 bool EnableNarrowZeroStOpt) {
2741 if (EnableNarrowZeroStOpt)
2758 DefinedInBB.
clear();
2759 DefinedInBB.addLiveIns(
MBB);
2767 if (
TII->isPairableLdStInst(*
MBBI) && tryToPairLdStInst(
MBBI))
2813 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
2818 ModifiedRegUnits.init(*
TRI);
2819 UsedRegUnits.init(*
TRI);
2820 DefinedInBB.init(*
TRI);
2823 bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign();
2824 for (
auto &
MBB : Fn) {
2845 return new AArch64LoadStoreOpt();
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
static cl::opt< bool > EnableRenaming("aarch64-load-store-renaming", cl::init(true), cl::Hidden)
static MachineOperand & getLdStRegOp(MachineInstr &MI, unsigned PairedRegOp=0)
static bool isPromotableLoadFromStore(MachineInstr &MI)
static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale, int &MinOffset, int &MaxOffset)
static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride)
static unsigned getMatchingPairOpcode(unsigned Opc)
static bool isMergeableLdStUpdate(MachineInstr &MI)
static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI, LdStPairFlags &Flags, const AArch64InstrInfo *TII)
static std::optional< MCPhysReg > tryToFindRegisterToRename(const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static bool needsWinCFI(const MachineFunction *MF)
static bool canRenameUntilSecondLoad(MachineInstr &FirstLoad, MachineInstr &SecondLoad, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static std::optional< MCPhysReg > findRenameRegForSameLdStRegPair(std::optional< bool > MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static bool mayAlias(MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
static cl::opt< unsigned > LdStLimit("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden)
static bool canRenameMOP(const MachineOperand &MOP, const TargetRegisterInfo *TRI)
static unsigned getPreIndexedOpcode(unsigned Opc)
#define AARCH64_LOAD_STORE_OPT_NAME
static cl::opt< unsigned > UpdateLimit("aarch64-update-scan-limit", cl::init(100), cl::Hidden)
static bool isPromotableZeroStoreInst(MachineInstr &MI)
static unsigned getMatchingWideOpcode(unsigned Opc)
static unsigned getMatchingNonSExtOpcode(unsigned Opc, bool *IsValidLdStrOpc=nullptr)
static MachineBasicBlock::iterator maybeMoveCFI(MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI)
static int alignTo(int Num, int PowOf2)
static bool isTagStore(const MachineInstr &MI)
static unsigned isMatchingStore(MachineInstr &LoadInst, MachineInstr &StoreInst)
static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg, const TargetRegisterInfo *TRI, unsigned Limit, std::function< bool(MachineInstr &, bool)> &Fn)
static bool isRewritableImplicitDef(unsigned Opc)
static unsigned getPostIndexedOpcode(unsigned Opc)
static cl::opt< unsigned > LdStConstLimit("aarch64-load-store-const-scan-limit", cl::init(10), cl::Hidden)
static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst, MachineInstr &StoreInst, const AArch64InstrInfo *TII)
static bool isPreLdStPairCandidate(MachineInstr &FirstMI, MachineInstr &MI)
static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale)
static void updateDefinedRegisters(MachineInstr &MI, LiveRegUnits &Units, const TargetRegisterInfo *TRI)
static bool canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static unsigned getBaseAddressOpcode(unsigned Opc)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI)
static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, DomTreeUpdater *DTU)
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
static const MachineOperand & getLdStAmountOp(const MachineInstr &MI)
Returns the shift amount operator of a load/store.
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
const AArch64RegisterInfo * getRegisterInfo() const override
const AArch64InstrInfo * getInstrInfo() const override
const AArch64TargetLowering * getTargetLowering() const override
unsigned getRedZoneSize(const Function &F) const
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
static bool shouldExecute(unsigned CounterName)
FunctionPass class - This class is used to implement most global optimizations.
bool needsUnwindTableEntry() const
True if this function needs an unwind table.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
A set of register units used to track register liveness.
static void accumulateUsedDefed(const MachineInstr &MI, LiveRegUnits &ModifiedRegUnits, LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI)
For a machine instruction MI, adds all register units used in UsedRegUnits and defined or clobbered i...
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void init(const TargetRegisterInfo &TRI)
Initialize and clear the set.
void addReg(MCPhysReg Reg)
Adds register units covered by physical register Reg.
void removeReg(MCPhysReg Reg)
Removes all register units covered by physical register Reg.
void accumulate(const MachineInstr &MI)
Adds all register units used, defined or clobbered in MI.
An instruction for reading from memory.
bool usesWindowsCFI() const
OpType getOperation() const
Wrapper class representing physical registers. Should be passed by value.
reverse_instr_iterator instr_rend()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const std::vector< MCCFIInstruction > & getFrameInstructions() const
Returns a reference to a list of cfi instructions in the function's prologue.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & cloneMergedMemRefs(ArrayRef< const MachineInstr * > OtherMIs) const
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const
Returns true if this instruction's memory access aliases the memory access of Other.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
iterator_range< mop_iterator > operands()
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
bool isPseudo(QueryType Type=IgnoreBundle) const
Return true if this is a pseudo instruction that doesn't correspond to a real machine instruction.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
void setImplicit(bool Val=true)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
void setIsKill(bool Val=true)
bool isRenamable() const
isRenamable - Returns true if this register may be renamed, i.e.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
bool isEarlyClobber() const
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
const MCAsmInfo * getMCAsmInfo() const
Return target specific asm information.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
LLVM Value Representation.
self_iterator getIterator()
A range adaptor for a pair of iterators.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
@ Define
Register definition.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< filter_iterator< ConstMIBundleOperands, bool(*)(const MachineOperand &)> > phys_regs_and_masks(const MachineInstr &MI)
Returns an iterator range over all physical register and mask operands for MI and bundled instruction...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createAArch64LoadStoreOptimizationPass()
createAArch64LoadStoreOptimizationPass - returns an instance of the load / store optimization pass.
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
void initializeAArch64LoadStoreOptPass(PassRegistry &)
IterT prev_nodbg(IterT It, IterT Begin, bool SkipPseudoOp=true)
Decrement It, then continue decrementing it while it points to a debug instruction.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.