LLVM 17.0.0git
AArch64InstrInfo.cpp
Go to the documentation of this file.
1//===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
15#include "AArch64Subtarget.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/STLExtras.h"
35#include "llvm/IR/DebugLoc.h"
36#include "llvm/IR/GlobalValue.h"
37#include "llvm/MC/MCAsmInfo.h"
38#include "llvm/MC/MCInst.h"
40#include "llvm/MC/MCInstrDesc.h"
46#include "llvm/Support/LEB128.h"
50#include <cassert>
51#include <cstdint>
52#include <iterator>
53#include <utility>
54
55using namespace llvm;
56
57#define GET_INSTRINFO_CTOR_DTOR
58#include "AArch64GenInstrInfo.inc"
59
61 "aarch64-tbz-offset-bits", cl::Hidden, cl::init(14),
62 cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
63
65 "aarch64-cbz-offset-bits", cl::Hidden, cl::init(19),
66 cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
67
69 BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19),
70 cl::desc("Restrict range of Bcc instructions (DEBUG)"));
71
73 : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP,
74 AArch64::CATCHRET),
75 RI(STI.getTargetTriple()), Subtarget(STI) {}
76
77/// GetInstSize - Return the number of bytes of code the specified
78/// instruction may be. This returns the maximum number of bytes.
80 const MachineBasicBlock &MBB = *MI.getParent();
81 const MachineFunction *MF = MBB.getParent();
82 const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo();
83
84 {
85 auto Op = MI.getOpcode();
86 if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
87 return getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI);
88 }
89
90 // Meta-instructions emit no code.
91 if (MI.isMetaInstruction())
92 return 0;
93
94 // FIXME: We currently only handle pseudoinstructions that don't get expanded
95 // before the assembly printer.
96 unsigned NumBytes = 0;
97 const MCInstrDesc &Desc = MI.getDesc();
98
99 // Size should be preferably set in
100 // llvm/lib/Target/AArch64/AArch64InstrInfo.td (default case).
101 // Specific cases handle instructions of variable sizes
102 switch (Desc.getOpcode()) {
103 default:
104 if (Desc.getSize())
105 return Desc.getSize();
106
107 // Anything not explicitly designated otherwise (i.e. pseudo-instructions
108 // with fixed constant size but not specified in .td file) is a normal
109 // 4-byte insn.
110 NumBytes = 4;
111 break;
112 case TargetOpcode::STACKMAP:
113 // The upper bound for a stackmap intrinsic is the full length of its shadow
114 NumBytes = StackMapOpers(&MI).getNumPatchBytes();
115 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
116 break;
117 case TargetOpcode::PATCHPOINT:
118 // The size of the patchpoint intrinsic is the number of bytes requested
119 NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
120 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
121 break;
122 case TargetOpcode::STATEPOINT:
123 NumBytes = StatepointOpers(&MI).getNumPatchBytes();
124 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
125 // No patch bytes means a normal call inst is emitted
126 if (NumBytes == 0)
127 NumBytes = 4;
128 break;
129 case AArch64::SPACE:
130 NumBytes = MI.getOperand(1).getImm();
131 break;
132 case TargetOpcode::BUNDLE:
133 NumBytes = getInstBundleLength(MI);
134 break;
135 }
136
137 return NumBytes;
138}
139
140unsigned AArch64InstrInfo::getInstBundleLength(const MachineInstr &MI) const {
141 unsigned Size = 0;
143 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
144 while (++I != E && I->isInsideBundle()) {
145 assert(!I->isBundle() && "No nested bundle!");
147 }
148 return Size;
149}
150
153 // Block ends with fall-through condbranch.
154 switch (LastInst->getOpcode()) {
155 default:
156 llvm_unreachable("Unknown branch instruction?");
157 case AArch64::Bcc:
158 Target = LastInst->getOperand(1).getMBB();
159 Cond.push_back(LastInst->getOperand(0));
160 break;
161 case AArch64::CBZW:
162 case AArch64::CBZX:
163 case AArch64::CBNZW:
164 case AArch64::CBNZX:
165 Target = LastInst->getOperand(1).getMBB();
166 Cond.push_back(MachineOperand::CreateImm(-1));
167 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
168 Cond.push_back(LastInst->getOperand(0));
169 break;
170 case AArch64::TBZW:
171 case AArch64::TBZX:
172 case AArch64::TBNZW:
173 case AArch64::TBNZX:
174 Target = LastInst->getOperand(2).getMBB();
175 Cond.push_back(MachineOperand::CreateImm(-1));
176 Cond.push_back(MachineOperand::CreateImm(LastInst->getOpcode()));
177 Cond.push_back(LastInst->getOperand(0));
178 Cond.push_back(LastInst->getOperand(1));
179 }
180}
181
182static unsigned getBranchDisplacementBits(unsigned Opc) {
183 switch (Opc) {
184 default:
185 llvm_unreachable("unexpected opcode!");
186 case AArch64::B:
187 return 64;
188 case AArch64::TBNZW:
189 case AArch64::TBZW:
190 case AArch64::TBNZX:
191 case AArch64::TBZX:
192 return TBZDisplacementBits;
193 case AArch64::CBNZW:
194 case AArch64::CBZW:
195 case AArch64::CBNZX:
196 case AArch64::CBZX:
197 return CBZDisplacementBits;
198 case AArch64::Bcc:
199 return BCCDisplacementBits;
200 }
201}
202
204 int64_t BrOffset) const {
205 unsigned Bits = getBranchDisplacementBits(BranchOp);
206 assert(Bits >= 3 && "max branch displacement must be enough to jump"
207 "over conditional branch expansion");
208 return isIntN(Bits, BrOffset / 4);
209}
210
213 switch (MI.getOpcode()) {
214 default:
215 llvm_unreachable("unexpected opcode!");
216 case AArch64::B:
217 return MI.getOperand(0).getMBB();
218 case AArch64::TBZW:
219 case AArch64::TBNZW:
220 case AArch64::TBZX:
221 case AArch64::TBNZX:
222 return MI.getOperand(2).getMBB();
223 case AArch64::CBZW:
224 case AArch64::CBNZW:
225 case AArch64::CBZX:
226 case AArch64::CBNZX:
227 case AArch64::Bcc:
228 return MI.getOperand(1).getMBB();
229 }
230}
231
232// Branch analysis.
235 MachineBasicBlock *&FBB,
237 bool AllowModify) const {
238 // If the block has no terminators, it just falls into the block after it.
240 if (I == MBB.end())
241 return false;
242
243 // Skip over SpeculationBarrierEndBB terminators
244 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
245 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
246 --I;
247 }
248
249 if (!isUnpredicatedTerminator(*I))
250 return false;
251
252 // Get the last instruction in the block.
253 MachineInstr *LastInst = &*I;
254
255 // If there is only one terminator instruction, process it.
256 unsigned LastOpc = LastInst->getOpcode();
257 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
258 if (isUncondBranchOpcode(LastOpc)) {
259 TBB = LastInst->getOperand(0).getMBB();
260 return false;
261 }
262 if (isCondBranchOpcode(LastOpc)) {
263 // Block ends with fall-through condbranch.
264 parseCondBranch(LastInst, TBB, Cond);
265 return false;
266 }
267 return true; // Can't handle indirect branch.
268 }
269
270 // Get the instruction before it if it is a terminator.
271 MachineInstr *SecondLastInst = &*I;
272 unsigned SecondLastOpc = SecondLastInst->getOpcode();
273
274 // If AllowModify is true and the block ends with two or more unconditional
275 // branches, delete all but the first unconditional branch.
276 if (AllowModify && isUncondBranchOpcode(LastOpc)) {
277 while (isUncondBranchOpcode(SecondLastOpc)) {
278 LastInst->eraseFromParent();
279 LastInst = SecondLastInst;
280 LastOpc = LastInst->getOpcode();
281 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
282 // Return now the only terminator is an unconditional branch.
283 TBB = LastInst->getOperand(0).getMBB();
284 return false;
285 } else {
286 SecondLastInst = &*I;
287 SecondLastOpc = SecondLastInst->getOpcode();
288 }
289 }
290 }
291
292 // If we're allowed to modify and the block ends in a unconditional branch
293 // which could simply fallthrough, remove the branch. (Note: This case only
294 // matters when we can't understand the whole sequence, otherwise it's also
295 // handled by BranchFolding.cpp.)
296 if (AllowModify && isUncondBranchOpcode(LastOpc) &&
298 LastInst->eraseFromParent();
299 LastInst = SecondLastInst;
300 LastOpc = LastInst->getOpcode();
301 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
302 assert(!isUncondBranchOpcode(LastOpc) &&
303 "unreachable unconditional branches removed above");
304
305 if (isCondBranchOpcode(LastOpc)) {
306 // Block ends with fall-through condbranch.
307 parseCondBranch(LastInst, TBB, Cond);
308 return false;
309 }
310 return true; // Can't handle indirect branch.
311 } else {
312 SecondLastInst = &*I;
313 SecondLastOpc = SecondLastInst->getOpcode();
314 }
315 }
316
317 // If there are three terminators, we don't know what sort of block this is.
318 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(*--I))
319 return true;
320
321 // If the block ends with a B and a Bcc, handle it.
322 if (isCondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
323 parseCondBranch(SecondLastInst, TBB, Cond);
324 FBB = LastInst->getOperand(0).getMBB();
325 return false;
326 }
327
328 // If the block ends with two unconditional branches, handle it. The second
329 // one is not executed, so remove it.
330 if (isUncondBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
331 TBB = SecondLastInst->getOperand(0).getMBB();
332 I = LastInst;
333 if (AllowModify)
334 I->eraseFromParent();
335 return false;
336 }
337
338 // ...likewise if it ends with an indirect branch followed by an unconditional
339 // branch.
340 if (isIndirectBranchOpcode(SecondLastOpc) && isUncondBranchOpcode(LastOpc)) {
341 I = LastInst;
342 if (AllowModify)
343 I->eraseFromParent();
344 return true;
345 }
346
347 // Otherwise, can't handle this.
348 return true;
349}
350
352 MachineBranchPredicate &MBP,
353 bool AllowModify) const {
354 // For the moment, handle only a block which ends with a cb(n)zx followed by
355 // a fallthrough. Why this? Because it is a common form.
356 // TODO: Should we handle b.cc?
357
359 if (I == MBB.end())
360 return true;
361
362 // Skip over SpeculationBarrierEndBB terminators
363 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
364 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
365 --I;
366 }
367
368 if (!isUnpredicatedTerminator(*I))
369 return true;
370
371 // Get the last instruction in the block.
372 MachineInstr *LastInst = &*I;
373 unsigned LastOpc = LastInst->getOpcode();
374 if (!isCondBranchOpcode(LastOpc))
375 return true;
376
377 switch (LastOpc) {
378 default:
379 return true;
380 case AArch64::CBZW:
381 case AArch64::CBZX:
382 case AArch64::CBNZW:
383 case AArch64::CBNZX:
384 break;
385 };
386
387 MBP.TrueDest = LastInst->getOperand(1).getMBB();
388 assert(MBP.TrueDest && "expected!");
389 MBP.FalseDest = MBB.getNextNode();
390
391 MBP.ConditionDef = nullptr;
392 MBP.SingleUseCondition = false;
393
394 MBP.LHS = LastInst->getOperand(0);
395 MBP.RHS = MachineOperand::CreateImm(0);
396 MBP.Predicate = LastOpc == AArch64::CBNZX ? MachineBranchPredicate::PRED_NE
397 : MachineBranchPredicate::PRED_EQ;
398 return false;
399}
400
403 if (Cond[0].getImm() != -1) {
404 // Regular Bcc
407 } else {
408 // Folded compare-and-branch
409 switch (Cond[1].getImm()) {
410 default:
411 llvm_unreachable("Unknown conditional branch!");
412 case AArch64::CBZW:
413 Cond[1].setImm(AArch64::CBNZW);
414 break;
415 case AArch64::CBNZW:
416 Cond[1].setImm(AArch64::CBZW);
417 break;
418 case AArch64::CBZX:
419 Cond[1].setImm(AArch64::CBNZX);
420 break;
421 case AArch64::CBNZX:
422 Cond[1].setImm(AArch64::CBZX);
423 break;
424 case AArch64::TBZW:
425 Cond[1].setImm(AArch64::TBNZW);
426 break;
427 case AArch64::TBNZW:
428 Cond[1].setImm(AArch64::TBZW);
429 break;
430 case AArch64::TBZX:
431 Cond[1].setImm(AArch64::TBNZX);
432 break;
433 case AArch64::TBNZX:
434 Cond[1].setImm(AArch64::TBZX);
435 break;
436 }
437 }
438
439 return false;
440}
441
443 int *BytesRemoved) const {
445 if (I == MBB.end())
446 return 0;
447
448 if (!isUncondBranchOpcode(I->getOpcode()) &&
449 !isCondBranchOpcode(I->getOpcode()))
450 return 0;
451
452 // Remove the branch.
453 I->eraseFromParent();
454
455 I = MBB.end();
456
457 if (I == MBB.begin()) {
458 if (BytesRemoved)
459 *BytesRemoved = 4;
460 return 1;
461 }
462 --I;
463 if (!isCondBranchOpcode(I->getOpcode())) {
464 if (BytesRemoved)
465 *BytesRemoved = 4;
466 return 1;
467 }
468
469 // Remove the branch.
470 I->eraseFromParent();
471 if (BytesRemoved)
472 *BytesRemoved = 8;
473
474 return 2;
475}
476
477void AArch64InstrInfo::instantiateCondBranch(
480 if (Cond[0].getImm() != -1) {
481 // Regular Bcc
482 BuildMI(&MBB, DL, get(AArch64::Bcc)).addImm(Cond[0].getImm()).addMBB(TBB);
483 } else {
484 // Folded compare-and-branch
485 // Note that we use addOperand instead of addReg to keep the flags.
486 const MachineInstrBuilder MIB =
487 BuildMI(&MBB, DL, get(Cond[1].getImm())).add(Cond[2]);
488 if (Cond.size() > 3)
489 MIB.addImm(Cond[3].getImm());
490 MIB.addMBB(TBB);
491 }
492}
493
496 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
497 // Shouldn't be a fall through.
498 assert(TBB && "insertBranch must not be told to insert a fallthrough");
499
500 if (!FBB) {
501 if (Cond.empty()) // Unconditional branch?
502 BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB);
503 else
504 instantiateCondBranch(MBB, DL, TBB, Cond);
505
506 if (BytesAdded)
507 *BytesAdded = 4;
508
509 return 1;
510 }
511
512 // Two-way conditional branch.
513 instantiateCondBranch(MBB, DL, TBB, Cond);
514 BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB);
515
516 if (BytesAdded)
517 *BytesAdded = 8;
518
519 return 2;
520}
521
522// Find the original register that VReg is copied from.
523static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
524 while (Register::isVirtualRegister(VReg)) {
525 const MachineInstr *DefMI = MRI.getVRegDef(VReg);
526 if (!DefMI->isFullCopy())
527 return VReg;
528 VReg = DefMI->getOperand(1).getReg();
529 }
530 return VReg;
531}
532
533// Determine if VReg is defined by an instruction that can be folded into a
534// csel instruction. If so, return the folded opcode, and the replacement
535// register.
536static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
537 unsigned *NewVReg = nullptr) {
538 VReg = removeCopies(MRI, VReg);
540 return 0;
541
542 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(MRI.getRegClass(VReg));
543 const MachineInstr *DefMI = MRI.getVRegDef(VReg);
544 unsigned Opc = 0;
545 unsigned SrcOpNum = 0;
546 switch (DefMI->getOpcode()) {
547 case AArch64::ADDSXri:
548 case AArch64::ADDSWri:
549 // if NZCV is used, do not fold.
550 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
551 return 0;
552 // fall-through to ADDXri and ADDWri.
553 [[fallthrough]];
554 case AArch64::ADDXri:
555 case AArch64::ADDWri:
556 // add x, 1 -> csinc.
557 if (!DefMI->getOperand(2).isImm() || DefMI->getOperand(2).getImm() != 1 ||
558 DefMI->getOperand(3).getImm() != 0)
559 return 0;
560 SrcOpNum = 1;
561 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
562 break;
563
564 case AArch64::ORNXrr:
565 case AArch64::ORNWrr: {
566 // not x -> csinv, represented as orn dst, xzr, src.
567 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
568 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
569 return 0;
570 SrcOpNum = 2;
571 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
572 break;
573 }
574
575 case AArch64::SUBSXrr:
576 case AArch64::SUBSWrr:
577 // if NZCV is used, do not fold.
578 if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
579 return 0;
580 // fall-through to SUBXrr and SUBWrr.
581 [[fallthrough]];
582 case AArch64::SUBXrr:
583 case AArch64::SUBWrr: {
584 // neg x -> csneg, represented as sub dst, xzr, src.
585 unsigned ZReg = removeCopies(MRI, DefMI->getOperand(1).getReg());
586 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
587 return 0;
588 SrcOpNum = 2;
589 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
590 break;
591 }
592 default:
593 return 0;
594 }
595 assert(Opc && SrcOpNum && "Missing parameters");
596
597 if (NewVReg)
598 *NewVReg = DefMI->getOperand(SrcOpNum).getReg();
599 return Opc;
600}
601
604 Register DstReg, Register TrueReg,
605 Register FalseReg, int &CondCycles,
606 int &TrueCycles,
607 int &FalseCycles) const {
608 // Check register classes.
610 const TargetRegisterClass *RC =
611 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
612 if (!RC)
613 return false;
614
615 // Also need to check the dest regclass, in case we're trying to optimize
616 // something like:
617 // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
618 if (!RI.getCommonSubClass(RC, MRI.getRegClass(DstReg)))
619 return false;
620
621 // Expanding cbz/tbz requires an extra cycle of latency on the condition.
622 unsigned ExtraCondLat = Cond.size() != 1;
623
624 // GPRs are handled by csel.
625 // FIXME: Fold in x+1, -x, and ~x when applicable.
626 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
627 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
628 // Single-cycle csel, csinc, csinv, and csneg.
629 CondCycles = 1 + ExtraCondLat;
630 TrueCycles = FalseCycles = 1;
631 if (canFoldIntoCSel(MRI, TrueReg))
632 TrueCycles = 0;
633 else if (canFoldIntoCSel(MRI, FalseReg))
634 FalseCycles = 0;
635 return true;
636 }
637
638 // Scalar floating point is handled by fcsel.
639 // FIXME: Form fabs, fmin, and fmax when applicable.
640 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
641 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
642 CondCycles = 5 + ExtraCondLat;
643 TrueCycles = FalseCycles = 2;
644 return true;
645 }
646
647 // Can't do vectors.
648 return false;
649}
650
653 const DebugLoc &DL, Register DstReg,
655 Register TrueReg, Register FalseReg) const {
657
658 // Parse the condition code, see parseCondBranch() above.
660 switch (Cond.size()) {
661 default:
662 llvm_unreachable("Unknown condition opcode in Cond");
663 case 1: // b.cc
664 CC = AArch64CC::CondCode(Cond[0].getImm());
665 break;
666 case 3: { // cbz/cbnz
667 // We must insert a compare against 0.
668 bool Is64Bit;
669 switch (Cond[1].getImm()) {
670 default:
671 llvm_unreachable("Unknown branch opcode in Cond");
672 case AArch64::CBZW:
673 Is64Bit = false;
675 break;
676 case AArch64::CBZX:
677 Is64Bit = true;
679 break;
680 case AArch64::CBNZW:
681 Is64Bit = false;
683 break;
684 case AArch64::CBNZX:
685 Is64Bit = true;
687 break;
688 }
689 Register SrcReg = Cond[2].getReg();
690 if (Is64Bit) {
691 // cmp reg, #0 is actually subs xzr, reg, #0.
692 MRI.constrainRegClass(SrcReg, &AArch64::GPR64spRegClass);
693 BuildMI(MBB, I, DL, get(AArch64::SUBSXri), AArch64::XZR)
694 .addReg(SrcReg)
695 .addImm(0)
696 .addImm(0);
697 } else {
698 MRI.constrainRegClass(SrcReg, &AArch64::GPR32spRegClass);
699 BuildMI(MBB, I, DL, get(AArch64::SUBSWri), AArch64::WZR)
700 .addReg(SrcReg)
701 .addImm(0)
702 .addImm(0);
703 }
704 break;
705 }
706 case 4: { // tbz/tbnz
707 // We must insert a tst instruction.
708 switch (Cond[1].getImm()) {
709 default:
710 llvm_unreachable("Unknown branch opcode in Cond");
711 case AArch64::TBZW:
712 case AArch64::TBZX:
714 break;
715 case AArch64::TBNZW:
716 case AArch64::TBNZX:
718 break;
719 }
720 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
721 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
722 BuildMI(MBB, I, DL, get(AArch64::ANDSWri), AArch64::WZR)
723 .addReg(Cond[2].getReg())
724 .addImm(
725 AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 32));
726 else
727 BuildMI(MBB, I, DL, get(AArch64::ANDSXri), AArch64::XZR)
728 .addReg(Cond[2].getReg())
729 .addImm(
730 AArch64_AM::encodeLogicalImmediate(1ull << Cond[3].getImm(), 64));
731 break;
732 }
733 }
734
735 unsigned Opc = 0;
736 const TargetRegisterClass *RC = nullptr;
737 bool TryFold = false;
738 if (MRI.constrainRegClass(DstReg, &AArch64::GPR64RegClass)) {
739 RC = &AArch64::GPR64RegClass;
740 Opc = AArch64::CSELXr;
741 TryFold = true;
742 } else if (MRI.constrainRegClass(DstReg, &AArch64::GPR32RegClass)) {
743 RC = &AArch64::GPR32RegClass;
744 Opc = AArch64::CSELWr;
745 TryFold = true;
746 } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR64RegClass)) {
747 RC = &AArch64::FPR64RegClass;
748 Opc = AArch64::FCSELDrrr;
749 } else if (MRI.constrainRegClass(DstReg, &AArch64::FPR32RegClass)) {
750 RC = &AArch64::FPR32RegClass;
751 Opc = AArch64::FCSELSrrr;
752 }
753 assert(RC && "Unsupported regclass");
754
755 // Try folding simple instructions into the csel.
756 if (TryFold) {
757 unsigned NewVReg = 0;
758 unsigned FoldedOpc = canFoldIntoCSel(MRI, TrueReg, &NewVReg);
759 if (FoldedOpc) {
760 // The folded opcodes csinc, csinc and csneg apply the operation to
761 // FalseReg, so we need to invert the condition.
763 TrueReg = FalseReg;
764 } else
765 FoldedOpc = canFoldIntoCSel(MRI, FalseReg, &NewVReg);
766
767 // Fold the operation. Leave any dead instructions for DCE to clean up.
768 if (FoldedOpc) {
769 FalseReg = NewVReg;
770 Opc = FoldedOpc;
771 // The extends the live range of NewVReg.
772 MRI.clearKillFlags(NewVReg);
773 }
774 }
775
776 // Pull all virtual register into the appropriate class.
777 MRI.constrainRegClass(TrueReg, RC);
778 MRI.constrainRegClass(FalseReg, RC);
779
780 // Insert the csel.
781 BuildMI(MBB, I, DL, get(Opc), DstReg)
782 .addReg(TrueReg)
783 .addReg(FalseReg)
784 .addImm(CC);
785}
786
787/// Returns true if a MOVi32imm or MOVi64imm can be expanded to an ORRxx.
788static bool canBeExpandedToORR(const MachineInstr &MI, unsigned BitSize) {
789 uint64_t Imm = MI.getOperand(1).getImm();
790 uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize);
791 uint64_t Encoding;
792 return AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding);
793}
794
795// FIXME: this implementation should be micro-architecture dependent, so a
796// micro-architecture target hook should be introduced here in future.
798 if (!Subtarget.hasCustomCheapAsMoveHandling())
799 return MI.isAsCheapAsAMove();
800
801 const unsigned Opcode = MI.getOpcode();
802
803 // Firstly, check cases gated by features.
804
805 if (Subtarget.hasZeroCycleZeroingFP()) {
806 if (Opcode == AArch64::FMOVH0 ||
807 Opcode == AArch64::FMOVS0 ||
808 Opcode == AArch64::FMOVD0)
809 return true;
810 }
811
812 if (Subtarget.hasZeroCycleZeroingGP()) {
813 if (Opcode == TargetOpcode::COPY &&
814 (MI.getOperand(1).getReg() == AArch64::WZR ||
815 MI.getOperand(1).getReg() == AArch64::XZR))
816 return true;
817 }
818
819 // Secondly, check cases specific to sub-targets.
820
821 if (Subtarget.hasExynosCheapAsMoveHandling()) {
822 if (isExynosCheapAsMove(MI))
823 return true;
824
825 return MI.isAsCheapAsAMove();
826 }
827
828 // Finally, check generic cases.
829
830 switch (Opcode) {
831 default:
832 return false;
833
834 // add/sub on register without shift
835 case AArch64::ADDWri:
836 case AArch64::ADDXri:
837 case AArch64::SUBWri:
838 case AArch64::SUBXri:
839 return (MI.getOperand(3).getImm() == 0);
840
841 // logical ops on immediate
842 case AArch64::ANDWri:
843 case AArch64::ANDXri:
844 case AArch64::EORWri:
845 case AArch64::EORXri:
846 case AArch64::ORRWri:
847 case AArch64::ORRXri:
848 return true;
849
850 // logical ops on register without shift
851 case AArch64::ANDWrr:
852 case AArch64::ANDXrr:
853 case AArch64::BICWrr:
854 case AArch64::BICXrr:
855 case AArch64::EONWrr:
856 case AArch64::EONXrr:
857 case AArch64::EORWrr:
858 case AArch64::EORXrr:
859 case AArch64::ORNWrr:
860 case AArch64::ORNXrr:
861 case AArch64::ORRWrr:
862 case AArch64::ORRXrr:
863 return true;
864
865 // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
866 // ORRXri, it is as cheap as MOV
867 case AArch64::MOVi32imm:
868 return canBeExpandedToORR(MI, 32);
869 case AArch64::MOVi64imm:
870 return canBeExpandedToORR(MI, 64);
871 }
872
873 llvm_unreachable("Unknown opcode to check as cheap as a move!");
874}
875
877 switch (MI.getOpcode()) {
878 default:
879 return false;
880
881 case AArch64::ADDWrs:
882 case AArch64::ADDXrs:
883 case AArch64::ADDSWrs:
884 case AArch64::ADDSXrs: {
885 unsigned Imm = MI.getOperand(3).getImm();
886 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
887 if (ShiftVal == 0)
888 return true;
889 return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
890 }
891
892 case AArch64::ADDWrx:
893 case AArch64::ADDXrx:
894 case AArch64::ADDXrx64:
895 case AArch64::ADDSWrx:
896 case AArch64::ADDSXrx:
897 case AArch64::ADDSXrx64: {
898 unsigned Imm = MI.getOperand(3).getImm();
899 switch (AArch64_AM::getArithExtendType(Imm)) {
900 default:
901 return false;
902 case AArch64_AM::UXTB:
903 case AArch64_AM::UXTH:
904 case AArch64_AM::UXTW:
905 case AArch64_AM::UXTX:
906 return AArch64_AM::getArithShiftValue(Imm) <= 4;
907 }
908 }
909
910 case AArch64::SUBWrs:
911 case AArch64::SUBSWrs: {
912 unsigned Imm = MI.getOperand(3).getImm();
913 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
914 return ShiftVal == 0 ||
915 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
916 }
917
918 case AArch64::SUBXrs:
919 case AArch64::SUBSXrs: {
920 unsigned Imm = MI.getOperand(3).getImm();
921 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
922 return ShiftVal == 0 ||
923 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
924 }
925
926 case AArch64::SUBWrx:
927 case AArch64::SUBXrx:
928 case AArch64::SUBXrx64:
929 case AArch64::SUBSWrx:
930 case AArch64::SUBSXrx:
931 case AArch64::SUBSXrx64: {
932 unsigned Imm = MI.getOperand(3).getImm();
933 switch (AArch64_AM::getArithExtendType(Imm)) {
934 default:
935 return false;
936 case AArch64_AM::UXTB:
937 case AArch64_AM::UXTH:
938 case AArch64_AM::UXTW:
939 case AArch64_AM::UXTX:
940 return AArch64_AM::getArithShiftValue(Imm) == 0;
941 }
942 }
943
944 case AArch64::LDRBBroW:
945 case AArch64::LDRBBroX:
946 case AArch64::LDRBroW:
947 case AArch64::LDRBroX:
948 case AArch64::LDRDroW:
949 case AArch64::LDRDroX:
950 case AArch64::LDRHHroW:
951 case AArch64::LDRHHroX:
952 case AArch64::LDRHroW:
953 case AArch64::LDRHroX:
954 case AArch64::LDRQroW:
955 case AArch64::LDRQroX:
956 case AArch64::LDRSBWroW:
957 case AArch64::LDRSBWroX:
958 case AArch64::LDRSBXroW:
959 case AArch64::LDRSBXroX:
960 case AArch64::LDRSHWroW:
961 case AArch64::LDRSHWroX:
962 case AArch64::LDRSHXroW:
963 case AArch64::LDRSHXroX:
964 case AArch64::LDRSWroW:
965 case AArch64::LDRSWroX:
966 case AArch64::LDRSroW:
967 case AArch64::LDRSroX:
968 case AArch64::LDRWroW:
969 case AArch64::LDRWroX:
970 case AArch64::LDRXroW:
971 case AArch64::LDRXroX:
972 case AArch64::PRFMroW:
973 case AArch64::PRFMroX:
974 case AArch64::STRBBroW:
975 case AArch64::STRBBroX:
976 case AArch64::STRBroW:
977 case AArch64::STRBroX:
978 case AArch64::STRDroW:
979 case AArch64::STRDroX:
980 case AArch64::STRHHroW:
981 case AArch64::STRHHroX:
982 case AArch64::STRHroW:
983 case AArch64::STRHroX:
984 case AArch64::STRQroW:
985 case AArch64::STRQroX:
986 case AArch64::STRSroW:
987 case AArch64::STRSroX:
988 case AArch64::STRWroW:
989 case AArch64::STRWroX:
990 case AArch64::STRXroW:
991 case AArch64::STRXroX: {
992 unsigned IsSigned = MI.getOperand(3).getImm();
993 return !IsSigned;
994 }
995 }
996}
997
999 unsigned Opc = MI.getOpcode();
1000 switch (Opc) {
1001 default:
1002 return false;
1003 case AArch64::SEH_StackAlloc:
1004 case AArch64::SEH_SaveFPLR:
1005 case AArch64::SEH_SaveFPLR_X:
1006 case AArch64::SEH_SaveReg:
1007 case AArch64::SEH_SaveReg_X:
1008 case AArch64::SEH_SaveRegP:
1009 case AArch64::SEH_SaveRegP_X:
1010 case AArch64::SEH_SaveFReg:
1011 case AArch64::SEH_SaveFReg_X:
1012 case AArch64::SEH_SaveFRegP:
1013 case AArch64::SEH_SaveFRegP_X:
1014 case AArch64::SEH_SetFP:
1015 case AArch64::SEH_AddFP:
1016 case AArch64::SEH_Nop:
1017 case AArch64::SEH_PrologEnd:
1018 case AArch64::SEH_EpilogStart:
1019 case AArch64::SEH_EpilogEnd:
1020 case AArch64::SEH_PACSignLR:
1021 return true;
1022 }
1023}
1024
1026 Register &SrcReg, Register &DstReg,
1027 unsigned &SubIdx) const {
1028 switch (MI.getOpcode()) {
1029 default:
1030 return false;
1031 case AArch64::SBFMXri: // aka sxtw
1032 case AArch64::UBFMXri: // aka uxtw
1033 // Check for the 32 -> 64 bit extension case, these instructions can do
1034 // much more.
1035 if (MI.getOperand(2).getImm() != 0 || MI.getOperand(3).getImm() != 31)
1036 return false;
1037 // This is a signed or unsigned 32 -> 64 bit extension.
1038 SrcReg = MI.getOperand(1).getReg();
1039 DstReg = MI.getOperand(0).getReg();
1040 SubIdx = AArch64::sub_32;
1041 return true;
1042 }
1043}
1044
1046 const MachineInstr &MIa, const MachineInstr &MIb) const {
1048 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1049 int64_t OffsetA = 0, OffsetB = 0;
1050 unsigned WidthA = 0, WidthB = 0;
1051 bool OffsetAIsScalable = false, OffsetBIsScalable = false;
1052
1053 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1054 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1055
1058 return false;
1059
1060 // Retrieve the base, offset from the base and width. Width
1061 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If
1062 // base are identical, and the offset of a lower memory access +
1063 // the width doesn't overlap the offset of a higher memory access,
1064 // then the memory accesses are different.
1065 // If OffsetAIsScalable and OffsetBIsScalable are both true, they
1066 // are assumed to have the same scale (vscale).
1067 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, OffsetAIsScalable,
1068 WidthA, TRI) &&
1069 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, OffsetBIsScalable,
1070 WidthB, TRI)) {
1071 if (BaseOpA->isIdenticalTo(*BaseOpB) &&
1072 OffsetAIsScalable == OffsetBIsScalable) {
1073 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1074 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1075 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1076 if (LowOffset + LowWidth <= HighOffset)
1077 return true;
1078 }
1079 }
1080 return false;
1081}
1082
1084 const MachineBasicBlock *MBB,
1085 const MachineFunction &MF) const {
1087 return true;
1088 switch (MI.getOpcode()) {
1089 case AArch64::HINT:
1090 // CSDB hints are scheduling barriers.
1091 if (MI.getOperand(0).getImm() == 0x14)
1092 return true;
1093 break;
1094 case AArch64::DSB:
1095 case AArch64::ISB:
1096 // DSB and ISB also are scheduling barriers.
1097 return true;
1098 case AArch64::MSRpstatesvcrImm1:
1099 // SMSTART and SMSTOP are also scheduling barriers.
1100 return true;
1101 default:;
1102 }
1103 if (isSEHInstruction(MI))
1104 return true;
1105 auto Next = std::next(MI.getIterator());
1106 return Next != MBB->end() && Next->isCFIInstruction();
1107}
1108
1109/// analyzeCompare - For a comparison instruction, return the source registers
1110/// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1111/// Return true if the comparison instruction can be analyzed.
1113 Register &SrcReg2, int64_t &CmpMask,
1114 int64_t &CmpValue) const {
1115 // The first operand can be a frame index where we'd normally expect a
1116 // register.
1117 assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1118 if (!MI.getOperand(1).isReg())
1119 return false;
1120
1121 switch (MI.getOpcode()) {
1122 default:
1123 break;
1124 case AArch64::PTEST_PP:
1125 case AArch64::PTEST_PP_ANY:
1126 SrcReg = MI.getOperand(0).getReg();
1127 SrcReg2 = MI.getOperand(1).getReg();
1128 // Not sure about the mask and value for now...
1129 CmpMask = ~0;
1130 CmpValue = 0;
1131 return true;
1132 case AArch64::SUBSWrr:
1133 case AArch64::SUBSWrs:
1134 case AArch64::SUBSWrx:
1135 case AArch64::SUBSXrr:
1136 case AArch64::SUBSXrs:
1137 case AArch64::SUBSXrx:
1138 case AArch64::ADDSWrr:
1139 case AArch64::ADDSWrs:
1140 case AArch64::ADDSWrx:
1141 case AArch64::ADDSXrr:
1142 case AArch64::ADDSXrs:
1143 case AArch64::ADDSXrx:
1144 // Replace SUBSWrr with SUBWrr if NZCV is not used.
1145 SrcReg = MI.getOperand(1).getReg();
1146 SrcReg2 = MI.getOperand(2).getReg();
1147 CmpMask = ~0;
1148 CmpValue = 0;
1149 return true;
1150 case AArch64::SUBSWri:
1151 case AArch64::ADDSWri:
1152 case AArch64::SUBSXri:
1153 case AArch64::ADDSXri:
1154 SrcReg = MI.getOperand(1).getReg();
1155 SrcReg2 = 0;
1156 CmpMask = ~0;
1157 CmpValue = MI.getOperand(2).getImm();
1158 return true;
1159 case AArch64::ANDSWri:
1160 case AArch64::ANDSXri:
1161 // ANDS does not use the same encoding scheme as the others xxxS
1162 // instructions.
1163 SrcReg = MI.getOperand(1).getReg();
1164 SrcReg2 = 0;
1165 CmpMask = ~0;
1167 MI.getOperand(2).getImm(),
1168 MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1169 return true;
1170 }
1171
1172 return false;
1173}
1174
1176 MachineBasicBlock *MBB = Instr.getParent();
1177 assert(MBB && "Can't get MachineBasicBlock here");
1178 MachineFunction *MF = MBB->getParent();
1179 assert(MF && "Can't get MachineFunction here");
1183
1184 for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1185 ++OpIdx) {
1186 MachineOperand &MO = Instr.getOperand(OpIdx);
1187 const TargetRegisterClass *OpRegCstraints =
1188 Instr.getRegClassConstraint(OpIdx, TII, TRI);
1189
1190 // If there's no constraint, there's nothing to do.
1191 if (!OpRegCstraints)
1192 continue;
1193 // If the operand is a frame index, there's nothing to do here.
1194 // A frame index operand will resolve correctly during PEI.
1195 if (MO.isFI())
1196 continue;
1197
1198 assert(MO.isReg() &&
1199 "Operand has register constraints without being a register!");
1200
1201 Register Reg = MO.getReg();
1202 if (Reg.isPhysical()) {
1203 if (!OpRegCstraints->contains(Reg))
1204 return false;
1205 } else if (!OpRegCstraints->hasSubClassEq(MRI->getRegClass(Reg)) &&
1206 !MRI->constrainRegClass(Reg, OpRegCstraints))
1207 return false;
1208 }
1209
1210 return true;
1211}
1212
1213/// Return the opcode that does not set flags when possible - otherwise
1214/// return the original opcode. The caller is responsible to do the actual
1215/// substitution and legality checking.
1217 // Don't convert all compare instructions, because for some the zero register
1218 // encoding becomes the sp register.
1219 bool MIDefinesZeroReg = false;
1220 if (MI.definesRegister(AArch64::WZR) || MI.definesRegister(AArch64::XZR))
1221 MIDefinesZeroReg = true;
1222
1223 switch (MI.getOpcode()) {
1224 default:
1225 return MI.getOpcode();
1226 case AArch64::ADDSWrr:
1227 return AArch64::ADDWrr;
1228 case AArch64::ADDSWri:
1229 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1230 case AArch64::ADDSWrs:
1231 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1232 case AArch64::ADDSWrx:
1233 return AArch64::ADDWrx;
1234 case AArch64::ADDSXrr:
1235 return AArch64::ADDXrr;
1236 case AArch64::ADDSXri:
1237 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1238 case AArch64::ADDSXrs:
1239 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1240 case AArch64::ADDSXrx:
1241 return AArch64::ADDXrx;
1242 case AArch64::SUBSWrr:
1243 return AArch64::SUBWrr;
1244 case AArch64::SUBSWri:
1245 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1246 case AArch64::SUBSWrs:
1247 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1248 case AArch64::SUBSWrx:
1249 return AArch64::SUBWrx;
1250 case AArch64::SUBSXrr:
1251 return AArch64::SUBXrr;
1252 case AArch64::SUBSXri:
1253 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1254 case AArch64::SUBSXrs:
1255 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1256 case AArch64::SUBSXrx:
1257 return AArch64::SUBXrx;
1258 }
1259}
1260
1261enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1262
1263/// True when condition flags are accessed (either by writing or reading)
1264/// on the instruction trace starting at From and ending at To.
1265///
1266/// Note: If From and To are from different blocks it's assumed CC are accessed
1267/// on the path.
1270 const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1271 // Early exit if To is at the beginning of the BB.
1272 if (To == To->getParent()->begin())
1273 return true;
1274
1275 // Check whether the instructions are in the same basic block
1276 // If not, assume the condition flags might get modified somewhere.
1277 if (To->getParent() != From->getParent())
1278 return true;
1279
1280 // From must be above To.
1281 assert(std::any_of(
1282 ++To.getReverse(), To->getParent()->rend(),
1283 [From](MachineInstr &MI) { return MI.getIterator() == From; }));
1284
1285 // We iterate backward starting at \p To until we hit \p From.
1286 for (const MachineInstr &Instr :
1287 instructionsWithoutDebug(++To.getReverse(), From.getReverse())) {
1288 if (((AccessToCheck & AK_Write) &&
1289 Instr.modifiesRegister(AArch64::NZCV, TRI)) ||
1290 ((AccessToCheck & AK_Read) && Instr.readsRegister(AArch64::NZCV, TRI)))
1291 return true;
1292 }
1293 return false;
1294}
1295
1296/// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
1297/// operation which could set the flags in an identical manner
1298bool AArch64InstrInfo::optimizePTestInstr(
1299 MachineInstr *PTest, unsigned MaskReg, unsigned PredReg,
1300 const MachineRegisterInfo *MRI) const {
1301 auto *Mask = MRI->getUniqueVRegDef(MaskReg);
1302 auto *Pred = MRI->getUniqueVRegDef(PredReg);
1303 auto NewOp = Pred->getOpcode();
1304 bool OpChanged = false;
1305
1306 unsigned MaskOpcode = Mask->getOpcode();
1307 unsigned PredOpcode = Pred->getOpcode();
1308 bool PredIsPTestLike = isPTestLikeOpcode(PredOpcode);
1309 bool PredIsWhileLike = isWhileOpcode(PredOpcode);
1310
1311 if (isPTrueOpcode(MaskOpcode) && (PredIsPTestLike || PredIsWhileLike) &&
1312 getElementSizeForOpcode(MaskOpcode) ==
1313 getElementSizeForOpcode(PredOpcode) &&
1314 Mask->getOperand(1).getImm() == 31) {
1315 // For PTEST(PTRUE_ALL, WHILE), if the element size matches, the PTEST is
1316 // redundant since WHILE performs an implicit PTEST with an all active
1317 // mask. Must be an all active predicate of matching element size.
1318
1319 // For PTEST(PTRUE_ALL, PTEST_LIKE), the PTEST is redundant if the
1320 // PTEST_LIKE instruction uses the same all active mask and the element
1321 // size matches. If the PTEST has a condition of any then it is always
1322 // redundant.
1323 if (PredIsPTestLike) {
1324 auto PTestLikeMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1325 if (Mask != PTestLikeMask && PTest->getOpcode() != AArch64::PTEST_PP_ANY)
1326 return false;
1327 }
1328
1329 // Fallthough to simply remove the PTEST.
1330 } else if ((Mask == Pred) && (PredIsPTestLike || PredIsWhileLike) &&
1331 PTest->getOpcode() == AArch64::PTEST_PP_ANY) {
1332 // For PTEST(PG, PG), PTEST is redundant when PG is the result of an
1333 // instruction that sets the flags as PTEST would. This is only valid when
1334 // the condition is any.
1335
1336 // Fallthough to simply remove the PTEST.
1337 } else if (PredIsPTestLike) {
1338 // For PTEST(PG, PTEST_LIKE(PG, ...)), the PTEST is redundant since the
1339 // flags are set based on the same mask 'PG', but PTEST_LIKE must operate
1340 // on 8-bit predicates like the PTEST. Otherwise, for instructions like
1341 // compare that also support 16/32/64-bit predicates, the implicit PTEST
1342 // performed by the compare could consider fewer lanes for these element
1343 // sizes.
1344 //
1345 // For example, consider
1346 //
1347 // ptrue p0.b ; P0=1111-1111-1111-1111
1348 // index z0.s, #0, #1 ; Z0=<0,1,2,3>
1349 // index z1.s, #1, #1 ; Z1=<1,2,3,4>
1350 // cmphi p1.s, p0/z, z1.s, z0.s ; P1=0001-0001-0001-0001
1351 // ; ^ last active
1352 // ptest p0, p1.b ; P1=0001-0001-0001-0001
1353 // ; ^ last active
1354 //
1355 // where the compare generates a canonical all active 32-bit predicate
1356 // (equivalent to 'ptrue p1.s, all'). The implicit PTEST sets the last
1357 // active flag, whereas the PTEST instruction with the same mask doesn't.
1358 // For PTEST_ANY this doesn't apply as the flags in this case would be
1359 // identical regardless of element size.
1360 auto PTestLikeMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1361 uint64_t PredElementSize = getElementSizeForOpcode(PredOpcode);
1362 if ((Mask != PTestLikeMask) ||
1363 (PredElementSize != AArch64::ElementSizeB &&
1364 PTest->getOpcode() != AArch64::PTEST_PP_ANY))
1365 return false;
1366
1367 // Fallthough to simply remove the PTEST.
1368 } else {
1369 // If OP in PTEST(PG, OP(PG, ...)) has a flag-setting variant change the
1370 // opcode so the PTEST becomes redundant.
1371 switch (PredOpcode) {
1372 case AArch64::AND_PPzPP:
1373 case AArch64::BIC_PPzPP:
1374 case AArch64::EOR_PPzPP:
1375 case AArch64::NAND_PPzPP:
1376 case AArch64::NOR_PPzPP:
1377 case AArch64::ORN_PPzPP:
1378 case AArch64::ORR_PPzPP:
1379 case AArch64::BRKA_PPzP:
1380 case AArch64::BRKPA_PPzPP:
1381 case AArch64::BRKB_PPzP:
1382 case AArch64::BRKPB_PPzPP:
1383 case AArch64::RDFFR_PPz: {
1384 // Check to see if our mask is the same. If not the resulting flag bits
1385 // may be different and we can't remove the ptest.
1386 auto *PredMask = MRI->getUniqueVRegDef(Pred->getOperand(1).getReg());
1387 if (Mask != PredMask)
1388 return false;
1389 break;
1390 }
1391 case AArch64::BRKN_PPzP: {
1392 // BRKN uses an all active implicit mask to set flags unlike the other
1393 // flag-setting instructions.
1394 // PTEST(PTRUE_B(31), BRKN(PG, A, B)) -> BRKNS(PG, A, B).
1395 if ((MaskOpcode != AArch64::PTRUE_B) ||
1396 (Mask->getOperand(1).getImm() != 31))
1397 return false;
1398 break;
1399 }
1400 case AArch64::PTRUE_B:
1401 // PTEST(OP=PTRUE_B(A), OP) -> PTRUES_B(A)
1402 break;
1403 default:
1404 // Bail out if we don't recognize the input
1405 return false;
1406 }
1407
1408 NewOp = convertToFlagSettingOpc(PredOpcode);
1409 OpChanged = true;
1410 }
1411
1413
1414 // If another instruction between Pred and PTest accesses flags, don't remove
1415 // the ptest or update the earlier instruction to modify them.
1416 if (areCFlagsAccessedBetweenInstrs(Pred, PTest, TRI))
1417 return false;
1418
1419 // If we pass all the checks, it's safe to remove the PTEST and use the flags
1420 // as they are prior to PTEST. Sometimes this requires the tested PTEST
1421 // operand to be replaced with an equivalent instruction that also sets the
1422 // flags.
1423 Pred->setDesc(get(NewOp));
1424 PTest->eraseFromParent();
1425 if (OpChanged) {
1426 bool succeeded = UpdateOperandRegClass(*Pred);
1427 (void)succeeded;
1428 assert(succeeded && "Operands have incompatible register classes!");
1429 Pred->addRegisterDefined(AArch64::NZCV, TRI);
1430 }
1431
1432 // Ensure that the flags def is live.
1433 if (Pred->registerDefIsDead(AArch64::NZCV, TRI)) {
1434 unsigned i = 0, e = Pred->getNumOperands();
1435 for (; i != e; ++i) {
1436 MachineOperand &MO = Pred->getOperand(i);
1437 if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::NZCV) {
1438 MO.setIsDead(false);
1439 break;
1440 }
1441 }
1442 }
1443 return true;
1444}
1445
1446/// Try to optimize a compare instruction. A compare instruction is an
1447/// instruction which produces AArch64::NZCV. It can be truly compare
1448/// instruction
1449/// when there are no uses of its destination register.
1450///
1451/// The following steps are tried in order:
1452/// 1. Convert CmpInstr into an unconditional version.
1453/// 2. Remove CmpInstr if above there is an instruction producing a needed
1454/// condition code or an instruction which can be converted into such an
1455/// instruction.
1456/// Only comparison with zero is supported.
1458 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
1459 int64_t CmpValue, const MachineRegisterInfo *MRI) const {
1460 assert(CmpInstr.getParent());
1461 assert(MRI);
1462
1463 // Replace SUBSWrr with SUBWrr if NZCV is not used.
1464 int DeadNZCVIdx = CmpInstr.findRegisterDefOperandIdx(AArch64::NZCV, true);
1465 if (DeadNZCVIdx != -1) {
1466 if (CmpInstr.definesRegister(AArch64::WZR) ||
1467 CmpInstr.definesRegister(AArch64::XZR)) {
1468 CmpInstr.eraseFromParent();
1469 return true;
1470 }
1471 unsigned Opc = CmpInstr.getOpcode();
1472 unsigned NewOpc = convertToNonFlagSettingOpc(CmpInstr);
1473 if (NewOpc == Opc)
1474 return false;
1475 const MCInstrDesc &MCID = get(NewOpc);
1476 CmpInstr.setDesc(MCID);
1477 CmpInstr.removeOperand(DeadNZCVIdx);
1478 bool succeeded = UpdateOperandRegClass(CmpInstr);
1479 (void)succeeded;
1480 assert(succeeded && "Some operands reg class are incompatible!");
1481 return true;
1482 }
1483
1484 if (CmpInstr.getOpcode() == AArch64::PTEST_PP ||
1485 CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY)
1486 return optimizePTestInstr(&CmpInstr, SrcReg, SrcReg2, MRI);
1487
1488 if (SrcReg2 != 0)
1489 return false;
1490
1491 // CmpInstr is a Compare instruction if destination register is not used.
1492 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
1493 return false;
1494
1495 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, *MRI))
1496 return true;
1497 return (CmpValue == 0 || CmpValue == 1) &&
1498 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, *MRI);
1499}
1500
1501/// Get opcode of S version of Instr.
1502/// If Instr is S version its opcode is returned.
1503/// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
1504/// or we are not interested in it.
1505static unsigned sForm(MachineInstr &Instr) {
1506 switch (Instr.getOpcode()) {
1507 default:
1508 return AArch64::INSTRUCTION_LIST_END;
1509
1510 case AArch64::ADDSWrr:
1511 case AArch64::ADDSWri:
1512 case AArch64::ADDSXrr:
1513 case AArch64::ADDSXri:
1514 case AArch64::SUBSWrr:
1515 case AArch64::SUBSWri:
1516 case AArch64::SUBSXrr:
1517 case AArch64::SUBSXri:
1518 return Instr.getOpcode();
1519
1520 case AArch64::ADDWrr:
1521 return AArch64::ADDSWrr;
1522 case AArch64::ADDWri:
1523 return AArch64::ADDSWri;
1524 case AArch64::ADDXrr:
1525 return AArch64::ADDSXrr;
1526 case AArch64::ADDXri:
1527 return AArch64::ADDSXri;
1528 case AArch64::ADCWr:
1529 return AArch64::ADCSWr;
1530 case AArch64::ADCXr:
1531 return AArch64::ADCSXr;
1532 case AArch64::SUBWrr:
1533 return AArch64::SUBSWrr;
1534 case AArch64::SUBWri:
1535 return AArch64::SUBSWri;
1536 case AArch64::SUBXrr:
1537 return AArch64::SUBSXrr;
1538 case AArch64::SUBXri:
1539 return AArch64::SUBSXri;
1540 case AArch64::SBCWr:
1541 return AArch64::SBCSWr;
1542 case AArch64::SBCXr:
1543 return AArch64::SBCSXr;
1544 case AArch64::ANDWri:
1545 return AArch64::ANDSWri;
1546 case AArch64::ANDXri:
1547 return AArch64::ANDSXri;
1548 }
1549}
1550
1551/// Check if AArch64::NZCV should be alive in successors of MBB.
1553 for (auto *BB : MBB->successors())
1554 if (BB->isLiveIn(AArch64::NZCV))
1555 return true;
1556 return false;
1557}
1558
1559/// \returns The condition code operand index for \p Instr if it is a branch
1560/// or select and -1 otherwise.
1561static int
1563 switch (Instr.getOpcode()) {
1564 default:
1565 return -1;
1566
1567 case AArch64::Bcc: {
1568 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1569 assert(Idx >= 2);
1570 return Idx - 2;
1571 }
1572
1573 case AArch64::CSINVWr:
1574 case AArch64::CSINVXr:
1575 case AArch64::CSINCWr:
1576 case AArch64::CSINCXr:
1577 case AArch64::CSELWr:
1578 case AArch64::CSELXr:
1579 case AArch64::CSNEGWr:
1580 case AArch64::CSNEGXr:
1581 case AArch64::FCSELSrrr:
1582 case AArch64::FCSELDrrr: {
1583 int Idx = Instr.findRegisterUseOperandIdx(AArch64::NZCV);
1584 assert(Idx >= 1);
1585 return Idx - 1;
1586 }
1587 }
1588}
1589
1590/// Find a condition code used by the instruction.
1591/// Returns AArch64CC::Invalid if either the instruction does not use condition
1592/// codes or we don't optimize CmpInstr in the presence of such instructions.
1595 return CCIdx >= 0 ? static_cast<AArch64CC::CondCode>(
1596 Instr.getOperand(CCIdx).getImm())
1598}
1599
1602 UsedNZCV UsedFlags;
1603 switch (CC) {
1604 default:
1605 break;
1606
1607 case AArch64CC::EQ: // Z set
1608 case AArch64CC::NE: // Z clear
1609 UsedFlags.Z = true;
1610 break;
1611
1612 case AArch64CC::HI: // Z clear and C set
1613 case AArch64CC::LS: // Z set or C clear
1614 UsedFlags.Z = true;
1615 [[fallthrough]];
1616 case AArch64CC::HS: // C set
1617 case AArch64CC::LO: // C clear
1618 UsedFlags.C = true;
1619 break;
1620
1621 case AArch64CC::MI: // N set
1622 case AArch64CC::PL: // N clear
1623 UsedFlags.N = true;
1624 break;
1625
1626 case AArch64CC::VS: // V set
1627 case AArch64CC::VC: // V clear
1628 UsedFlags.V = true;
1629 break;
1630
1631 case AArch64CC::GT: // Z clear, N and V the same
1632 case AArch64CC::LE: // Z set, N and V differ
1633 UsedFlags.Z = true;
1634 [[fallthrough]];
1635 case AArch64CC::GE: // N and V the same
1636 case AArch64CC::LT: // N and V differ
1637 UsedFlags.N = true;
1638 UsedFlags.V = true;
1639 break;
1640 }
1641 return UsedFlags;
1642}
1643
1644/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
1645/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
1646/// \returns std::nullopt otherwise.
1647///
1648/// Collect instructions using that flags in \p CCUseInstrs if provided.
1649std::optional<UsedNZCV>
1651 const TargetRegisterInfo &TRI,
1652 SmallVectorImpl<MachineInstr *> *CCUseInstrs) {
1653 MachineBasicBlock *CmpParent = CmpInstr.getParent();
1654 if (MI.getParent() != CmpParent)
1655 return std::nullopt;
1656
1657 if (areCFlagsAliveInSuccessors(CmpParent))
1658 return std::nullopt;
1659
1660 UsedNZCV NZCVUsedAfterCmp;
1662 std::next(CmpInstr.getIterator()), CmpParent->instr_end())) {
1663 if (Instr.readsRegister(AArch64::NZCV, &TRI)) {
1665 if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
1666 return std::nullopt;
1667 NZCVUsedAfterCmp |= getUsedNZCV(CC);
1668 if (CCUseInstrs)
1669 CCUseInstrs->push_back(&Instr);
1670 }
1671 if (Instr.modifiesRegister(AArch64::NZCV, &TRI))
1672 break;
1673 }
1674 return NZCVUsedAfterCmp;
1675}
1676
1677static bool isADDSRegImm(unsigned Opcode) {
1678 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
1679}
1680
1681static bool isSUBSRegImm(unsigned Opcode) {
1682 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
1683}
1684
1685/// Check if CmpInstr can be substituted by MI.
1686///
1687/// CmpInstr can be substituted:
1688/// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
1689/// - and, MI and CmpInstr are from the same MachineBB
1690/// - and, condition flags are not alive in successors of the CmpInstr parent
1691/// - and, if MI opcode is the S form there must be no defs of flags between
1692/// MI and CmpInstr
1693/// or if MI opcode is not the S form there must be neither defs of flags
1694/// nor uses of flags between MI and CmpInstr.
1695/// - and, if C/V flags are not used after CmpInstr
1696/// or if N flag is used but MI produces poison value if signed overflow
1697/// occurs.
1699 const TargetRegisterInfo &TRI) {
1700 // NOTE this assertion guarantees that MI.getOpcode() is add or subtraction
1701 // that may or may not set flags.
1702 assert(sForm(MI) != AArch64::INSTRUCTION_LIST_END);
1703
1704 const unsigned CmpOpcode = CmpInstr.getOpcode();
1705 if (!isADDSRegImm(CmpOpcode) && !isSUBSRegImm(CmpOpcode))
1706 return false;
1707
1708 assert((CmpInstr.getOperand(2).isImm() &&
1709 CmpInstr.getOperand(2).getImm() == 0) &&
1710 "Caller guarantees that CmpInstr compares with constant 0");
1711
1712 std::optional<UsedNZCV> NZVCUsed = examineCFlagsUse(MI, CmpInstr, TRI);
1713 if (!NZVCUsed || NZVCUsed->C)
1714 return false;
1715
1716 // CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0', and MI is either
1717 // '%vreg = add ...' or '%vreg = sub ...'.
1718 // Condition flag V is used to indicate signed overflow.
1719 // 1) MI and CmpInstr set N and V to the same value.
1720 // 2) If MI is add/sub with no-signed-wrap, it produces a poison value when
1721 // signed overflow occurs, so CmpInstr could still be simplified away.
1722 if (NZVCUsed->V && !MI.getFlag(MachineInstr::NoSWrap))
1723 return false;
1724
1725 AccessKind AccessToCheck = AK_Write;
1726 if (sForm(MI) != MI.getOpcode())
1727 AccessToCheck = AK_All;
1728 return !areCFlagsAccessedBetweenInstrs(&MI, &CmpInstr, &TRI, AccessToCheck);
1729}
1730
1731/// Substitute an instruction comparing to zero with another instruction
1732/// which produces needed condition flags.
1733///
1734/// Return true on success.
1735bool AArch64InstrInfo::substituteCmpToZero(
1736 MachineInstr &CmpInstr, unsigned SrcReg,
1737 const MachineRegisterInfo &MRI) const {
1738 // Get the unique definition of SrcReg.
1739 MachineInstr *MI = MRI.getUniqueVRegDef(SrcReg);
1740 if (!MI)
1741 return false;
1742
1744
1745 unsigned NewOpc = sForm(*MI);
1746 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
1747 return false;
1748
1749 if (!canInstrSubstituteCmpInstr(*MI, CmpInstr, TRI))
1750 return false;
1751
1752 // Update the instruction to set NZCV.
1753 MI->setDesc(get(NewOpc));
1754 CmpInstr.eraseFromParent();
1755 bool succeeded = UpdateOperandRegClass(*MI);
1756 (void)succeeded;
1757 assert(succeeded && "Some operands reg class are incompatible!");
1758 MI->addRegisterDefined(AArch64::NZCV, &TRI);
1759 return true;
1760}
1761
1762/// \returns True if \p CmpInstr can be removed.
1763///
1764/// \p IsInvertCC is true if, after removing \p CmpInstr, condition
1765/// codes used in \p CCUseInstrs must be inverted.
1767 int CmpValue, const TargetRegisterInfo &TRI,
1769 bool &IsInvertCC) {
1770 assert((CmpValue == 0 || CmpValue == 1) &&
1771 "Only comparisons to 0 or 1 considered for removal!");
1772
1773 // MI is 'CSINCWr %vreg, wzr, wzr, <cc>' or 'CSINCXr %vreg, xzr, xzr, <cc>'
1774 unsigned MIOpc = MI.getOpcode();
1775 if (MIOpc == AArch64::CSINCWr) {
1776 if (MI.getOperand(1).getReg() != AArch64::WZR ||
1777 MI.getOperand(2).getReg() != AArch64::WZR)
1778 return false;
1779 } else if (MIOpc == AArch64::CSINCXr) {
1780 if (MI.getOperand(1).getReg() != AArch64::XZR ||
1781 MI.getOperand(2).getReg() != AArch64::XZR)
1782 return false;
1783 } else {
1784 return false;
1785 }
1787 if (MICC == AArch64CC::Invalid)
1788 return false;
1789
1790 // NZCV needs to be defined
1791 if (MI.findRegisterDefOperandIdx(AArch64::NZCV, true) != -1)
1792 return false;
1793
1794 // CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0' or 'SUBS %vreg, 1'
1795 const unsigned CmpOpcode = CmpInstr.getOpcode();
1796 bool IsSubsRegImm = isSUBSRegImm(CmpOpcode);
1797 if (CmpValue && !IsSubsRegImm)
1798 return false;
1799 if (!CmpValue && !IsSubsRegImm && !isADDSRegImm(CmpOpcode))
1800 return false;
1801
1802 // MI conditions allowed: eq, ne, mi, pl
1803 UsedNZCV MIUsedNZCV = getUsedNZCV(MICC);
1804 if (MIUsedNZCV.C || MIUsedNZCV.V)
1805 return false;
1806
1807 std::optional<UsedNZCV> NZCVUsedAfterCmp =
1808 examineCFlagsUse(MI, CmpInstr, TRI, &CCUseInstrs);
1809 // Condition flags are not used in CmpInstr basic block successors and only
1810 // Z or N flags allowed to be used after CmpInstr within its basic block
1811 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
1812 return false;
1813 // Z or N flag used after CmpInstr must correspond to the flag used in MI
1814 if ((MIUsedNZCV.Z && NZCVUsedAfterCmp->N) ||
1815 (MIUsedNZCV.N && NZCVUsedAfterCmp->Z))
1816 return false;
1817 // If CmpInstr is comparison to zero MI conditions are limited to eq, ne
1818 if (MIUsedNZCV.N && !CmpValue)
1819 return false;
1820
1821 // There must be no defs of flags between MI and CmpInstr
1822 if (areCFlagsAccessedBetweenInstrs(&MI, &CmpInstr, &TRI, AK_Write))
1823 return false;
1824
1825 // Condition code is inverted in the following cases:
1826 // 1. MI condition is ne; CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
1827 // 2. MI condition is eq, pl; CmpInstr is 'SUBS %vreg, 1'
1828 IsInvertCC = (CmpValue && (MICC == AArch64CC::EQ || MICC == AArch64CC::PL)) ||
1829 (!CmpValue && MICC == AArch64CC::NE);
1830 return true;
1831}
1832
1833/// Remove comparison in csinc-cmp sequence
1834///
1835/// Examples:
1836/// 1. \code
1837/// csinc w9, wzr, wzr, ne
1838/// cmp w9, #0
1839/// b.eq
1840/// \endcode
1841/// to
1842/// \code
1843/// csinc w9, wzr, wzr, ne
1844/// b.ne
1845/// \endcode
1846///
1847/// 2. \code
1848/// csinc x2, xzr, xzr, mi
1849/// cmp x2, #1
1850/// b.pl
1851/// \endcode
1852/// to
1853/// \code
1854/// csinc x2, xzr, xzr, mi
1855/// b.pl
1856/// \endcode
1857///
1858/// \param CmpInstr comparison instruction
1859/// \return True when comparison removed
1860bool AArch64InstrInfo::removeCmpToZeroOrOne(
1861 MachineInstr &CmpInstr, unsigned SrcReg, int CmpValue,
1862 const MachineRegisterInfo &MRI) const {
1863 MachineInstr *MI = MRI.getUniqueVRegDef(SrcReg);
1864 if (!MI)
1865 return false;
1868 bool IsInvertCC = false;
1869 if (!canCmpInstrBeRemoved(*MI, CmpInstr, CmpValue, TRI, CCUseInstrs,
1870 IsInvertCC))
1871 return false;
1872 // Make transformation
1873 CmpInstr.eraseFromParent();
1874 if (IsInvertCC) {
1875 // Invert condition codes in CmpInstr CC users
1876 for (MachineInstr *CCUseInstr : CCUseInstrs) {
1878 assert(Idx >= 0 && "Unexpected instruction using CC.");
1879 MachineOperand &CCOperand = CCUseInstr->getOperand(Idx);
1881 static_cast<AArch64CC::CondCode>(CCOperand.getImm()));
1882 CCOperand.setImm(CCUse);
1883 }
1884 }
1885 return true;
1886}
1887
1889 if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
1890 MI.getOpcode() != AArch64::CATCHRET)
1891 return false;
1892
1893 MachineBasicBlock &MBB = *MI.getParent();
1894 auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
1895 auto TRI = Subtarget.getRegisterInfo();
1896 DebugLoc DL = MI.getDebugLoc();
1897
1898 if (MI.getOpcode() == AArch64::CATCHRET) {
1899 // Skip to the first instruction before the epilog.
1900 const TargetInstrInfo *TII =
1902 MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
1904 MachineBasicBlock::iterator FirstEpilogSEH = std::prev(MBBI);
1905 while (FirstEpilogSEH->getFlag(MachineInstr::FrameDestroy) &&
1906 FirstEpilogSEH != MBB.begin())
1907 FirstEpilogSEH = std::prev(FirstEpilogSEH);
1908 if (FirstEpilogSEH != MBB.begin())
1909 FirstEpilogSEH = std::next(FirstEpilogSEH);
1910 BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADRP))
1911 .addReg(AArch64::X0, RegState::Define)
1912 .addMBB(TargetMBB);
1913 BuildMI(MBB, FirstEpilogSEH, DL, TII->get(AArch64::ADDXri))
1914 .addReg(AArch64::X0, RegState::Define)
1915 .addReg(AArch64::X0)
1916 .addMBB(TargetMBB)
1917 .addImm(0);
1918 return true;
1919 }
1920
1921 Register Reg = MI.getOperand(0).getReg();
1923 if (M.getStackProtectorGuard() == "sysreg") {
1924 const AArch64SysReg::SysReg *SrcReg =
1925 AArch64SysReg::lookupSysRegByName(M.getStackProtectorGuardReg());
1926 if (!SrcReg)
1927 report_fatal_error("Unknown SysReg for Stack Protector Guard Register");
1928
1929 // mrs xN, sysreg
1930 BuildMI(MBB, MI, DL, get(AArch64::MRS))
1932 .addImm(SrcReg->Encoding);
1933 int Offset = M.getStackProtectorGuardOffset();
1934 if (Offset >= 0 && Offset <= 32760 && Offset % 8 == 0) {
1935 // ldr xN, [xN, #offset]
1936 BuildMI(MBB, MI, DL, get(AArch64::LDRXui))
1937 .addDef(Reg)
1938 .addUse(Reg, RegState::Kill)
1939 .addImm(Offset / 8);
1940 } else if (Offset >= -256 && Offset <= 255) {
1941 // ldur xN, [xN, #offset]
1942 BuildMI(MBB, MI, DL, get(AArch64::LDURXi))
1943 .addDef(Reg)
1944 .addUse(Reg, RegState::Kill)
1945 .addImm(Offset);
1946 } else if (Offset >= -4095 && Offset <= 4095) {
1947 if (Offset > 0) {
1948 // add xN, xN, #offset
1949 BuildMI(MBB, MI, DL, get(AArch64::ADDXri))
1950 .addDef(Reg)
1951 .addUse(Reg, RegState::Kill)
1952 .addImm(Offset)
1953 .addImm(0);
1954 } else {
1955 // sub xN, xN, #offset
1956 BuildMI(MBB, MI, DL, get(AArch64::SUBXri))
1957 .addDef(Reg)
1958 .addUse(Reg, RegState::Kill)
1959 .addImm(-Offset)
1960 .addImm(0);
1961 }
1962 // ldr xN, [xN]
1963 BuildMI(MBB, MI, DL, get(AArch64::LDRXui))
1964 .addDef(Reg)
1965 .addUse(Reg, RegState::Kill)
1966 .addImm(0);
1967 } else {
1968 // Cases that are larger than +/- 4095 and not a multiple of 8, or larger
1969 // than 23760.
1970 // It might be nice to use AArch64::MOVi32imm here, which would get
1971 // expanded in PreSched2 after PostRA, but our lone scratch Reg already
1972 // contains the MRS result. findScratchNonCalleeSaveRegister() in
1973 // AArch64FrameLowering might help us find such a scratch register
1974 // though. If we failed to find a scratch register, we could emit a
1975 // stream of add instructions to build up the immediate. Or, we could try
1976 // to insert a AArch64::MOVi32imm before register allocation so that we
1977 // didn't need to scavenge for a scratch register.
1978 report_fatal_error("Unable to encode Stack Protector Guard Offset");
1979 }
1980 MBB.erase(MI);
1981 return true;
1982 }
1983
1984 const GlobalValue *GV =
1985 cast<GlobalValue>((*MI.memoperands_begin())->getValue());
1986 const TargetMachine &TM = MBB.getParent()->getTarget();
1987 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
1988 const unsigned char MO_NC = AArch64II::MO_NC;
1989
1990 if ((OpFlags & AArch64II::MO_GOT) != 0) {
1991 BuildMI(MBB, MI, DL, get(AArch64::LOADgot), Reg)
1992 .addGlobalAddress(GV, 0, OpFlags);
1993 if (Subtarget.isTargetILP32()) {
1994 unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
1995 BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
1996 .addDef(Reg32, RegState::Dead)
1997 .addUse(Reg, RegState::Kill)
1998 .addImm(0)
1999 .addMemOperand(*MI.memoperands_begin())
2001 } else {
2002 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
2003 .addReg(Reg, RegState::Kill)
2004 .addImm(0)
2005 .addMemOperand(*MI.memoperands_begin());
2006 }
2007 } else if (TM.getCodeModel() == CodeModel::Large) {
2008 assert(!Subtarget.isTargetILP32() && "how can large exist in ILP32?");
2009 BuildMI(MBB, MI, DL, get(AArch64::MOVZXi), Reg)
2010 .addGlobalAddress(GV, 0, AArch64II::MO_G0 | MO_NC)
2011 .addImm(0);
2012 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
2013 .addReg(Reg, RegState::Kill)
2014 .addGlobalAddress(GV, 0, AArch64II::MO_G1 | MO_NC)
2015 .addImm(16);
2016 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
2017 .addReg(Reg, RegState::Kill)
2018 .addGlobalAddress(GV, 0, AArch64II::MO_G2 | MO_NC)
2019 .addImm(32);
2020 BuildMI(MBB, MI, DL, get(AArch64::MOVKXi), Reg)
2021 .addReg(Reg, RegState::Kill)
2023 .addImm(48);
2024 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
2025 .addReg(Reg, RegState::Kill)
2026 .addImm(0)
2027 .addMemOperand(*MI.memoperands_begin());
2028 } else if (TM.getCodeModel() == CodeModel::Tiny) {
2029 BuildMI(MBB, MI, DL, get(AArch64::ADR), Reg)
2030 .addGlobalAddress(GV, 0, OpFlags);
2031 } else {
2032 BuildMI(MBB, MI, DL, get(AArch64::ADRP), Reg)
2033 .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE);
2034 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
2035 if (Subtarget.isTargetILP32()) {
2036 unsigned Reg32 = TRI->getSubReg(Reg, AArch64::sub_32);
2037 BuildMI(MBB, MI, DL, get(AArch64::LDRWui))
2038 .addDef(Reg32, RegState::Dead)
2039 .addUse(Reg, RegState::Kill)
2040 .addGlobalAddress(GV, 0, LoFlags)
2041 .addMemOperand(*MI.memoperands_begin())
2043 } else {
2044 BuildMI(MBB, MI, DL, get(AArch64::LDRXui), Reg)
2045 .addReg(Reg, RegState::Kill)
2046 .addGlobalAddress(GV, 0, LoFlags)
2047 .addMemOperand(*MI.memoperands_begin());
2048 }
2049 }
2050
2051 MBB.erase(MI);
2052
2053 return true;
2054}
2055
2056// Return true if this instruction simply sets its single destination register
2057// to zero. This is equivalent to a register rename of the zero-register.
2059 switch (MI.getOpcode()) {
2060 default:
2061 break;
2062 case AArch64::MOVZWi:
2063 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
2064 if (MI.getOperand(1).isImm() && MI.getOperand(1).getImm() == 0) {
2065 assert(MI.getDesc().getNumOperands() == 3 &&
2066 MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
2067 return true;
2068 }
2069 break;
2070 case AArch64::ANDWri: // and Rd, Rzr, #imm
2071 return MI.getOperand(1).getReg() == AArch64::WZR;
2072 case AArch64::ANDXri:
2073 return MI.getOperand(1).getReg() == AArch64::XZR;
2074 case TargetOpcode::COPY:
2075 return MI.getOperand(1).getReg() == AArch64::WZR;
2076 }
2077 return false;
2078}
2079
2080// Return true if this instruction simply renames a general register without
2081// modifying bits.
2083 switch (MI.getOpcode()) {
2084 default:
2085 break;
2086 case TargetOpcode::COPY: {
2087 // GPR32 copies will by lowered to ORRXrs
2088 Register DstReg = MI.getOperand(0).getReg();
2089 return (AArch64::GPR32RegClass.contains(DstReg) ||
2090 AArch64::GPR64RegClass.contains(DstReg));
2091 }
2092 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
2093 if (MI.getOperand(1).getReg() == AArch64::XZR) {
2094 assert(MI.getDesc().getNumOperands() == 4 &&
2095 MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
2096 return true;
2097 }
2098 break;
2099 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
2100 if (MI.getOperand(2).getImm() == 0) {
2101 assert(MI.getDesc().getNumOperands() == 4 &&
2102 MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
2103 return true;
2104 }
2105 break;
2106 }
2107 return false;
2108}
2109
2110// Return true if this instruction simply renames a general register without
2111// modifying bits.
2113 switch (MI.getOpcode()) {
2114 default:
2115 break;
2116 case TargetOpcode::COPY: {
2117 Register DstReg = MI.getOperand(0).getReg();
2118 return AArch64::FPR128RegClass.contains(DstReg);
2119 }
2120 case AArch64::ORRv16i8:
2121 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
2122 assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
2123 "invalid ORRv16i8 operands");
2124 return true;
2125 }
2126 break;
2127 }
2128 return false;
2129}
2130
2132 int &FrameIndex) const {
2133 switch (MI.getOpcode()) {
2134 default:
2135 break;
2136 case AArch64::LDRWui:
2137 case AArch64::LDRXui:
2138 case AArch64::LDRBui:
2139 case AArch64::LDRHui:
2140 case AArch64::LDRSui:
2141 case AArch64::LDRDui:
2142 case AArch64::LDRQui:
2143 if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
2144 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
2145 FrameIndex = MI.getOperand(1).getIndex();
2146 return MI.getOperand(0).getReg();
2147 }
2148 break;
2149 }
2150
2151 return 0;
2152}
2153
2155 int &FrameIndex) const {
2156 switch (MI.getOpcode()) {
2157 default:
2158 break;
2159 case AArch64::STRWui:
2160 case AArch64::STRXui:
2161 case AArch64::STRBui:
2162 case AArch64::STRHui:
2163 case AArch64::STRSui:
2164 case AArch64::STRDui:
2165 case AArch64::STRQui:
2166 case AArch64::LDR_PXI:
2167 case AArch64::STR_PXI:
2168 if (MI.getOperand(0).getSubReg() == 0 && MI.getOperand(1).isFI() &&
2169 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0) {
2170 FrameIndex = MI.getOperand(1).getIndex();
2171 return MI.getOperand(0).getReg();
2172 }
2173 break;
2174 }
2175 return 0;
2176}
2177
2178/// Check all MachineMemOperands for a hint to suppress pairing.
2180 return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
2181 return MMO->getFlags() & MOSuppressPair;
2182 });
2183}
2184
2185/// Set a flag on the first MachineMemOperand to suppress pairing.
2187 if (MI.memoperands_empty())
2188 return;
2189 (*MI.memoperands_begin())->setFlags(MOSuppressPair);
2190}
2191
2192/// Check all MachineMemOperands for a hint that the load/store is strided.
2194 return llvm::any_of(MI.memoperands(), [](MachineMemOperand *MMO) {
2195 return MMO->getFlags() & MOStridedAccess;
2196 });
2197}
2198
2200 switch (Opc) {
2201 default:
2202 return false;
2203 case AArch64::STURSi:
2204 case AArch64::STRSpre:
2205 case AArch64::STURDi:
2206 case AArch64::STRDpre:
2207 case AArch64::STURQi:
2208 case AArch64::STRQpre:
2209 case AArch64::STURBBi:
2210 case AArch64::STURHHi:
2211 case AArch64::STURWi:
2212 case AArch64::STRWpre:
2213 case AArch64::STURXi:
2214 case AArch64::STRXpre:
2215 case AArch64::LDURSi:
2216 case AArch64::LDRSpre:
2217 case AArch64::LDURDi:
2218 case AArch64::LDRDpre:
2219 case AArch64::LDURQi:
2220 case AArch64::LDRQpre:
2221 case AArch64::LDURWi:
2222 case AArch64::LDRWpre:
2223 case AArch64::LDURXi:
2224 case AArch64::LDRXpre:
2225 case AArch64::LDURSWi:
2226 case AArch64::LDURHHi:
2227 case AArch64::LDURBBi:
2228 case AArch64::LDURSBWi:
2229 case AArch64::LDURSHWi:
2230 return true;
2231 }
2232}
2233
2234std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
2235 switch (Opc) {
2236 default: return {};
2237 case AArch64::PRFMui: return AArch64::PRFUMi;
2238 case AArch64::LDRXui: return AArch64::LDURXi;
2239 case AArch64::LDRWui: return AArch64::LDURWi;
2240 case AArch64::LDRBui: return AArch64::LDURBi;
2241 case AArch64::LDRHui: return AArch64::LDURHi;
2242 case AArch64::LDRSui: return AArch64::LDURSi;
2243 case AArch64::LDRDui: return AArch64::LDURDi;
2244 case AArch64::LDRQui: return AArch64::LDURQi;
2245 case AArch64::LDRBBui: return AArch64::LDURBBi;
2246 case AArch64::LDRHHui: return AArch64::LDURHHi;
2247 case AArch64::LDRSBXui: return AArch64::LDURSBXi;
2248 case AArch64::LDRSBWui: return AArch64::LDURSBWi;
2249 case AArch64::LDRSHXui: return AArch64::LDURSHXi;
2250 case AArch64::LDRSHWui: return AArch64::LDURSHWi;
2251 case AArch64::LDRSWui: return AArch64::LDURSWi;
2252 case AArch64::STRXui: return AArch64::STURXi;
2253 case AArch64::STRWui: return AArch64::STURWi;
2254 case AArch64::STRBui: return AArch64::STURBi;
2255 case AArch64::STRHui: return AArch64::STURHi;
2256 case AArch64::STRSui: return AArch64::STURSi;
2257 case AArch64::STRDui: return AArch64::STURDi;
2258 case AArch64::STRQui: return AArch64::STURQi;
2259 case AArch64::STRBBui: return AArch64::STURBBi;
2260 case AArch64::STRHHui: return AArch64::STURHHi;
2261 }
2262}
2263
2265 switch (Opc) {
2266 default:
2267 return 2;
2268 case AArch64::LDPXi:
2269 case AArch64::LDPDi:
2270 case AArch64::STPXi:
2271 case AArch64::STPDi:
2272 case AArch64::LDNPXi:
2273 case AArch64::LDNPDi:
2274 case AArch64::STNPXi:
2275 case AArch64::STNPDi:
2276 case AArch64::LDPQi:
2277 case AArch64::STPQi:
2278 case AArch64::LDNPQi:
2279 case AArch64::STNPQi:
2280 case AArch64::LDPWi:
2281 case AArch64::LDPSi:
2282 case AArch64::STPWi:
2283 case AArch64::STPSi:
2284 case AArch64::LDNPWi:
2285 case AArch64::LDNPSi:
2286 case AArch64::STNPWi:
2287 case AArch64::STNPSi:
2288 case AArch64::LDG:
2289 case AArch64::STGPi:
2290
2291 case AArch64::LD1B_IMM:
2292 case AArch64::LD1B_H_IMM:
2293 case AArch64::LD1B_S_IMM:
2294 case AArch64::LD1B_D_IMM:
2295 case AArch64::LD1SB_H_IMM:
2296 case AArch64::LD1SB_S_IMM:
2297 case AArch64::LD1SB_D_IMM:
2298 case AArch64::LD1H_IMM:
2299 case AArch64::LD1H_S_IMM:
2300 case AArch64::LD1H_D_IMM:
2301 case AArch64::LD1SH_S_IMM:
2302 case AArch64::LD1SH_D_IMM:
2303 case AArch64::LD1W_IMM:
2304 case AArch64::LD1W_D_IMM:
2305 case AArch64::LD1SW_D_IMM:
2306 case AArch64::LD1D_IMM:
2307
2308 case AArch64::LD2B_IMM:
2309 case AArch64::LD2H_IMM:
2310 case AArch64::LD2W_IMM:
2311 case AArch64::LD2D_IMM:
2312 case AArch64::LD3B_IMM:
2313 case AArch64::LD3H_IMM:
2314 case AArch64::LD3W_IMM:
2315 case AArch64::LD3D_IMM:
2316 case AArch64::LD4B_IMM:
2317 case AArch64::LD4H_IMM:
2318 case AArch64::LD4W_IMM:
2319 case AArch64::LD4D_IMM:
2320
2321 case AArch64::ST1B_IMM:
2322 case AArch64::ST1B_H_IMM:
2323 case AArch64::ST1B_S_IMM:
2324 case AArch64::ST1B_D_IMM:
2325 case AArch64::ST1H_IMM:
2326 case AArch64::ST1H_S_IMM:
2327 case AArch64::ST1H_D_IMM:
2328 case AArch64::ST1W_IMM:
2329 case AArch64::ST1W_D_IMM:
2330 case AArch64::ST1D_IMM:
2331
2332 case AArch64::ST2B_IMM:
2333 case AArch64::ST2H_IMM:
2334 case AArch64::ST2W_IMM:
2335 case AArch64::ST2D_IMM:
2336 case AArch64::ST3B_IMM:
2337 case AArch64::ST3H_IMM:
2338 case AArch64::ST3W_IMM:
2339 case AArch64::ST3D_IMM:
2340 case AArch64::ST4B_IMM:
2341 case AArch64::ST4H_IMM:
2342 case AArch64::ST4W_IMM:
2343 case AArch64::ST4D_IMM:
2344
2345 case AArch64::LD1RB_IMM:
2346 case AArch64::LD1RB_H_IMM:
2347 case AArch64::LD1RB_S_IMM:
2348 case AArch64::LD1RB_D_IMM:
2349 case AArch64::LD1RSB_H_IMM:
2350 case AArch64::LD1RSB_S_IMM:
2351 case AArch64::LD1RSB_D_IMM:
2352 case AArch64::LD1RH_IMM:
2353 case AArch64::LD1RH_S_IMM:
2354 case AArch64::LD1RH_D_IMM:
2355 case AArch64::LD1RSH_S_IMM:
2356 case AArch64::LD1RSH_D_IMM:
2357 case AArch64::LD1RW_IMM:
2358 case AArch64::LD1RW_D_IMM:
2359 case AArch64::LD1RSW_IMM:
2360 case AArch64::LD1RD_IMM:
2361
2362 case AArch64::LDNT1B_ZRI:
2363 case AArch64::LDNT1H_ZRI:
2364 case AArch64::LDNT1W_ZRI:
2365 case AArch64::LDNT1D_ZRI:
2366 case AArch64::STNT1B_ZRI:
2367 case AArch64::STNT1H_ZRI:
2368 case AArch64::STNT1W_ZRI:
2369 case AArch64::STNT1D_ZRI:
2370
2371 case AArch64::LDNF1B_IMM:
2372 case AArch64::LDNF1B_H_IMM:
2373 case AArch64::LDNF1B_S_IMM:
2374 case AArch64::LDNF1B_D_IMM:
2375 case AArch64::LDNF1SB_H_IMM:
2376 case AArch64::LDNF1SB_S_IMM:
2377 case AArch64::LDNF1SB_D_IMM:
2378 case AArch64::LDNF1H_IMM:
2379 case AArch64::LDNF1H_S_IMM:
2380 case AArch64::LDNF1H_D_IMM:
2381 case AArch64::LDNF1SH_S_IMM:
2382 case AArch64::LDNF1SH_D_IMM:
2383 case AArch64::LDNF1W_IMM:
2384 case AArch64::LDNF1W_D_IMM:
2385 case AArch64::LDNF1SW_D_IMM:
2386 case AArch64::LDNF1D_IMM:
2387 return 3;
2388 case AArch64::ADDG:
2389 case AArch64::STGi:
2390 case AArch64::LDR_PXI:
2391 case AArch64::STR_PXI:
2392 return 2;
2393 }
2394}
2395
2397 switch (MI.getOpcode()) {
2398 default:
2399 return false;
2400 // Scaled instructions.
2401 case AArch64::STRSui:
2402 case AArch64::STRDui:
2403 case AArch64::STRQui:
2404 case AArch64::STRXui:
2405 case AArch64::STRWui:
2406 case AArch64::LDRSui:
2407 case AArch64::LDRDui:
2408 case AArch64::LDRQui:
2409 case AArch64::LDRXui:
2410 case AArch64::LDRWui:
2411 case AArch64::LDRSWui:
2412 // Unscaled instructions.
2413 case AArch64::STURSi:
2414 case AArch64::STRSpre:
2415 case AArch64::STURDi:
2416 case AArch64::STRDpre:
2417 case AArch64::STURQi:
2418 case AArch64::STRQpre:
2419 case AArch64::STURWi:
2420 case AArch64::STRWpre:
2421 case AArch64::STURXi:
2422 case AArch64::STRXpre:
2423 case AArch64::LDURSi:
2424 case AArch64::LDRSpre:
2425 case AArch64::LDURDi:
2426 case AArch64::LDRDpre:
2427 case AArch64::LDURQi:
2428 case AArch64::LDRQpre:
2429 case AArch64::LDURWi:
2430 case AArch64::LDRWpre:
2431 case AArch64::LDURXi:
2432 case AArch64::LDRXpre:
2433 case AArch64::LDURSWi:
2434 return true;
2435 }
2436}
2437
2439 switch (Opc) {
2440 default:
2441 llvm_unreachable("Opcode has no flag setting equivalent!");
2442 // 32-bit cases:
2443 case AArch64::ADDWri:
2444 return AArch64::ADDSWri;
2445 case AArch64::ADDWrr:
2446 return AArch64::ADDSWrr;
2447 case AArch64::ADDWrs:
2448 return AArch64::ADDSWrs;
2449 case AArch64::ADDWrx:
2450 return AArch64::ADDSWrx;
2451 case AArch64::ANDWri:
2452 return AArch64::ANDSWri;
2453 case AArch64::ANDWrr:
2454 return AArch64::ANDSWrr;
2455 case AArch64::ANDWrs:
2456 return AArch64::ANDSWrs;
2457 case AArch64::BICWrr:
2458 return AArch64::BICSWrr;
2459 case AArch64::BICWrs:
2460 return AArch64::BICSWrs;
2461 case AArch64::SUBWri:
2462 return AArch64::SUBSWri;
2463 case AArch64::SUBWrr:
2464 return AArch64::SUBSWrr;
2465 case AArch64::SUBWrs:
2466 return AArch64::SUBSWrs;
2467 case AArch64::SUBWrx:
2468 return AArch64::SUBSWrx;
2469 // 64-bit cases:
2470 case AArch64::ADDXri:
2471 return AArch64::ADDSXri;
2472 case AArch64::ADDXrr:
2473 return AArch64::ADDSXrr;
2474 case AArch64::ADDXrs:
2475 return AArch64::ADDSXrs;
2476 case AArch64::ADDXrx:
2477 return AArch64::ADDSXrx;
2478 case AArch64::ANDXri:
2479 return AArch64::ANDSXri;
2480 case AArch64::ANDXrr:
2481 return AArch64::ANDSXrr;
2482 case AArch64::ANDXrs:
2483 return AArch64::ANDSXrs;
2484 case AArch64::BICXrr:
2485 return AArch64::BICSXrr;
2486 case AArch64::BICXrs:
2487 return AArch64::BICSXrs;
2488 case AArch64::SUBXri:
2489 return AArch64::SUBSXri;
2490 case AArch64::SUBXrr:
2491 return AArch64::SUBSXrr;
2492 case AArch64::SUBXrs:
2493 return AArch64::SUBSXrs;
2494 case AArch64::SUBXrx:
2495 return AArch64::SUBSXrx;
2496 // SVE instructions:
2497 case AArch64::AND_PPzPP:
2498 return AArch64::ANDS_PPzPP;
2499 case AArch64::BIC_PPzPP:
2500 return AArch64::BICS_PPzPP;
2501 case AArch64::EOR_PPzPP:
2502 return AArch64::EORS_PPzPP;
2503 case AArch64::NAND_PPzPP:
2504 return AArch64::NANDS_PPzPP;
2505 case AArch64::NOR_PPzPP:
2506 return AArch64::NORS_PPzPP;
2507 case AArch64::ORN_PPzPP:
2508 return AArch64::ORNS_PPzPP;
2509 case AArch64::ORR_PPzPP:
2510 return AArch64::ORRS_PPzPP;
2511 case AArch64::BRKA_PPzP:
2512 return AArch64::BRKAS_PPzP;
2513 case AArch64::BRKPA_PPzPP:
2514 return AArch64::BRKPAS_PPzPP;
2515 case AArch64::BRKB_PPzP:
2516 return AArch64::BRKBS_PPzP;
2517 case AArch64::BRKPB_PPzPP:
2518 return AArch64::BRKPBS_PPzPP;
2519 case AArch64::BRKN_PPzP:
2520 return AArch64::BRKNS_PPzP;
2521 case AArch64::RDFFR_PPz:
2522 return AArch64::RDFFRS_PPz;
2523 case AArch64::PTRUE_B:
2524 return AArch64::PTRUES_B;
2525 }
2526}
2527
2528// Is this a candidate for ld/st merging or pairing? For example, we don't
2529// touch volatiles or load/stores that have a hint to avoid pair formation.
2531
2532 bool IsPreLdSt = isPreLdSt(MI);
2533
2534 // If this is a volatile load/store, don't mess with it.
2535 if (MI.hasOrderedMemoryRef())
2536 return false;
2537
2538 // Make sure this is a reg/fi+imm (as opposed to an address reloc).
2539 // For Pre-inc LD/ST, the operand is shifted by one.
2540 assert((MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
2541 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
2542 "Expected a reg or frame index operand.");
2543
2544 // For Pre-indexed addressing quadword instructions, the third operand is the
2545 // immediate value.
2546 bool IsImmPreLdSt = IsPreLdSt && MI.getOperand(3).isImm();
2547
2548 if (!MI.getOperand(2).isImm() && !IsImmPreLdSt)
2549 return false;
2550
2551 // Can't merge/pair if the instruction modifies the base register.
2552 // e.g., ldr x0, [x0]
2553 // This case will never occur with an FI base.
2554 // However, if the instruction is an LDR/STR<S,D,Q,W,X>pre, it can be merged.
2555 // For example:
2556 // ldr q0, [x11, #32]!
2557 // ldr q1, [x11, #16]
2558 // to
2559 // ldp q0, q1, [x11, #32]!
2560 if (MI.getOperand(1).isReg() && !IsPreLdSt) {
2561 Register BaseReg = MI.getOperand(1).getReg();
2563 if (MI.modifiesRegister(BaseReg, TRI))
2564 return false;
2565 }
2566
2567 // Check if this load/store has a hint to avoid pair formation.
2568 // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
2570 return false;
2571
2572 // Do not pair any callee-save store/reload instructions in the
2573 // prologue/epilogue if the CFI information encoded the operations as separate
2574 // instructions, as that will cause the size of the actual prologue to mismatch
2575 // with the prologue size recorded in the Windows CFI.
2576 const MCAsmInfo *MAI = MI.getMF()->getTarget().getMCAsmInfo();
2577 bool NeedsWinCFI = MAI->usesWindowsCFI() &&
2578 MI.getMF()->getFunction().needsUnwindTableEntry();
2579 if (NeedsWinCFI && (MI.getFlag(MachineInstr::FrameSetup) ||
2581 return false;
2582
2583 // On some CPUs quad load/store pairs are slower than two single load/stores.
2584 if (Subtarget.isPaired128Slow()) {
2585 switch (MI.getOpcode()) {
2586 default:
2587 break;
2588 case AArch64::LDURQi:
2589 case AArch64::STURQi:
2590 case AArch64::LDRQui:
2591 case AArch64::STRQui:
2592 return false;
2593 }
2594 }
2595
2596 return true;
2597}
2598
2601 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2602 const TargetRegisterInfo *TRI) const {
2603 if (!LdSt.mayLoadOrStore())
2604 return false;
2605
2606 const MachineOperand *BaseOp;
2607 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, OffsetIsScalable,
2608 Width, TRI))
2609 return false;
2610 BaseOps.push_back(BaseOp);
2611 return true;
2612}
2613
2614std::optional<ExtAddrMode>
2616 const TargetRegisterInfo *TRI) const {
2617 const MachineOperand *Base; // Filled with the base operand of MI.
2618 int64_t Offset; // Filled with the offset of MI.
2619 bool OffsetIsScalable;
2620 if (!getMemOperandWithOffset(MemI, Base, Offset, OffsetIsScalable, TRI))
2621 return std::nullopt;
2622
2623 if (!Base->isReg())
2624 return std::nullopt;
2625 ExtAddrMode AM;
2626 AM.BaseReg = Base->getReg();
2627 AM.Displacement = Offset;
2628 AM.ScaledReg = 0;
2629 AM.Scale = 0;
2630 return AM;
2631}
2632
2634 const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
2635 bool &OffsetIsScalable, unsigned &Width,
2636 const TargetRegisterInfo *TRI) const {
2637 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2638 // Handle only loads/stores with base register followed by immediate offset.
2639 if (LdSt.getNumExplicitOperands() == 3) {
2640 // Non-paired instruction (e.g., ldr x1, [x0, #8]).
2641 if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||
2642 !LdSt.getOperand(2).isImm())
2643 return false;
2644 } else if (LdSt.getNumExplicitOperands() == 4) {
2645 // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
2646 if (!LdSt.getOperand(1).isReg() ||
2647 (!LdSt.getOperand(2).isReg() && !LdSt.getOperand(2).isFI()) ||
2648 !LdSt.getOperand(3).isImm())
2649 return false;
2650 } else
2651 return false;
2652
2653 // Get the scaling factor for the instruction and set the width for the
2654 // instruction.
2655 TypeSize Scale(0U, false);
2656 int64_t Dummy1, Dummy2;
2657
2658 // If this returns false, then it's an instruction we don't want to handle.
2659 if (!getMemOpInfo(LdSt.getOpcode(), Scale, Width, Dummy1, Dummy2))
2660 return false;
2661
2662 // Compute the offset. Offset is calculated as the immediate operand
2663 // multiplied by the scaling factor. Unscaled instructions have scaling factor
2664 // set to 1.
2665 if (LdSt.getNumExplicitOperands() == 3) {
2666 BaseOp = &LdSt.getOperand(1);
2667 Offset = LdSt.getOperand(2).getImm() * Scale.getKnownMinValue();
2668 } else {
2669 assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
2670 BaseOp = &LdSt.getOperand(2);
2671 Offset = LdSt.getOperand(3).getImm() * Scale.getKnownMinValue();
2672 }
2673 OffsetIsScalable = Scale.isScalable();
2674
2675 if (!BaseOp->isReg() && !BaseOp->isFI())
2676 return false;
2677
2678 return true;
2679}
2680
2683 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
2684 MachineOperand &OfsOp = LdSt.getOperand(LdSt.getNumExplicitOperands() - 1);
2685 assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
2686 return OfsOp;
2687}
2688
2689bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
2690 unsigned &Width, int64_t &MinOffset,
2691 int64_t &MaxOffset) {
2692 const unsigned SVEMaxBytesPerVector = AArch64::SVEMaxBitsPerVector / 8;
2693 switch (Opcode) {
2694 // Not a memory operation or something we want to handle.
2695 default:
2696 Scale = TypeSize::Fixed(0);
2697 Width = 0;
2698 MinOffset = MaxOffset = 0;
2699 return false;
2700 case AArch64::STRWpost:
2701 case AArch64::LDRWpost:
2702 Width = 32;
2703 Scale = TypeSize::Fixed(4);
2704 MinOffset = -256;
2705 MaxOffset = 255;
2706 break;
2707 case AArch64::LDURQi:
2708 case AArch64::STURQi:
2709 Width = 16;
2710 Scale = TypeSize::Fixed(1);
2711 MinOffset = -256;
2712 MaxOffset = 255;
2713 break;
2714 case AArch64::PRFUMi:
2715 case AArch64::LDURXi:
2716 case AArch64::LDURDi:
2717 case AArch64::STURXi:
2718 case AArch64::STURDi:
2719 Width = 8;
2720 Scale = TypeSize::Fixed(1);
2721 MinOffset = -256;
2722 MaxOffset = 255;
2723 break;
2724 case AArch64::LDURWi:
2725 case AArch64::LDURSi:
2726 case AArch64::LDURSWi:
2727 case AArch64::STURWi:
2728 case AArch64::STURSi:
2729 Width = 4;
2730 Scale = TypeSize::Fixed(1);
2731 MinOffset = -256;
2732 MaxOffset = 255;
2733 break;
2734 case AArch64::LDURHi:
2735 case AArch64::LDURHHi:
2736 case AArch64::LDURSHXi:
2737 case AArch64::LDURSHWi:
2738 case AArch64::STURHi:
2739 case AArch64::STURHHi:
2740 Width = 2;
2741 Scale = TypeSize::Fixed(1);
2742 MinOffset = -256;
2743 MaxOffset = 255;
2744 break;
2745 case AArch64::LDURBi:
2746 case AArch64::LDURBBi:
2747 case AArch64::LDURSBXi:
2748 case AArch64::LDURSBWi:
2749 case AArch64::STURBi:
2750 case AArch64::STURBBi:
2751 Width = 1;
2752 Scale = TypeSize::Fixed(1);
2753 MinOffset = -256;
2754 MaxOffset = 255;
2755 break;
2756 case AArch64::LDPQi:
2757 case AArch64::LDNPQi:
2758 case AArch64::STPQi:
2759 case AArch64::STNPQi:
2760 Scale = TypeSize::Fixed(16);
2761 Width = 32;
2762 MinOffset = -64;
2763 MaxOffset = 63;
2764 break;
2765 case AArch64::LDRQui:
2766 case AArch64::STRQui:
2767 Scale = TypeSize::Fixed(16);
2768 Width = 16;
2769 MinOffset = 0;
2770 MaxOffset = 4095;
2771 break;
2772 case AArch64::LDPXi:
2773 case AArch64::LDPDi:
2774 case AArch64::LDNPXi:
2775 case AArch64::LDNPDi:
2776 case AArch64::STPXi:
2777 case AArch64::STPDi:
2778 case AArch64::STNPXi:
2779 case AArch64::STNPDi:
2780 Scale = TypeSize::Fixed(8);
2781 Width = 16;
2782 MinOffset = -64;
2783 MaxOffset = 63;
2784 break;
2785 case AArch64::PRFMui:
2786 case AArch64::LDRXui:
2787 case AArch64::LDRDui:
2788 case AArch64::STRXui:
2789 case AArch64::STRDui:
2790 Scale = TypeSize::Fixed(8);
2791 Width = 8;
2792 MinOffset = 0;
2793 MaxOffset = 4095;
2794 break;
2795 case AArch64::StoreSwiftAsyncContext:
2796 // Store is an STRXui, but there might be an ADDXri in the expansion too.
2797 Scale = TypeSize::Fixed(1);
2798 Width = 8;
2799 MinOffset = 0;
2800 MaxOffset = 4095;
2801 break;
2802 case AArch64::LDPWi:
2803 case AArch64::LDPSi:
2804 case AArch64::LDNPWi:
2805 case AArch64::LDNPSi:
2806 case AArch64::STPWi:
2807 case AArch64::STPSi:
2808 case AArch64::STNPWi:
2809 case AArch64::STNPSi:
2810 Scale = TypeSize::Fixed(4);
2811 Width = 8;
2812 MinOffset = -64;
2813 MaxOffset = 63;
2814 break;
2815 case AArch64::LDRWui:
2816 case AArch64::LDRSui:
2817 case AArch64::LDRSWui:
2818 case AArch64::STRWui:
2819 case AArch64::STRSui:
2820 Scale = TypeSize::Fixed(4);
2821 Width = 4;
2822 MinOffset = 0;
2823 MaxOffset = 4095;
2824 break;
2825 case AArch64::LDRHui:
2826 case AArch64::LDRHHui:
2827 case AArch64::LDRSHWui:
2828 case AArch64::LDRSHXui:
2829 case AArch64::STRHui:
2830 case AArch64::STRHHui:
2831 Scale = TypeSize::Fixed(2);
2832 Width = 2;
2833 MinOffset = 0;
2834 MaxOffset = 4095;
2835 break;
2836 case AArch64::LDRBui:
2837 case AArch64::LDRBBui:
2838 case AArch64::LDRSBWui:
2839 case AArch64::LDRSBXui:
2840 case AArch64::STRBui:
2841 case AArch64::STRBBui:
2842 Scale = TypeSize::Fixed(1);
2843 Width = 1;
2844 MinOffset = 0;
2845 MaxOffset = 4095;
2846 break;
2847 case AArch64::STPXpre:
2848 case AArch64::LDPXpost:
2849 case AArch64::STPDpre:
2850 case AArch64::LDPDpost:
2851 Scale = TypeSize::Fixed(8);
2852 Width = 8;
2853 MinOffset = -512;
2854 MaxOffset = 504;
2855 break;
2856 case AArch64::STPQpre:
2857 case AArch64::LDPQpost:
2858 Scale = TypeSize::Fixed(16);
2859 Width = 16;
2860 MinOffset = -1024;
2861 MaxOffset = 1008;
2862 break;
2863 case AArch64::STRXpre:
2864 case AArch64::STRDpre:
2865 case AArch64::LDRXpost:
2866 case AArch64::LDRDpost:
2867 Scale = TypeSize::Fixed(1);
2868 Width = 8;
2869 MinOffset = -256;
2870 MaxOffset = 255;
2871 break;
2872 case AArch64::STRQpre:
2873 case AArch64::LDRQpost:
2874 Scale = TypeSize::Fixed(1);
2875 Width = 16;
2876 MinOffset = -256;
2877 MaxOffset = 255;
2878 break;
2879 case AArch64::ADDG:
2880 Scale = TypeSize::Fixed(16);
2881 Width = 0;
2882 MinOffset = 0;
2883 MaxOffset = 63;
2884 break;
2885 case AArch64::TAGPstack:
2886 Scale = TypeSize::Fixed(16);
2887 Width = 0;
2888 // TAGP with a negative offset turns into SUBP, which has a maximum offset
2889 // of 63 (not 64!).
2890 MinOffset = -63;
2891 MaxOffset = 63;
2892 break;
2893 case AArch64::LDG:
2894 case AArch64::STGi:
2895 case AArch64::STZGi:
2896 Scale = TypeSize::Fixed(16);
2897 Width = 16;
2898 MinOffset = -256;
2899 MaxOffset = 255;
2900 break;
2901 case AArch64::STR_ZZZZXI:
2902 case AArch64::LDR_ZZZZXI:
2903 Scale = TypeSize::Scalable(16);
2904 Width = SVEMaxBytesPerVector * 4;
2905 MinOffset = -256;
2906 MaxOffset = 252;
2907 break;
2908 case AArch64::STR_ZZZXI:
2909 case AArch64::LDR_ZZZXI:
2910 Scale = TypeSize::Scalable(16);
2911 Width = SVEMaxBytesPerVector * 3;
2912 MinOffset = -256;
2913 MaxOffset = 253;
2914 break;
2915 case AArch64::STR_ZZXI:
2916 case AArch64::LDR_ZZXI:
2917 Scale = TypeSize::Scalable(16);
2918 Width = SVEMaxBytesPerVector * 2;
2919 MinOffset = -256;
2920 MaxOffset = 254;
2921 break;
2922 case AArch64::LDR_PXI:
2923 case AArch64::STR_PXI:
2924 Scale = TypeSize::Scalable(2);
2925 Width = SVEMaxBytesPerVector / 8;
2926 MinOffset = -256;
2927 MaxOffset = 255;
2928 break;
2929 case AArch64::LDR_ZXI:
2930 case AArch64::STR_ZXI:
2931 Scale = TypeSize::Scalable(16);
2932 Width = SVEMaxBytesPerVector;
2933 MinOffset = -256;
2934 MaxOffset = 255;
2935 break;
2936 case AArch64::LD1B_IMM:
2937 case AArch64::LD1H_IMM:
2938 case AArch64::LD1W_IMM:
2939 case AArch64::LD1D_IMM:
2940 case AArch64::LDNT1B_ZRI:
2941 case AArch64::LDNT1H_ZRI:
2942 case AArch64::LDNT1W_ZRI:
2943 case AArch64::LDNT1D_ZRI:
2944 case AArch64::ST1B_IMM:
2945 case AArch64::ST1H_IMM:
2946 case AArch64::ST1W_IMM:
2947 case AArch64::ST1D_IMM:
2948 case AArch64::STNT1B_ZRI:
2949 case AArch64::STNT1H_ZRI:
2950 case AArch64::STNT1W_ZRI:
2951 case AArch64::STNT1D_ZRI:
2952 case AArch64::LDNF1B_IMM:
2953 case AArch64::LDNF1H_IMM:
2954 case AArch64::LDNF1W_IMM:
2955 case AArch64::LDNF1D_IMM:
2956 // A full vectors worth of data
2957 // Width = mbytes * elements
2958 Scale = TypeSize::Scalable(16);
2959 Width = SVEMaxBytesPerVector;
2960 MinOffset = -8;
2961 MaxOffset = 7;
2962 break;
2963 case AArch64::LD2B_IMM:
2964 case AArch64::LD2H_IMM:
2965 case AArch64::LD2W_IMM:
2966 case AArch64::LD2D_IMM:
2967 case AArch64::ST2B_IMM:
2968 case AArch64::ST2H_IMM:
2969 case AArch64::ST2W_IMM:
2970 case AArch64::ST2D_IMM:
2971 Scale = TypeSize::Scalable(32);
2972 Width = SVEMaxBytesPerVector * 2;
2973 MinOffset = -8;
2974 MaxOffset = 7;
2975 break;
2976 case AArch64::LD3B_IMM:
2977 case AArch64::LD3H_IMM:
2978 case AArch64::LD3W_IMM:
2979 case AArch64::LD3D_IMM:
2980 case AArch64::ST3B_IMM:
2981 case AArch64::ST3H_IMM:
2982 case AArch64::ST3W_IMM:
2983 case AArch64::ST3D_IMM:
2984 Scale = TypeSize::Scalable(48);
2985 Width = SVEMaxBytesPerVector * 3;
2986 MinOffset = -8;
2987 MaxOffset = 7;
2988 break;
2989 case AArch64::LD4B_IMM:
2990 case AArch64::LD4H_IMM:
2991 case AArch64::LD4W_IMM:
2992 case AArch64::LD4D_IMM:
2993 case AArch64::ST4B_IMM:
2994 case AArch64::ST4H_IMM:
2995 case AArch64::ST4W_IMM:
2996 case AArch64::ST4D_IMM:
2997 Scale = TypeSize::Scalable(64);
2998 Width = SVEMaxBytesPerVector * 4;
2999 MinOffset = -8;
3000 MaxOffset = 7;
3001 break;
3002 case AArch64::LD1B_H_IMM:
3003 case AArch64::LD1SB_H_IMM:
3004 case AArch64::LD1H_S_IMM:
3005 case AArch64::LD1SH_S_IMM:
3006 case AArch64::LD1W_D_IMM:
3007 case AArch64::LD1SW_D_IMM:
3008 case AArch64::ST1B_H_IMM:
3009 case AArch64::ST1H_S_IMM:
3010 case AArch64::ST1W_D_IMM:
3011 case AArch64::LDNF1B_H_IMM:
3012 case AArch64::LDNF1SB_H_IMM:
3013 case AArch64::LDNF1H_S_IMM:
3014 case AArch64::LDNF1SH_S_IMM:
3015 case AArch64::LDNF1W_D_IMM:
3016 case AArch64::LDNF1SW_D_IMM:
3017 // A half vector worth of data
3018 // Width = mbytes * elements
3019 Scale = TypeSize::Scalable(8);
3020 Width = SVEMaxBytesPerVector / 2;
3021 MinOffset = -8;
3022 MaxOffset = 7;
3023 break;
3024 case AArch64::LD1B_S_IMM:
3025 case AArch64::LD1SB_S_IMM:
3026 case AArch64::LD1H_D_IMM:
3027 case AArch64::LD1SH_D_IMM:
3028 case AArch64::ST1B_S_IMM:
3029 case AArch64::ST1H_D_IMM:
3030 case AArch64::LDNF1B_S_IMM:
3031 case AArch64::LDNF1SB_S_IMM:
3032 case AArch64::LDNF1H_D_IMM:
3033 case AArch64::LDNF1SH_D_IMM:
3034 // A quarter vector worth of data
3035 // Width = mbytes * elements
3036 Scale = TypeSize::Scalable(4);
3037 Width = SVEMaxBytesPerVector / 4;
3038 MinOffset = -8;
3039 MaxOffset = 7;
3040 break;
3041 case AArch64::LD1B_D_IMM:
3042 case AArch64::LD1SB_D_IMM:
3043 case AArch64::ST1B_D_IMM:
3044 case AArch64::LDNF1B_D_IMM:
3045 case AArch64::LDNF1SB_D_IMM:
3046 // A eighth vector worth of data
3047 // Width = mbytes * elements
3048 Scale = TypeSize::Scalable(2);
3049 Width = SVEMaxBytesPerVector / 8;
3050 MinOffset = -8;
3051 MaxOffset = 7;
3052 break;
3053 case AArch64::ST2Gi:
3054 case AArch64::STZ2Gi:
3055 Scale = TypeSize::Fixed(16);
3056 Width = 32;
3057 MinOffset = -256;
3058 MaxOffset = 255;
3059 break;
3060 case AArch64::STGPi:
3061 Scale = TypeSize::Fixed(16);
3062 Width = 16;
3063 MinOffset = -64;
3064 MaxOffset = 63;
3065 break;
3066 case AArch64::LD1RB_IMM:
3067 case AArch64::LD1RB_H_IMM:
3068 case AArch64::LD1RB_S_IMM:
3069 case AArch64::LD1RB_D_IMM:
3070 case AArch64::LD1RSB_H_IMM:
3071 case AArch64::LD1RSB_S_IMM:
3072 case AArch64::LD1RSB_D_IMM:
3073 Scale = TypeSize::Fixed(1);
3074 Width = 1;
3075 MinOffset = 0;
3076 MaxOffset = 63;
3077 break;
3078 case AArch64::LD1RH_IMM:
3079 case AArch64::LD1RH_S_IMM:
3080 case AArch64::LD1RH_D_IMM:
3081 case AArch64::LD1RSH_S_IMM:
3082 case AArch64::LD1RSH_D_IMM:
3083 Scale = TypeSize::Fixed(2);
3084 Width = 2;
3085 MinOffset = 0;
3086 MaxOffset = 63;
3087 break;
3088 case AArch64::LD1RW_IMM:
3089 case AArch64::LD1RW_D_IMM:
3090 case AArch64::LD1RSW_IMM:
3091 Scale = TypeSize::Fixed(4);
3092 Width = 4;
3093 MinOffset = 0;
3094 MaxOffset = 63;
3095 break;
3096 case AArch64::LD1RD_IMM:
3097 Scale = TypeSize::Fixed(8);
3098 Width = 8;
3099 MinOffset = 0;
3100 MaxOffset = 63;
3101 break;
3102 }
3103
3104 return true;
3105}
3106
3107// Scaling factor for unscaled load or store.
3109 switch (Opc) {
3110 default:
3111 llvm_unreachable("Opcode has unknown scale!");
3112 case AArch64::LDRBBui:
3113 case AArch64::LDURBBi:
3114 case AArch64::LDRSBWui:
3115 case AArch64::LDURSBWi:
3116 case AArch64::STRBBui:
3117 case AArch64::STURBBi:
3118 return 1;
3119 case AArch64::LDRHHui:
3120 case AArch64::LDURHHi:
3121 case AArch64::LDRSHWui:
3122 case AArch64::LDURSHWi:
3123 case AArch64::STRHHui:
3124 case AArch64::STURHHi:
3125 return 2;
3126 case AArch64::LDRSui:
3127 case AArch64::LDURSi:
3128 case AArch64::LDRSpre:
3129 case AArch64::LDRSWui:
3130 case AArch64::LDURSWi:
3131 case AArch64::LDRWpre:
3132 case AArch64::LDRWui:
3133 case AArch64::LDURWi:
3134 case AArch64::STRSui:
3135 case AArch64::STURSi:
3136 case AArch64::STRSpre:
3137 case AArch64::STRWui:
3138 case AArch64::STURWi:
3139 case AArch64::STRWpre:
3140 case AArch64::LDPSi:
3141 case AArch64::LDPSWi:
3142 case AArch64::LDPWi:
3143 case AArch64::STPSi:
3144 case AArch64::STPWi:
3145 return 4;
3146 case AArch64::LDRDui:
3147 case AArch64::LDURDi:
3148 case AArch64::LDRDpre:
3149 case AArch64::LDRXui:
3150 case AArch64::LDURXi:
3151 case AArch64::LDRXpre:
3152 case AArch64::STRDui:
3153 case AArch64::STURDi:
3154 case AArch64::STRDpre:
3155 case AArch64::STRXui:
3156 case AArch64::STURXi:
3157 case AArch64::STRXpre:
3158 case AArch64::LDPDi:
3159 case AArch64::LDPXi:
3160 case AArch64::STPDi:
3161 case AArch64::STPXi:
3162 return 8;
3163 case AArch64::LDRQui:
3164 case AArch64::LDURQi:
3165 case AArch64::STRQui:
3166 case AArch64::STURQi:
3167 case AArch64::STRQpre:
3168 case AArch64::LDPQi:
3169 case AArch64::LDRQpre:
3170 case AArch64::STPQi:
3171 case AArch64::STGi:
3172 case AArch64::STZGi:
3173 case AArch64::ST2Gi:
3174 case AArch64::STZ2Gi:
3175 case AArch64::STGPi:
3176 return 16;
3177 }
3178}
3179
3181 switch (MI.getOpcode()) {
3182 default:
3183 return false;
3184 case AArch64::LDRWpre:
3185 case AArch64::LDRXpre:
3186 case AArch64::LDRSpre:
3187 case AArch64::LDRDpre:
3188 case AArch64::LDRQpre:
3189 return true;
3190 }
3191}
3192
3194 switch (MI.getOpcode()) {
3195 default:
3196 return false;
3197 case AArch64::STRWpre:
3198 case AArch64::STRXpre:
3199 case AArch64::STRSpre:
3200 case AArch64::STRDpre:
3201 case AArch64::STRQpre:
3202 return true;
3203 }
3204}
3205
3207 return isPreLd(MI) || isPreSt(MI);
3208}
3209
3211 switch (MI.getOpcode()) {
3212 default:
3213 return false;
3214 case AArch64::LDPSi:
3215 case AArch64::LDPSWi:
3216 case AArch64::LDPDi:
3217 case AArch64::LDPQi:
3218 case AArch64::LDPWi:
3219 case AArch64::LDPXi:
3220 case AArch64::STPSi:
3221 case AArch64::STPDi:
3222 case AArch64::STPQi:
3223 case AArch64::STPWi:
3224 case AArch64::STPXi:
3225 case AArch64::STGPi:
3226 return true;
3227 }
3228}
3229
3231 unsigned Idx =
3233 : 1;
3234 return MI.getOperand(Idx);
3235}
3236
3237const MachineOperand &
3239 unsigned Idx =
3241 : 2;
3242 return MI.getOperand(Idx);
3243}
3244
3246 Register Reg) {
3247 if (MI.getParent() == nullptr)
3248 return nullptr;
3249 const MachineFunction *MF = MI.getParent()->getParent();
3250 return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr;
3251}
3252
3254 auto IsQFPR = [&](const MachineOperand &Op) {
3255 if (!Op.isReg())
3256 return false;
3257 auto Reg = Op.getReg();
3258 if (Reg.isPhysical())
3259 return AArch64::FPR128RegClass.contains(Reg);
3260 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
3261 return TRC == &AArch64::FPR128RegClass ||
3262 TRC == &AArch64::FPR128_loRegClass;
3263 };
3264 return llvm::any_of(MI.operands(), IsQFPR);
3265}
3266
3268 auto IsFPR = [&](const MachineOperand &Op) {
3269 if (!Op.isReg())
3270 return false;
3271 auto Reg = Op.getReg();
3272 if (Reg.isPhysical())
3273 return AArch64::FPR128RegClass.contains(Reg) ||
3274 AArch64::FPR64RegClass.contains(Reg) ||
3275 AArch64::FPR32RegClass.contains(Reg) ||
3276 AArch64::FPR16RegClass.contains(Reg) ||
3277 AArch64::FPR8RegClass.contains(Reg);
3278
3279 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
3280 return TRC == &AArch64::FPR128RegClass ||
3281 TRC == &AArch64::FPR128_loRegClass ||
3282 TRC == &AArch64::FPR64RegClass ||
3283 TRC == &AArch64::FPR64_loRegClass ||
3284 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
3285 TRC == &AArch64::FPR8RegClass;
3286 };
3287 return llvm::any_of(MI.operands(), IsFPR);
3288}
3289
3290// Scale the unscaled offsets. Returns false if the unscaled offset can't be
3291// scaled.
3292static bool scaleOffset(unsigned Opc, int64_t &Offset) {
3293 int Scale = AArch64InstrInfo::getMemScale(Opc);
3294
3295 // If the byte-offset isn't a multiple of the stride, we can't scale this
3296 // offset.
3297 if (Offset % Scale != 0)
3298 return false;
3299
3300 // Convert the byte-offset used by unscaled into an "element" offset used
3301 // by the scaled pair load/store instructions.
3302 Offset /= Scale;
3303 return true;
3304}
3305
3306static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
3307 if (FirstOpc == SecondOpc)
3308 return true;
3309 // We can also pair sign-ext and zero-ext instructions.
3310 switch (FirstOpc) {
3311 default:
3312 return false;
3313 case AArch64::LDRWui:
3314 case AArch64::LDURWi:
3315 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
3316 case AArch64::LDRSWui:
3317 case AArch64::LDURSWi:
3318 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
3319 }
3320 // These instructions can't be paired based on their opcodes.
3321 return false;
3322}
3323
3324static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
3325 int64_t Offset1, unsigned Opcode1, int FI2,
3326 int64_t Offset2, unsigned Opcode2) {
3327 // Accesses through fixed stack object frame indices may access a different
3328 // fixed stack slot. Check that the object offsets + offsets match.
3329 if (MFI.isFixedObjectIndex(FI1) && MFI.isFixedObjectIndex(FI2)) {
3330 int64_t ObjectOffset1 = MFI.getObjectOffset(FI1);
3331 int64_t ObjectOffset2 = MFI.getObjectOffset(FI2);
3332 assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
3333 // Convert to scaled object offsets.
3334 int Scale1 = AArch64InstrInfo::getMemScale(Opcode1);
3335 if (ObjectOffset1 % Scale1 != 0)
3336 return false;
3337 ObjectOffset1 /= Scale1;
3338 int Scale2 = AArch64InstrInfo::getMemScale(Opcode2);
3339 if (ObjectOffset2 % Scale2 != 0)
3340 return false;
3341 ObjectOffset2 /= Scale2;
3342 ObjectOffset1 += Offset1;
3343 ObjectOffset2 += Offset2;
3344 return ObjectOffset1 + 1 == ObjectOffset2;
3345 }
3346
3347 return FI1 == FI2;
3348}
3349
3350/// Detect opportunities for ldp/stp formation.
3351///
3352/// Only called for LdSt for which getMemOperandWithOffset returns true.
3355 ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads,
3356 unsigned NumBytes) const {
3357 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
3358 const MachineOperand &BaseOp1 = *BaseOps1.front();
3359 const MachineOperand &BaseOp2 = *BaseOps2.front();
3360 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
3361 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
3362 if (BaseOp1.getType() != BaseOp2.getType())
3363 return false;
3364
3365 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
3366 "Only base registers and frame indices are supported.");
3367
3368 // Check for both base regs and base FI.
3369 if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
3370 return false;
3371
3372 // Only cluster up to a single pair.
3373 if (NumLoads > 2)
3374 return false;
3375
3376 if (!isPairableLdStInst(FirstLdSt) || !isPairableLdStInst(SecondLdSt))
3377 return false;
3378
3379 // Can we pair these instructions based on their opcodes?
3380 unsigned FirstOpc = FirstLdSt.getOpcode();
3381 unsigned SecondOpc = SecondLdSt.getOpcode();
3382 if (!canPairLdStOpc(FirstOpc, SecondOpc))
3383 return false;
3384
3385 // Can't merge volatiles or load/stores that have a hint to avoid pair
3386 // formation, for example.
3387 if (!isCandidateToMergeOrPair(FirstLdSt) ||
3388 !isCandidateToMergeOrPair(SecondLdSt))
3389 return false;
3390
3391 // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
3392 int64_t Offset1 = FirstLdSt.getOperand(2).getImm();
3393 if (hasUnscaledLdStOffset(FirstOpc) && !scaleOffset(FirstOpc, Offset1))
3394 return false;
3395
3396 int64_t Offset2 = SecondLdSt.getOperand(2).getImm();
3397 if (hasUnscaledLdStOffset(SecondOpc) && !scaleOffset(SecondOpc, Offset2))
3398 return false;
3399
3400 // Pairwise instructions have a 7-bit signed offset field.
3401 if (Offset1 > 63 || Offset1 < -64)
3402 return false;
3403
3404 // The caller should already have ordered First/SecondLdSt by offset.
3405 // Note: except for non-equal frame index bases
3406 if (BaseOp1.isFI()) {
3407 assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
3408 "Caller should have ordered offsets.");
3409
3410 const MachineFrameInfo &MFI =
3411 FirstLdSt.getParent()->getParent()->getFrameInfo();
3412 return shouldClusterFI(MFI, BaseOp1.getIndex(), Offset1, FirstOpc,
3413 BaseOp2.getIndex(), Offset2, SecondOpc);
3414 }
3415
3416 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
3417
3418 return Offset1 + 1 == Offset2;
3419}
3420
3422 unsigned Reg, unsigned SubIdx,
3423 unsigned State,
3424 const TargetRegisterInfo *TRI) {
3425 if (!SubIdx)
3426 return MIB.addReg(Reg, State);
3427
3429 return MIB.addReg(TRI->getSubReg(Reg, SubIdx), State);
3430 return MIB.addReg(Reg, State, SubIdx);
3431}
3432
3433static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
3434 unsigned NumRegs) {
3435 // We really want the positive remainder mod 32 here, that happens to be
3436 // easily obtainable with a mask.
3437 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
3438}
3439
3442 const DebugLoc &DL, MCRegister DestReg,
3443 MCRegister SrcReg, bool KillSrc,
3444 unsigned Opcode,
3445 ArrayRef<unsigned> Indices) const {
3446 assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
3448 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
3449 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
3450 unsigned NumRegs = Indices.size();
3451
3452 int SubReg = 0, End = NumRegs, Incr = 1;
3453 if (forwardCopyWillClobberTuple(DestEncoding, SrcEncoding, NumRegs)) {
3454 SubReg = NumRegs - 1;
3455 End = -1;
3456 Incr = -1;
3457 }
3458
3459 for (; SubReg != End; SubReg += Incr) {
3460 const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
3461 AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
3462 AddSubReg(MIB, SrcReg, Indices[SubReg], 0, TRI);
3463 AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
3464 }
3465}
3466
3469 DebugLoc DL, unsigned DestReg,
3470 unsigned SrcReg, bool KillSrc,
3471 unsigned Opcode, unsigned ZeroReg,
3472 llvm::ArrayRef<unsigned> Indices) const {
3474 unsigned NumRegs = Indices.size();
3475
3476#ifndef NDEBUG
3477 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
3478 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
3479 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
3480 "GPR reg sequences should not be able to overlap");
3481#endif
3482
3483 for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
3484 const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
3485 AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
3486 MIB.addReg(ZeroReg);
3487 AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
3488 MIB.addImm(0);
3489 }
3490}
3491
3494 const DebugLoc &DL, MCRegister DestReg,
3495 MCRegister SrcReg, bool KillSrc) const {
3496 if (AArch64::GPR32spRegClass.contains(DestReg) &&
3497 (AArch64::GPR32spRegClass.contains(SrcReg) || SrcReg == AArch64::WZR)) {
3499
3500 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
3501 // If either operand is WSP, expand to ADD #0.
3502 if (Subtarget.hasZeroCycleRegMove()) {
3503 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
3504 MCRegister DestRegX = TRI->getMatchingSuperReg(
3505 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
3506 MCRegister SrcRegX = TRI->getMatchingSuperReg(
3507 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
3508 // This instruction is reading and writing X registers. This may upset
3509 // the register scavenger and machine verifier, so we need to indicate
3510 // that we are reading an undefined value from SrcRegX, but a proper
3511 // value from SrcReg.
3512 BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestRegX)
3513 .addReg(SrcRegX, RegState::Undef)
3514 .addImm(0)
3516 .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
3517 } else {
3518 BuildMI(MBB, I, DL, get(AArch64::ADDWri), DestReg)
3519 .addReg(SrcReg, getKillRegState(KillSrc))
3520 .addImm(0)
3522 }
3523 } else if (SrcReg == AArch64::WZR && Subtarget.hasZeroCycleZeroingGP()) {
3524 BuildMI(MBB, I, DL, get(AArch64::MOVZWi), DestReg)
3525 .addImm(0)
3527 } else {
3528 if (Subtarget.hasZeroCycleRegMove()) {
3529 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
3530 MCRegister DestRegX = TRI->getMatchingSuperReg(
3531 DestReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
3532 MCRegister SrcRegX = TRI->getMatchingSuperReg(
3533 SrcReg, AArch64::sub_32, &AArch64::GPR64spRegClass);
3534 // This instruction is reading and writing X registers. This may upset
3535 // the register scavenger and machine verifier, so we need to indicate
3536 // that we are reading an undefined value from SrcRegX, but a proper
3537 // value from SrcReg.
3538 BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestRegX)
3539 .addReg(AArch64::XZR)
3540 .addReg(SrcRegX, RegState::Undef)
3541 .addReg(SrcReg, RegState::Implicit | getKillRegState(KillSrc));
3542 } else {
3543 // Otherwise, expand to ORR WZR.
3544 BuildMI(MBB, I, DL, get(AArch64::ORRWrr), DestReg)
3545 .addReg(AArch64::WZR)
3546 .addReg(SrcReg, getKillRegState(KillSrc));
3547 }
3548 }
3549 return;
3550 }
3551
3552 // Copy a Predicate register by ORRing with itself.
3553 if (AArch64::PPRRegClass.contains(DestReg) &&
3554 AArch64::PPRRegClass.contains(SrcReg)) {
3555 assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
3556 BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), DestReg)
3557 .addReg(SrcReg) // Pg
3558 .addReg(SrcReg)
3559 .addReg(SrcReg, getKillRegState(KillSrc));
3560 return;
3561 }
3562
3563 // Copy a Z register by ORRing with itself.
3564 if (AArch64::ZPRRegClass.contains(DestReg) &&
3565 AArch64::ZPRRegClass.contains(SrcReg)) {
3566 assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
3567 BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ), DestReg)
3568 .addReg(SrcReg)
3569 .addReg(SrcReg, getKillRegState(KillSrc));
3570 return;
3571 }
3572
3573 // Copy a Z register pair by copying the individual sub-registers.
3574 if (AArch64::ZPR2RegClass.contains(DestReg) &&
3575 AArch64::ZPR2RegClass.contains(SrcReg)) {
3576 assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
3577 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
3578 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
3579 Indices);
3580 return;
3581 }
3582
3583 // Copy a Z register triple by copying the individual sub-registers.
3584 if (AArch64::ZPR3RegClass.contains(DestReg) &&
3585 AArch64::ZPR3RegClass.contains(SrcReg)) {
3586 assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
3587 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
3588 AArch64::zsub2};
3589 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
3590 Indices);
3591 return;
3592 }
3593
3594 // Copy a Z register quad by copying the individual sub-registers.
3595 if (AArch64::ZPR4RegClass.contains(DestReg) &&
3596 AArch64::ZPR4RegClass.contains(SrcReg)) {
3597 assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
3598 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
3599 AArch64::zsub2, AArch64::zsub3};
3600 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
3601 Indices);
3602 return;
3603 }
3604
3605 if (AArch64::GPR64spRegClass.contains(DestReg) &&
3606 (AArch64::GPR64spRegClass.contains(SrcReg) || SrcReg == AArch64::XZR)) {
3607 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
3608 // If either operand is SP, expand to ADD #0.
3609 BuildMI(MBB, I, DL, get(AArch64::ADDXri), DestReg)
3610 .addReg(SrcReg, getKillRegState(KillSrc))
3611 .addImm(0)
3613 } else if (SrcReg == AArch64::XZR && Subtarget.hasZeroCycleZeroingGP()) {
3614 BuildMI(MBB, I, DL, get(AArch64::MOVZXi), DestReg)
3615 .addImm(0)
3617 } else {
3618 // Otherwise, expand to ORR XZR.
3619 BuildMI(MBB, I, DL, get(AArch64::ORRXrr), DestReg)
3620 .addReg(AArch64::XZR)
3621 .addReg(SrcReg, getKillRegState(KillSrc));
3622 }
3623 return;
3624 }
3625
3626 // Copy a DDDD register quad by copying the individual sub-registers.
3627 if (AArch64::DDDDRegClass.contains(DestReg) &&
3628 AArch64::DDDDRegClass.contains(SrcReg)) {
3629 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
3630 AArch64::dsub2, AArch64::dsub3};
3631 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
3632 Indices);
3633 return;
3634 }
3635
3636 // Copy a DDD register triple by copying the individual sub-registers.
3637 if (AArch64::DDDRegClass.contains(DestReg) &&
3638 AArch64::DDDRegClass.contains(SrcReg)) {
3639 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
3640 AArch64::dsub2};
3641 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
3642 Indices);
3643 return;
3644 }
3645
3646 // Copy a DD register pair by copying the individual sub-registers.
3647 if (AArch64::DDRegClass.contains(DestReg) &&
3648 AArch64::DDRegClass.contains(SrcReg)) {
3649 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
3650 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv8i8,
3651 Indices);
3652 return;
3653 }
3654
3655 // Copy a QQQQ register quad by copying the individual sub-registers.
3656 if (AArch64::QQQQRegClass.contains(DestReg) &&
3657 AArch64::QQQQRegClass.contains(SrcReg)) {
3658 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
3659 AArch64::qsub2, AArch64::qsub3};
3660 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
3661 Indices);
3662 return;
3663 }
3664
3665 // Copy a QQQ register triple by copying the individual sub-registers.
3666 if (AArch64::QQQRegClass.contains(DestReg) &&
3667 AArch64::QQQRegClass.contains(SrcReg)) {
3668 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
3669 AArch64::qsub2};
3670 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
3671 Indices);
3672 return;
3673 }
3674
3675 // Copy a QQ register pair by copying the individual sub-registers.
3676 if (AArch64::QQRegClass.contains(DestReg) &&
3677 AArch64::QQRegClass.contains(SrcReg)) {
3678 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
3679 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRv16i8,
3680 Indices);
3681 return;
3682 }
3683
3684 if (AArch64::XSeqPairsClassRegClass.contains(DestReg) &&
3685 AArch64::XSeqPairsClassRegClass.contains(SrcReg)) {
3686 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
3687 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRXrs,
3688 AArch64::XZR, Indices);
3689 return;
3690 }
3691
3692 if (AArch64::WSeqPairsClassRegClass.contains(DestReg) &&
3693 AArch64::WSeqPairsClassRegClass.contains(SrcReg)) {
3694 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
3695 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRWrs,
3696 AArch64::WZR, Indices);
3697 return;
3698 }
3699
3700 if (AArch64::FPR128RegClass.contains(DestReg) &&
3701 AArch64::FPR128RegClass.contains(SrcReg)) {
3702 if (Subtarget.forceStreamingCompatibleSVE()) {
3703 BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ))
3704 .addReg(AArch64::Z0 + (DestReg - AArch64::Q0), RegState::Define)
3705 .addReg(AArch64::Z0 + (SrcReg - AArch64::Q0))
3706 .addReg(AArch64::Z0 + (SrcReg - AArch64::Q0));
3707 } else if (Subtarget.hasNEON()) {
3708 BuildMI(MBB, I, DL, get(AArch64::ORRv16i8), DestReg)
3709 .addReg(SrcReg)
3710 .addReg(SrcReg, getKillRegState(KillSrc));
3711 } else {
3712 BuildMI(MBB, I, DL, get(AArch64::STRQpre))
3713 .addReg(AArch64::SP, RegState::Define)
3714 .addReg(SrcReg, getKillRegState(KillSrc))
3715 .addReg(AArch64::SP)
3716 .addImm(-16);
3717 BuildMI(MBB, I, DL, get(AArch64::LDRQpre))
3718 .addReg(AArch64::SP, RegState::Define)
3719 .addReg(DestReg, RegState::Define)
3720 .addReg(AArch64::SP)
3721 .addImm(16);
3722 }
3723 return;
3724 }
3725
3726 if (AArch64::FPR64RegClass.contains(DestReg) &&
3727 AArch64::FPR64RegClass.contains(SrcReg)) {
3728 BuildMI(MBB, I, DL, get(AArch64::FMOVDr), DestReg)
3729 .addReg(SrcReg, getKillRegState(KillSrc));
3730 return;
3731 }
3732
3733 if (AArch64::FPR32RegClass.contains(DestReg) &&
3734 AArch64::FPR32RegClass.contains(SrcReg)) {
3735 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
3736 .addReg(SrcReg, getKillRegState(KillSrc));
3737 return;
3738 }
3739
3740 if (AArch64::FPR16RegClass.contains(DestReg) &&
3741 AArch64::FPR16RegClass.contains(SrcReg)) {
3742 DestReg =
3743 RI.getMatchingSuperReg(DestReg, AArch64::hsub, &AArch64::FPR32RegClass);
3744 SrcReg =
3745 RI.getMatchingSuperReg(SrcReg, AArch64::hsub, &AArch64::FPR32RegClass);
3746 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
3747 .addReg(SrcReg, getKillRegState(KillSrc));
3748 return;
3749 }
3750
3751 if (AArch64::FPR8RegClass.contains(DestReg) &&
3752 AArch64::FPR8RegClass.contains(SrcReg)) {
3753 DestReg =
3754 RI.getMatchingSuperReg(DestReg, AArch64::bsub, &AArch64::FPR32RegClass);
3755 SrcReg =
3756 RI.getMatchingSuperReg(SrcReg, AArch64::bsub, &AArch64::FPR32RegClass);
3757 BuildMI(MBB, I, DL, get(AArch64::FMOVSr), DestReg)
3758 .addReg(SrcReg, getKillRegState(KillSrc));
3759 return;
3760 }
3761
3762 // Copies between GPR64 and FPR64.
3763 if (AArch64::FPR64RegClass.contains(DestReg) &&
3764 AArch64::GPR64RegClass.contains(SrcReg)) {
3765 BuildMI(MBB, I, DL, get(AArch64::FMOVXDr), DestReg)
3766 .addReg(SrcReg, getKillRegState(KillSrc));
3767 return;
3768 }
3769 if (AArch64::GPR64RegClass.contains(DestReg) &&
3770 AArch64::FPR64RegClass.contains(SrcReg)) {
3771 BuildMI(MBB, I, DL, get(AArch64::FMOVDXr), DestReg)
3772 .addReg(SrcReg, getKillRegState(KillSrc));
3773 return;
3774 }
3775 // Copies between GPR32 and FPR32.
3776 if (AArch64::FPR32RegClass.contains(DestReg) &&
3777 AArch64::GPR32RegClass.contains(SrcReg)) {
3778 BuildMI(MBB, I, DL, get(AArch64::FMOVWSr), DestReg)
3779 .addReg(SrcReg, getKillRegState(KillSrc));
3780 return;
3781 }
3782 if (AArch64::GPR32RegClass.contains(DestReg) &&
3783 AArch64::FPR32RegClass.contains(SrcReg)) {
3784 BuildMI(MBB, I, DL, get(AArch64::FMOVSWr), DestReg)
3785 .addReg(SrcReg, getKillRegState(KillSrc));
3786 return;
3787 }
3788
3789 if (DestReg == AArch64::NZCV) {
3790 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
3791 BuildMI(MBB, I, DL, get(AArch64::MSR))
3792 .addImm(AArch64SysReg::NZCV)
3793 .addReg(SrcReg, getKillRegState(KillSrc))
3794 .addReg(AArch64::NZCV, RegState::Implicit | RegState::Define);
3795 return;
3796 }
3797
3798 if (SrcReg == AArch64::NZCV) {
3799 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
3800 BuildMI(MBB, I, DL, get(AArch64::MRS), DestReg)
3801 .addImm(AArch64SysReg::NZCV)
3802 .addReg(AArch64::NZCV, RegState::Implicit | getKillRegState(KillSrc));
3803 return;
3804 }
3805
3806#ifndef NDEBUG
3808 errs() << TRI.getRegAsmName(DestReg) << " = COPY "
3809 << TRI.getRegAsmName(SrcReg) << "\n";
3810#endif
3811 llvm_unreachable("unimplemented reg-to-reg copy");
3812}
3813
3816 MachineBasicBlock::iterator InsertBefore,
3817 const MCInstrDesc &MCID,
3818 Register SrcReg, bool IsKill,
3819 unsigned SubIdx0, unsigned SubIdx1, int FI,
3820 MachineMemOperand *MMO) {
3821 Register SrcReg0 = SrcReg;
3822 Register SrcReg1 = SrcReg;
3823 if (SrcReg.isPhysical()) {
3824 SrcReg0 = TRI.getSubReg(SrcReg, SubIdx0);
3825 SubIdx0 = 0;
3826 SrcReg1 = TRI.getSubReg(SrcReg, SubIdx1);
3827 SubIdx1 = 0;
3828 }
3829 BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
3830 .addReg(SrcReg0, getKillRegState(IsKill), SubIdx0)
3831 .addReg(SrcReg1, getKillRegState(IsKill), SubIdx1)
3832 .addFrameIndex(FI)
3833 .addImm(0)
3834 .addMemOperand(MMO);
3835}
3836
3839 Register SrcReg, bool isKill, int FI,
3840 const TargetRegisterClass *RC,
3841 const TargetRegisterInfo *TRI,
3842 Register VReg) const {
3843 MachineFunction &MF = *MBB.getParent();
3844 MachineFrameInfo &MFI = MF.getFrameInfo();
3845
3847 MachineMemOperand *MMO =
3849 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
3850 unsigned Opc = 0;
3851 bool Offset = true;
3852 unsigned StackID = TargetStackID::Default;
3853 switch (TRI->getSpillSize(*RC)) {
3854 case 1:
3855 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
3856 Opc = AArch64::STRBui;
3857 break;
3858 case 2:
3859 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
3860 Opc = AArch64::STRHui;
3861 else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
3862 assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3863 Opc = AArch64::STR_PXI;
3865 }
3866 break;
3867 case 4:
3868 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
3869 Opc = AArch64::STRWui;
3870 if (SrcReg.isVirtual())
3871 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR32RegClass);
3872 else
3873 assert(SrcReg != AArch64::WSP);
3874 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
3875 Opc = AArch64::STRSui;
3876 break;
3877 case 8:
3878 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
3879 Opc = AArch64::STRXui;
3880 if (SrcReg.isVirtual())
3881 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
3882 else
3883 assert(SrcReg != AArch64::SP);
3884 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
3885 Opc = AArch64::STRDui;
3886 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
3888 get(AArch64::STPWi), SrcReg, isKill,
3889 AArch64::sube32, AArch64::subo32, FI, MMO);
3890 return;
3891 }
3892 break;
3893 case 16:
3894 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
3895 Opc = AArch64::STRQui;
3896 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
3897 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3898 Opc = AArch64::ST1Twov1d;
3899 Offset = false;
3900 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
3902 get(AArch64::STPXi), SrcReg, isKill,
3903 AArch64::sube64, AArch64::subo64, FI, MMO);
3904 return;
3905 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
3906 assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3907 Opc = AArch64::STR_ZXI;
3909 }
3910 break;
3911 case 24:
3912 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
3913 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3914 Opc = AArch64::ST1Threev1d;
3915 Offset = false;
3916 }
3917 break;
3918 case 32:
3919 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
3920 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3921 Opc = AArch64::ST1Fourv1d;
3922 Offset = false;
3923 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
3924 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3925 Opc = AArch64::ST1Twov2d;
3926 Offset = false;
3927 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
3928 assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3929 Opc = AArch64::STR_ZZXI;
3931 }
3932 break;
3933 case 48:
3934 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
3935 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3936 Opc = AArch64::ST1Threev2d;
3937 Offset = false;
3938 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
3939 assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3940 Opc = AArch64::STR_ZZZXI;
3942 }
3943 break;
3944 case 64:
3945 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
3946 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
3947 Opc = AArch64::ST1Fourv2d;
3948 Offset = false;
3949 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
3950 assert(Subtarget.hasSVE() && "Unexpected register store without SVE");
3951 Opc = AArch64::STR_ZZZZXI;
3953 }
3954 break;
3955 }
3956 assert(Opc && "Unknown register class");
3957 MFI.setStackID(FI, StackID);
3958
3959 const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
3960 .addReg(SrcReg, getKillRegState(isKill))
3961 .addFrameIndex(FI);
3962
3963 if (Offset)
3964 MI.addImm(0);
3965 MI.addMemOperand(MMO);
3966}
3967
3970 MachineBasicBlock::iterator InsertBefore,
3971 const MCInstrDesc &MCID,
3972 Register DestReg, unsigned SubIdx0,
3973 unsigned SubIdx1, int FI,
3974 MachineMemOperand *MMO) {
3975 Register DestReg0 = DestReg;
3976 Register DestReg1 = DestReg;
3977 bool IsUndef = true;
3978 if (DestReg.isPhysical()) {
3979 DestReg0 = TRI.getSubReg(DestReg, SubIdx0);
3980 SubIdx0 = 0;
3981 DestReg1 = TRI.getSubReg(DestReg, SubIdx1);
3982 SubIdx1 = 0;
3983 IsUndef = false;
3984 }
3985 BuildMI(MBB, InsertBefore, DebugLoc(), MCID)
3986 .addReg(DestReg0, RegState::Define | getUndefRegState(IsUndef), SubIdx0)
3987 .addReg(DestReg1, RegState::Define | getUndefRegState(IsUndef), SubIdx1)
3988 .addFrameIndex(FI)
3989 .addImm(0)
3990 .addMemOperand(MMO);
3991}
3992
3995 Register DestReg, int FI,
3996 const TargetRegisterClass *RC,
3997 const TargetRegisterInfo *TRI,
3998 Register VReg) const {
3999 MachineFunction &MF = *MBB.getParent();
4000 MachineFrameInfo &MFI = MF.getFrameInfo();
4002 MachineMemOperand *MMO =
4004 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
4005
4006 unsigned Opc = 0;
4007 bool Offset = true;
4008 unsigned StackID = TargetStackID::Default;
4009 switch (TRI->getSpillSize(*RC)) {
4010 case 1:
4011 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
4012 Opc = AArch64::LDRBui;
4013 break;
4014 case 2:
4015 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
4016 Opc = AArch64::LDRHui;
4017 else if (AArch64::PPRRegClass.hasSubClassEq(RC)) {
4018 assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
4019 Opc = AArch64::LDR_PXI;
4021 }
4022 break;
4023 case 4:
4024 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
4025 Opc = AArch64::LDRWui;
4026 if (DestReg.isVirtual())
4027 MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR32RegClass);
4028 else
4029 assert(DestReg != AArch64::WSP);
4030 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
4031 Opc = AArch64::LDRSui;
4032 break;
4033 case 8:
4034 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
4035 Opc = AArch64::LDRXui;
4036 if (DestReg.isVirtual())
4037 MF.getRegInfo().constrainRegClass(DestReg, &AArch64::GPR64RegClass);
4038 else
4039 assert(DestReg != AArch64::SP);
4040 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
4041 Opc = AArch64::LDRDui;
4042 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
4044 get(AArch64::LDPWi), DestReg, AArch64::sube32,
4045 AArch64::subo32, FI, MMO);
4046 return;
4047 }
4048 break;
4049 case 16:
4050 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
4051 Opc = AArch64::LDRQui;
4052 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
4053 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
4054 Opc = AArch64::LD1Twov1d;
4055 Offset = false;
4056 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
4058 get(AArch64::LDPXi), DestReg, AArch64::sube64,
4059 AArch64::subo64, FI, MMO);
4060 return;
4061 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
4062 assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
4063 Opc = AArch64::LDR_ZXI;
4065 }
4066 break;
4067 case 24:
4068 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
4069 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
4070 Opc = AArch64::LD1Threev1d;
4071 Offset = false;
4072 }
4073 break;
4074 case 32:
4075 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
4076 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
4077 Opc = AArch64::LD1Fourv1d;
4078 Offset = false;
4079 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
4080 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
4081 Opc = AArch64::LD1Twov2d;
4082 Offset = false;
4083 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
4084 assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
4085 Opc = AArch64::LDR_ZZXI;
4087 }
4088 break;
4089 case 48:
4090 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
4091 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
4092 Opc = AArch64::LD1Threev2d;
4093 Offset = false;
4094 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
4095 assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
4096 Opc = AArch64::LDR_ZZZXI;
4098 }
4099 break;
4100 case 64:
4101 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
4102 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
4103 Opc = AArch64::LD1Fourv2d;
4104 Offset = false;
4105 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
4106 assert(Subtarget.hasSVE() && "Unexpected register load without SVE");
4107 Opc = AArch64::LDR_ZZZZXI;
4109 }
4110 break;
4111 }
4112
4113 assert(Opc && "Unknown register class");
4114 MFI.setStackID(FI, StackID);
4115
4116 const MachineInstrBuilder MI = BuildMI(MBB, MBBI, DebugLoc(), get(Opc))
4117 .addReg(DestReg, getDefRegState(true))
4118 .addFrameIndex(FI);
4119 if (Offset)
4120 MI.addImm(0);
4121 MI.addMemOperand(MMO);
4122}
4123
4125 const MachineInstr &UseMI,
4126 const TargetRegisterInfo *TRI) {
4127 return any_of(instructionsWithoutDebug(std::next(DefMI.getIterator()),
4128 UseMI.getIterator()),
4129 [TRI](const MachineInstr &I) {
4130 return I.modifiesRegister(AArch64::NZCV, TRI) ||
4131 I.readsRegister(AArch64::NZCV, TRI);
4132 });
4133}
4134
4136 const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized) {
4137 // The smallest scalable element supported by scaled SVE addressing
4138 // modes are predicates, which are 2 scalable bytes in size. So the scalable
4139 // byte offset must always be a multiple of 2.
4140 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
4141
4142 // VGSized offsets are divided by '2', because the VG register is the
4143 // the number of 64bit granules as opposed to 128bit vector chunks,
4144 // which is how the 'n' in e.g. MVT::nxv1i8 is modelled.
4145 // So, for a stack offset of 16 MVT::nxv1i8's, the size is n x 16 bytes.
4146 // VG = n * 2 and the dwarf offset must be VG * 8 bytes.
4147 ByteSized = Offset.getFixed();
4148 VGSized = Offset.getScalable() / 2;
4149}
4150
4151/// Returns the offset in parts to which this frame offset can be
4152/// decomposed for the purpose of describing a frame offset.
4153/// For non-scalable offsets this is simply its byte size.
4155 const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors,
4156 int64_t &NumDataVectors) {
4157 // The smallest scalable element supported by scaled SVE addressing
4158 // modes are predicates, which are 2 scalable bytes in size. So the scalable
4159 // byte offset must always be a multiple of 2.
4160 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
4161
4162 NumBytes = Offset.getFixed();
4163 NumDataVectors = 0;
4164 NumPredicateVectors = Offset.getScalable() / 2;
4165 // This method is used to get the offsets to adjust the frame offset.
4166 // If the function requires ADDPL to be used and needs more than two ADDPL
4167 // instructions, part of the offset is folded into NumDataVectors so that it
4168 // uses ADDVL for part of it, reducing the number of ADDPL instructions.
4169 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
4170 NumPredicateVectors > 62) {
4171 NumDataVectors = NumPredicateVectors / 8;
4172 NumPredicateVectors -= NumDataVectors * 8;
4173 }
4174}
4175
4176// Convenience function to create a DWARF expression for
4177// Expr + NumBytes + NumVGScaledBytes * AArch64::VG
4178static void appendVGScaledOffsetExpr(SmallVectorImpl<char> &Expr, int NumBytes,
4179 int NumVGScaledBytes, unsigned VG,
4180 llvm::raw_string_ostream &Comment) {
4181 uint8_t buffer[16];
4182
4183 if (NumBytes) {
4184 Expr.push_back(dwarf::DW_OP_consts);
4185 Expr.append(buffer, buffer + encodeSLEB128(NumBytes, buffer));
4186 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
4187 Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(NumBytes);
4188 }
4189
4190 if (NumVGScaledBytes) {
4191 Expr.push_back((uint8_t)dwarf::DW_OP_consts);
4192 Expr.append(buffer, buffer + encodeSLEB128(NumVGScaledBytes, buffer));
4193
4194 Expr.push_back((uint8_t)dwarf::DW_OP_bregx);
4195 Expr.append(buffer, buffer + encodeULEB128(VG, buffer));
4196 Expr.push_back(0);
4197
4198 Expr.push_back((uint8_t)dwarf::DW_OP_mul);
4199 Expr.push_back((uint8_t)dwarf::DW_OP_plus);
4200
4201 Comment << (NumVGScaledBytes < 0 ? " - " : " + ")
4202 << std::abs(NumVGScaledBytes) << " * VG";
4203 }
4204}
4205
4206// Creates an MCCFIInstruction:
4207// { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
4209 unsigned Reg,
4210 const StackOffset &Offset) {
4211 int64_t NumBytes, NumVGScaledBytes;
4213 NumVGScaledBytes);
4214 std::string CommentBuffer;
4215 llvm::raw_string_ostream Comment(CommentBuffer);
4216
4217 if (Reg == AArch64::SP)
4218 Comment << "sp";
4219 else if (Reg == AArch64::FP)
4220 Comment << "fp";
4221 else
4222 Comment << printReg(Reg, &TRI);
4223
4224 // Build up the expression (Reg + NumBytes + NumVGScaledBytes * AArch64::VG)
4225 SmallString<64> Expr;
4226 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
4227 Expr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg));
4228 Expr.push_back(0);
4229 appendVGScaledOffsetExpr(Expr, NumBytes, NumVGScaledBytes,
4230 TRI.getDwarfRegNum(AArch64::VG, true), Comment);
4231
4232 // Wrap this into DW_CFA_def_cfa.
4233 SmallString<64> DefCfaExpr;
4234 DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression);
4235 uint8_t buffer[16];
4236 DefCfaExpr.append(buffer, buffer + encodeULEB128(Expr.size(), buffer));
4237 DefCfaExpr.append(Expr.str());
4238 return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(),
4239 Comment.str());
4240}
4241
4243 unsigned FrameReg, unsigned Reg,
4244 const StackOffset &Offset,
4245 bool LastAdjustmentWasScalable) {
4246 if (Offset.getScalable())
4247 return createDefCFAExpression(TRI, Reg, Offset);
4248
4249 if (FrameReg == Reg && !LastAdjustmentWasScalable)
4250 return MCCFIInstruction::cfiDefCfaOffset(nullptr, int(Offset.getFixed()));
4251
4252 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
4253 return MCCFIInstruction::cfiDefCfa(nullptr, DwarfReg, (int)Offset.getFixed());
4254}
4255
4257 unsigned Reg,
4258 const StackOffset &OffsetFromDefCFA) {
4259 int64_t NumBytes, NumVGScaledBytes;
4261 OffsetFromDefCFA, NumBytes, NumVGScaledBytes);
4262
4263 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
4264
4265 // Non-scalable offsets can use DW_CFA_offset directly.
4266 if (!NumVGScaledBytes)
4267 return MCCFIInstruction::createOffset(nullptr, DwarfReg, NumBytes);
4268
4269 std::string CommentBuffer;
4270 llvm::raw_string_ostream Comment(CommentBuffer);
4271 Comment << printReg(Reg, &TRI) << " @ cfa";
4272
4273 // Build up expression (NumBytes + NumVGScaledBytes * AArch64::VG)
4274 SmallString<64> OffsetExpr;
4275 appendVGScaledOffsetExpr(OffsetExpr, NumBytes, NumVGScaledBytes,
4276 TRI.getDwarfRegNum(AArch64::VG, true), Comment);
4277
4278 // Wrap this into DW_CFA_expression
4279 SmallString<64> CfaExpr;
4280 CfaExpr.push_back(dwarf::DW_CFA_expression);
4281 uint8_t buffer[16];
4282 CfaExpr.append(buffer, buffer + encodeULEB128(DwarfReg, buffer));
4283 CfaExpr.append(buffer, buffer + encodeULEB128(OffsetExpr.size(), buffer));
4284 CfaExpr.append(OffsetExpr.str());
4285
4286 return MCCFIInstruction::createEscape(nullptr, CfaExpr.str(), Comment.str());
4287}
4288
4289// Helper function to emit a frame offset adjustment from a given
4290// pointer (SrcReg), stored into DestReg. This function is explicit
4291// in that it requires the opcode.
4294 const DebugLoc &DL, unsigned DestReg,
4295 unsigned SrcReg, int64_t Offset, unsigned Opc,
4296 const TargetInstrInfo *TII,
4297 MachineInstr::MIFlag Flag, bool NeedsWinCFI,
4298 bool *HasWinCFI, bool EmitCFAOffset,
4299 StackOffset CFAOffset, unsigned FrameReg) {
4300 int Sign = 1;
4301 unsigned MaxEncoding, ShiftSize;
4302 switch (Opc) {
4303 case AArch64::ADDXri:
4304 case AArch64::ADDSXri:
4305 case AArch64::SUBXri:
4306 case AArch64::SUBSXri:
4307 MaxEncoding = 0xfff;
4308 ShiftSize = 12;
4309 break;
4310 case AArch64::ADDVL_XXI:
4311 case AArch64::ADDPL_XXI:
4312 case AArch64::ADDSVL_XXI:
4313 case AArch64::ADDSPL_XXI:
4314 MaxEncoding = 31;
4315 ShiftSize = 0;
4316 if (Offset < 0) {
4317 MaxEncoding = 32;
4318 Sign = -1;
4319 Offset = -Offset;
4320 }
4321 break;
4322 default:
4323 llvm_unreachable("Unsupported opcode");
4324 }
4325
4326 // `Offset` can be in bytes or in "scalable bytes".
4327 int VScale = 1;
4328 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
4329 VScale = 16;
4330 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
4331 VScale = 2;
4332
4333 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
4334 // scratch register. If DestReg is a virtual register, use it as the
4335 // scratch register; otherwise, create a new virtual register (to be
4336 // replaced by the scavenger at the end of PEI). That case can be optimized
4337 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
4338 // register can be loaded with offset%8 and the add/sub can use an extending
4339 // instruction with LSL#3.
4340 // Currently the function handles any offsets but generates a poor sequence
4341 // of code.
4342 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
4343
4344 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
4345 Register TmpReg = DestReg;
4346 if (TmpReg == AArch64::XZR)
4348 &AArch64::GPR64RegClass);
4349 do {
4350 uint64_t ThisVal = std::min<uint64_t>(Offset, MaxEncodableValue);
4351 unsigned LocalShiftSize = 0;
4352 if (ThisVal > MaxEncoding) {
4353 ThisVal = ThisVal >> ShiftSize;
4354 LocalShiftSize = ShiftSize;
4355 }
4356 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
4357 "Encoding cannot handle value that big");
4358
4359 Offset -= ThisVal << LocalShiftSize;
4360 if (Offset == 0)
4361 TmpReg = DestReg;
4362 auto MBI = BuildMI(MBB, MBBI, DL, TII->get(Opc), TmpReg)
4363 .addReg(SrcReg)
4364 .addImm(Sign * (int)ThisVal);
4365 if (ShiftSize)
4366 MBI = MBI.addImm(
4368 MBI = MBI.setMIFlag(Flag);
4369
4370 auto Change =
4371 VScale == 1
4372 ? StackOffset::getFixed(ThisVal << LocalShiftSize)
4373 : StackOffset::getScalable(VScale * (ThisVal << LocalShiftSize));
4374 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
4375 CFAOffset += Change;
4376 else
4377 CFAOffset -= Change;
4378 if (EmitCFAOffset && DestReg == TmpReg) {
4379 MachineFunction &MF = *MBB.getParent();
4380 const TargetSubtargetInfo &STI = MF.getSubtarget();
4381 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
4382
4383 unsigned CFIIndex = MF.addFrameInst(
4384 createDefCFA(TRI, FrameReg, DestReg, CFAOffset, VScale != 1));
4385 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION))
4386 .addCFIIndex(CFIIndex)
4387 .setMIFlags(Flag);
4388 }
4389
4390 if (NeedsWinCFI) {
4391 assert(Sign == 1 && "SEH directives should always have a positive sign");
4392 int Imm = (int)(ThisVal << LocalShiftSize);
4393 if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
4394 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
4395 if (HasWinCFI)
4396 *HasWinCFI = true;
4397 if (Imm == 0)
4398 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_SetFP)).setMIFlag(Flag);
4399 else
4400 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_AddFP))
4401 .addImm(Imm)
4402 .setMIFlag(Flag);
4403 assert(Offset == 0 && "Expected remaining offset to be zero to "
4404 "emit a single SEH directive");
4405 } else if (DestReg == AArch64::SP) {
4406 if (HasWinCFI)
4407 *HasWinCFI = true;
4408 assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
4409 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_StackAlloc))
4410 .addImm(Imm)
4411 .setMIFlag(Flag);
4412 }
4413 }
4414
4415 SrcReg = TmpReg;
4416 } while (Offset);
4417}
4418
4421 unsigned DestReg, unsigned SrcReg,
4423 MachineInstr::MIFlag Flag, bool SetNZCV,
4424 bool NeedsWinCFI, bool *HasWinCFI,
4425 bool EmitCFAOffset, StackOffset CFAOffset,
4426 unsigned FrameReg) {
4427 // If a function is marked as arm_locally_streaming, then the runtime value of
4428 // vscale in the prologue/epilogue is different the runtime value of vscale
4429 // in the function's body. To avoid having to consider multiple vscales,
4430 // we can use `addsvl` to allocate any scalable stack-slots, which under
4431 // most circumstances will be only locals, not callee-save slots.
4432 const Function &F = MBB.getParent()->getFunction();
4433 bool UseSVL = F.hasFnAttribute("aarch64_pstate_sm_body");
4434
4435 int64_t Bytes, NumPredicateVectors, NumDataVectors;
4437 Offset, Bytes, NumPredicateVectors, NumDataVectors);
4438
4439 // First emit non-scalable frame offsets, or a simple 'mov'.
4440 if (Bytes || (!Offset && SrcReg != DestReg)) {
4441 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
4442 "SP increment/decrement not 8-byte aligned");
4443 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
4444 if (Bytes < 0) {
4445 Bytes = -Bytes;
4446 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
4447 }
4448 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Bytes, Opc, TII, Flag,
4449 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
4450 FrameReg);
4451 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
4452 ? StackOffset::getFixed(-Bytes)
4453 : StackOffset::getFixed(Bytes);
4454 SrcReg = DestReg;
4455 FrameReg = DestReg;
4456 }
4457
4458 assert(!(SetNZCV && (NumPredicateVectors || NumDataVectors)) &&
4459 "SetNZCV not supported with SVE vectors");
4460 assert(!(NeedsWinCFI && (NumPredicateVectors || NumDataVectors)) &&
4461 "WinCFI not supported with SVE vectors");
4462
4463 if (NumDataVectors) {
4464 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumDataVectors,
4465 UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI,
4466 TII, Flag, NeedsWinCFI, nullptr, EmitCFAOffset,
4467 CFAOffset, FrameReg);
4468 CFAOffset += StackOffset::getScalable(-NumDataVectors * 16);
4469 SrcReg = DestReg;
4470 }
4471
4472 if (NumPredicateVectors) {
4473 assert(DestReg != AArch64::SP && "Unaligned access to SP");
4474 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, NumPredicateVectors,
4475 UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI,
4476 TII, Flag, NeedsWinCFI, nullptr, EmitCFAOffset,
4477 CFAOffset, FrameReg);
4478 }
4479}
4480
4483 MachineBasicBlock::iterator InsertPt, int FrameIndex,
4484 LiveIntervals *LIS, VirtRegMap *VRM) const {
4485 // This is a bit of a hack. Consider this instruction:
4486 //
4487 // %0 = COPY %sp; GPR64all:%0
4488 //
4489 // We explicitly chose GPR64all for the virtual register so such a copy might
4490 // be eliminated by RegisterCoalescer. However, that may not be possible, and
4491 // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
4492 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
4493 //
4494 // To prevent that, we are going to constrain the %0 register class here.
4495 //
4496 // <rdar://problem/11522048>
4497 //
4498 if (MI.isFullCopy()) {
4499 Register DstReg = MI.getOperand(0).getReg();
4500 Register SrcReg = MI.getOperand(1).getReg();
4501 if (SrcReg == AArch64::SP && DstReg.isVirtual()) {
4502 MF.getRegInfo().constrainRegClass(DstReg, &AArch64::GPR64RegClass);
4503 return nullptr;
4504 }
4505 if (DstReg == AArch64::SP && SrcReg.isVirtual()) {
4506 MF.getRegInfo().constrainRegClass(SrcReg, &AArch64::GPR64RegClass);
4507 return nullptr;
4508 }
4509 // Nothing can folded with copy from/to NZCV.
4510 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
4511 return nullptr;
4512 }
4513
4514 // Handle the case where a copy is being spilled or filled but the source
4515 // and destination register class don't match. For example:
4516 //
4517 // %0 = COPY %xzr; GPR64common:%0
4518 //
4519 // In this case we can still safely fold away the COPY and generate the
4520 // following spill code:
4521 //
4522 // STRXui %xzr, %stack.0
4523 //
4524 // This also eliminates spilled cross register class COPYs (e.g. between x and
4525 // d regs) of the same size. For example:
4526 //
4527 // %0 = COPY %1; GPR64:%0, FPR64:%1
4528 //
4529 // will be filled as
4530 //
4531 // LDRDui %0, fi<#0>
4532 //
4533 // instead of
4534 //
4535 // LDRXui %Temp, fi<#0>
4536 // %0 = FMOV %Temp
4537 //
4538 if (MI.isCopy() && Ops.size() == 1 &&
4539 // Make sure we're only folding the explicit COPY defs/uses.
4540 (Ops[0] == 0 || Ops[0] == 1)) {
4541 bool IsSpill = Ops[0] == 0;
4542 bool IsFill = !IsSpill;
4544 const MachineRegisterInfo &MRI = MF.getRegInfo();
4545 MachineBasicBlock &MBB = *MI.getParent();
4546 const MachineOperand &DstMO = MI.getOperand(0);
4547 const MachineOperand &SrcMO = MI.getOperand(1);
4548 Register DstReg = DstMO.getReg();
4549 Register SrcReg = SrcMO.getReg();
4550 // This is slightly expensive to compute for physical regs since
4551 // getMinimalPhysRegClass is slow.
4552 auto getRegClass = [&](unsigned Reg) {
4553 return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
4554 : TRI.getMinimalPhysRegClass(Reg);
4555 };
4556
4557 if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
4558 assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
4559 TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
4560 "Mismatched register size in non subreg COPY");
4561 if (IsSpill)
4562 storeRegToStackSlot(MBB, InsertPt, SrcReg, SrcMO.isKill(), FrameIndex,
4563 getRegClass(SrcReg), &TRI, Register());
4564 else
4565 loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex,
4566 getRegClass(DstReg), &TRI, Register());
4567 return &*--InsertPt;
4568 }
4569
4570 // Handle cases like spilling def of:
4571 //
4572 // %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
4573 //
4574 // where the physical register source can be widened and stored to the full
4575 // virtual reg destination stack slot, in this case producing:
4576 //
4577 // STRXui %xzr, %stack.0
4578 //
4579 if (IsSpill && DstMO.isUndef() && SrcReg.isPhysical()) {
4580 assert(SrcMO.getSubReg() == 0 &&
4581 "Unexpected subreg on physical register");
4582 const TargetRegisterClass *SpillRC;
4583 unsigned SpillSubreg;
4584 switch (DstMO.getSubReg()) {
4585 default:
4586 SpillRC = nullptr;
4587 break;
4588 case AArch64::sub_32:
4589 case AArch64::ssub:
4590 if (AArch64::GPR32RegClass.contains(SrcReg)) {
4591 SpillRC = &AArch64::GPR64RegClass;
4592 SpillSubreg = AArch64::sub_32;
4593 } else if (AArch64::FPR32RegClass.contains(SrcReg)) {
4594 SpillRC = &AArch64::FPR64RegClass;
4595 SpillSubreg = AArch64::ssub;
4596 } else
4597 SpillRC = nullptr;
4598 break;
4599 case AArch64::dsub:
4600 if (AArch64::FPR64RegClass.contains(SrcReg)) {
4601 SpillRC = &AArch64::FPR128RegClass;
4602 SpillSubreg = AArch64::dsub;
4603 } else
4604 SpillRC = nullptr;
4605 break;
4606 }
4607
4608 if (SpillRC)
4609 if (unsigned WidenedSrcReg =
4610 TRI.getMatchingSuperReg(SrcReg, SpillSubreg, SpillRC)) {
4611 storeRegToStackSlot(MBB, InsertPt, WidenedSrcReg, SrcMO.isKill(),
4612 FrameIndex, SpillRC, &TRI, Register());
4613 return &*--InsertPt;
4614 }
4615 }
4616
4617 // Handle cases like filling use of:
4618 //
4619 // %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
4620 //
4621 // where we can load the full virtual reg source stack slot, into the subreg
4622 // destination, in this case producing:
4623 //
4624 // LDRWui %0:sub_32<def,read-undef>, %stack.0
4625 //
4626 if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
4627 const TargetRegisterClass *FillRC;
4628 switch (DstMO.getSubReg()) {
4629 default:
4630 FillRC = nullptr;
4631 break;
4632 case AArch64::sub_32:
4633 FillRC = &AArch64::GPR32RegClass;
4634 break;
4635 case AArch64::ssub:
4636 FillRC = &AArch64::FPR32RegClass;
4637 break;
4638 case AArch64::dsub:
4639 FillRC = &AArch64::FPR64RegClass;
4640 break;
4641 }
4642
4643 if (FillRC) {
4644 assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
4645 TRI.getRegSizeInBits(*FillRC) &&
4646 "Mismatched regclass size on folded subreg COPY");
4647 loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex, FillRC, &TRI,
4648 Register());
4649 MachineInstr &LoadMI = *--InsertPt;
4650 MachineOperand &LoadDst = LoadMI.getOperand(0);
4651 assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
4652 LoadDst.setSubReg(DstMO.getSubReg());
4653 LoadDst.setIsUndef();
4654 return &LoadMI;
4655 }
4656 }
4657 }
4658
4659 // Cannot fold.
4660 return nullptr;
4661}
4662
4664 StackOffset &SOffset,
4665 bool *OutUseUnscaledOp,
4666 unsigned *OutUnscaledOp,
4667 int64_t *EmittableOffset) {
4668 // Set output values in case of early exit.
4669 if (EmittableOffset)
4670 *EmittableOffset = 0;
4671 if (OutUseUnscaledOp)
4672 *OutUseUnscaledOp = false;
4673 if (OutUnscaledOp)
4674 *OutUnscaledOp = 0;
4675
4676 // Exit early for structured vector spills/fills as they can't take an
4677 // immediate offset.
4678 switch (MI.getOpcode()) {
4679 default:
4680 break;
4681 case AArch64::LD1Twov2d:
4682 case AArch64::LD1Threev2d:
4683 case AArch64::LD1Fourv2d:
4684 case AArch64::LD1Twov1d:
4685 case AArch64::LD1Threev1d:
4686 case AArch64::LD1Fourv1d:
4687 case AArch64::ST1Twov2d:
4688 case AArch64::ST1Threev2d:
4689 case AArch64::ST1Fourv2d:
4690 case AArch64::ST1Twov1d:
4691 case AArch64::ST1Threev1d:
4692 case AArch64::ST1Fourv1d:
4693 case AArch64::ST1i8:
4694 case AArch64::ST1i16:
4695 case AArch64::ST1i32:
4696 case AArch64::ST1i64:
4697 case AArch64::IRG:
4698 case AArch64::IRGstack:
4699 case AArch64::STGloop:
4700 case AArch64::STZGloop:
4702 }
4703
4704 // Get the min/max offset and the scale.
4705 TypeSize ScaleValue(0U, false);
4706 unsigned Width;
4707 int64_t MinOff, MaxOff;
4708 if (!AArch64InstrInfo::getMemOpInfo(MI.getOpcode(), ScaleValue, Width, MinOff,
4709 MaxOff))
4710 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
4711
4712 // Construct the complete offset.
4713 bool IsMulVL = ScaleValue.isScalable();
4714 unsigned Scale = ScaleValue.getKnownMinValue();
4715 int64_t Offset = IsMulVL ? SOffset.getScalable() : SOffset.getFixed();
4716
4717 const MachineOperand &ImmOpnd =
4718 MI.getOperand(AArch64InstrInfo::getLoadStoreImmIdx(MI.getOpcode()));
4719 Offset += ImmOpnd.getImm() * Scale;
4720
4721 // If the offset doesn't match the scale, we rewrite the instruction to
4722 // use the unscaled instruction instead. Likewise, if we have a negative
4723 // offset and there is an unscaled op to use.
4724 std::optional<unsigned> UnscaledOp =
4726 bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
4727 if (useUnscaledOp &&
4728 !AArch64InstrInfo::getMemOpInfo(*UnscaledOp, ScaleValue, Width, MinOff,
4729 MaxOff))
4730 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
4731
4732 Scale = ScaleValue.getKnownMinValue();
4733 assert(IsMulVL == ScaleValue.isScalable() &&
4734 "Unscaled opcode has different value for scalable");
4735
4736 int64_t Remainder = Offset % Scale;
4737 assert(!(Remainder && useUnscaledOp) &&
4738 "Cannot have remainder when using unscaled op");
4739
4740 assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
4741 int64_t NewOffset = Offset / Scale;
4742 if (MinOff <= NewOffset && NewOffset <= MaxOff)
4743 Offset = Remainder;
4744 else {
4745 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
4746 Offset = Offset - NewOffset * Scale + Remainder;
4747 }
4748
4749 if (EmittableOffset)
4750 *EmittableOffset = NewOffset;
4751 if (OutUseUnscaledOp)
4752 *OutUseUnscaledOp = useUnscaledOp;
4753 if (OutUnscaledOp && UnscaledOp)
4754 *OutUnscaledOp = *UnscaledOp;
4755
4756 if (IsMulVL)
4757 SOffset = StackOffset::get(SOffset.getFixed(), Offset);
4758 else
4759 SOffset = StackOffset::get(Offset, SOffset.getScalable());
4761 (SOffset ? 0 : AArch64FrameOffsetIsLegal);
4762}
4763
4765 unsigned FrameReg, StackOffset &Offset,
4766 const AArch64InstrInfo *TII) {
4767 unsigned Opcode = MI.getOpcode();
4768 unsigned ImmIdx = FrameRegIdx + 1;
4769
4770 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
4771 Offset += StackOffset::getFixed(MI.getOperand(ImmIdx).getImm());
4772 emitFrameOffset(*MI.getParent(), MI, MI.getDebugLoc(),
4773 MI.getOperand(0).getReg(), FrameReg, Offset, TII,
4774 MachineInstr::NoFlags, (Opcode == AArch64::ADDSXri));
4775 MI.eraseFromParent();
4776 Offset = StackOffset();
4777 return true;
4778 }
4779
4780 int64_t NewOffset;
4781 unsigned UnscaledOp;
4782 bool UseUnscaledOp;
4783 int Status = isAArch64FrameOffsetLegal(MI, Offset, &UseUnscaledOp,
4784 &UnscaledOp, &NewOffset);
4787 // Replace the FrameIndex with FrameReg.
4788 MI.getOperand(FrameRegIdx).ChangeToRegister(FrameReg, false);
4789 if (UseUnscaledOp)
4790 MI.setDesc(TII->get(UnscaledOp));
4791
4792 MI.getOperand(ImmIdx).ChangeToImmediate(NewOffset);
4793 return !Offset;
4794 }
4795
4796 return false;
4797}
4798
4800 return MCInstBuilder(AArch64::HINT).addImm(0);
4801}
4802
4803// AArch64 supports MachineCombiner.
4804bool AArch64InstrInfo::useMachineCombiner() const { return true; }
4805
4806// True when Opc sets flag
4807static bool isCombineInstrSettingFlag(unsigned Opc) {
4808 switch (Opc) {
4809 case AArch64::ADDSWrr:
4810 case AArch64::ADDSWri:
4811 case AArch64::ADDSXrr:
4812 case AArch64::ADDSXri:
4813 case AArch64::SUBSWrr:
4814 case AArch64::SUBSXrr:
4815 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
4816 case AArch64::SUBSWri:
4817 case AArch64::SUBSXri:
4818 return true;
4819 default:
4820 break;
4821 }
4822 return false;
4823}
4824
4825// 32b Opcodes that can be combined with a MUL
4826static bool isCombineInstrCandidate32(unsigned Opc) {
4827 switch (Opc) {
4828 case AArch64::ADDWrr:
4829 case AArch64::ADDWri:
4830 case AArch64::SUBWrr:
4831 case AArch64::ADDSWrr:
4832 case AArch64::ADDSWri:
4833 case AArch64::SUBSWrr:
4834 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
4835 case AArch64::SUBWri:
4836 case AArch64::SUBSWri:
4837 return true;
4838 default:
4839 break;
4840 }
4841 return false;
4842}
4843
4844// 64b Opcodes that can be combined with a MUL
4845static bool isCombineInstrCandidate64(unsigned Opc) {
4846 switch (Opc) {
4847 case AArch64::ADDXrr:
4848 case AArch64::ADDXri:
4849 case AArch64::SUBXrr:
4850 case AArch64::ADDSXrr:
4851 case AArch64::ADDSXri:
4852 case AArch64::SUBSXrr:
4853 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
4854 case AArch64::SUBXri:
4855 case AArch64::SUBSXri:
4856 case AArch64::ADDv8i8:
4857 case AArch64::ADDv16i8:
4858 case AArch64::ADDv4i16:
4859 case AArch64::ADDv8i16:
4860 case AArch64::ADDv2i32:
4861 case AArch64::ADDv4i32:
4862 case AArch64::SUBv8i8:
4863 case AArch64::SUBv16i8:
4864 case AArch64::SUBv4i16:
4865 case AArch64::SUBv8i16:
4866 case AArch64::SUBv2i32:
4867 case AArch64::SUBv4i32:
4868 return true;
4869 default:
4870 break;
4871 }
4872 return false;
4873}
4874
4875// FP Opcodes that can be combined with a FMUL.
4876static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
4877 switch (Inst.getOpcode()) {
4878 default:
4879 break;
4880 case AArch64::FADDHrr:
4881 case AArch64::FADDSrr:
4882 case AArch64::FADDDrr:
4883 case AArch64::FADDv4f16:
4884 case AArch64::FADDv8f16:
4885 case AArch64::FADDv2f32:
4886 case AArch64::FADDv2f64:
4887 case AArch64::FADDv4f32:
4888 case AArch64::FSUBHrr:
4889 case AArch64::FSUBSrr:
4890 case AArch64::FSUBDrr:
4891 case AArch64::FSUBv4f16:
4892 case AArch64::FSUBv8f16:
4893 case AArch64::FSUBv2f32:
4894 case AArch64::FSUBv2f64:
4895 case AArch64::FSUBv4f32:
4897 // We can fuse FADD/FSUB with FMUL, if fusion is either allowed globally by
4898 // the target options or if FADD/FSUB has the contract fast-math flag.
4899 return Options.UnsafeFPMath ||
4900 Options.AllowFPOpFusion == FPOpFusion::Fast ||
4902 return true;
4903 }
4904 return false;
4905}
4906
4907// Opcodes that can be combined with a MUL
4908static bool isCombineInstrCandidate(unsigned Opc) {
4910}
4911
4912//
4913// Utility routine that checks if \param MO is defined by an
4914// \param CombineOpc instruction in the basic block \param MBB
4916 unsigned CombineOpc, unsigned ZeroReg = 0,
4917 bool CheckZeroReg = false) {
4919 MachineInstr *MI = nullptr;
4920
4921 if (MO.isReg() && MO.getReg().isVirtual())
4922 MI = MRI.getUniqueVRegDef(MO.getReg());
4923 // And it needs to be in the trace (otherwise, it won't have a depth).
4924 if (!MI || MI->getParent() != &MBB || (unsigned)MI->getOpcode() != CombineOpc)
4925 return false;
4926 // Must only used by the user we combine with.
4927 if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
4928 return false;
4929
4930 if (CheckZeroReg) {
4931 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
4932 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
4933 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
4934 // The third input reg must be zero.
4935 if (MI->getOperand(3).getReg() != ZeroReg)
4936 return false;
4937 }
4938
4939 if (isCombineInstrSettingFlag(CombineOpc) &&
4940 MI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1)
4941 return false;
4942
4943 return true;
4944}
4945
4946//
4947// Is \param MO defined by an integer multiply and can be combined?
4949 unsigned MulOpc, unsigned ZeroReg) {
4950 return canCombine(MBB, MO, MulOpc, ZeroReg, true);
4951}
4952
4953//
4954// Is \param MO defined by a floating-point multiply and can be combined?
4956 unsigned MulOpc) {
4957 return canCombine(MBB, MO, MulOpc);
4958}
4959
4960// TODO: There are many more machine instruction opcodes to match:
4961// 1. Other data types (integer, vectors)
4962// 2. Other math / logic operations (xor, or)
4963// 3. Other forms of the same operation (intrinsics and other variants)
4965 bool Invert) const {
4966 if (Invert)
4967 return false;
4968 switch (Inst.getOpcode()) {
4969 // == Floating-point types ==
4970 // -- Floating-point instructions --
4971 case AArch64::FADDHrr:
4972 case AArch64::FADDSrr:
4973 case AArch64::FADDDrr:
4974 case AArch64::FMULHrr:
4975 case AArch64::FMULSrr:
4976 case AArch64::FMULDrr:
4977 case AArch64::FMULX16:
4978 case AArch64::FMULX32:
4979 case AArch64::FMULX64:
4980 // -- Advanced SIMD instructions --
4981 case AArch64::FADDv4f16:
4982 case AArch64::FADDv8f16:
4983 case AArch64::FADDv2f32:
4984 case AArch64::FADDv4f32:
4985 case AArch64::FADDv2f64:
4986 case AArch64::FMULv4f16:
4987 case AArch64::FMULv8f16:
4988 case AArch64::FMULv2f32:
4989 case AArch64::FMULv4f32:
4990 case AArch64::FMULv2f64:
4991 case AArch64::FMULXv4f16:
4992 case AArch64::FMULXv8f16:
4993 case AArch64::FMULXv2f32:
4994 case AArch64::FMULXv4f32:
4995 case AArch64::FMULXv2f64:
4996 // -- SVE instructions --
4997 // Opcodes FMULX_ZZZ_? don't exist because there is no unpredicated FMULX
4998 // in the SVE instruction set (though there are predicated ones).
4999 case AArch64::FADD_ZZZ_H:
5000 case AArch64::FADD_ZZZ_S:
5001 case AArch64::FADD_ZZZ_D:
5002 case AArch64::FMUL_ZZZ_H:
5003 case AArch64::FMUL_ZZZ_S:
5004 case AArch64::FMUL_ZZZ_D:
5005 return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath ||
5008
5009 // == Integer types ==
5010 // -- Base instructions --
5011 // Opcodes MULWrr and MULXrr don't exist because
5012 // `MUL <Wd>, <Wn>, <Wm>` and `MUL <Xd>, <Xn>, <Xm>` are aliases of
5013 // `MADD <Wd>, <Wn>, <Wm>, WZR` and `MADD <Xd>, <Xn>, <Xm>, XZR` respectively.
5014 // The machine-combiner does not support three-source-operands machine
5015 // instruction. So we cannot reassociate MULs.
5016 case AArch64::ADDWrr:
5017 case AArch64::ADDXrr:
5018 case AArch64::ANDWrr:
5019 case AArch64::ANDXrr:
5020 case AArch64::ORRWrr:
5021 case AArch64::ORRXrr:
5022 case AArch64::EORWrr:
5023 case AArch64::EORXrr:
5024 case AArch64::EONWrr:
5025 case AArch64::EONXrr:
5026 // -- Advanced SIMD instructions --
5027 // Opcodes MULv1i64 and MULv2i64 don't exist because there is no 64-bit MUL
5028 // in the Advanced SIMD instruction set.
5029 case AArch64::ADDv8i8:
5030 case AArch64::ADDv16i8:
5031 case AArch64::ADDv4i16:
5032 case AArch64::ADDv8i16:
5033 case AArch64::ADDv2i32:
5034 case AArch64::ADDv4i32:
5035 case AArch64::ADDv1i64:
5036 case AArch64::ADDv2i64:
5037 case AArch64::MULv8i8:
5038 case AArch64::MULv16i8:
5039 case AArch64::MULv4i16:
5040 case AArch64::MULv8i16:
5041 case AArch64::MULv2i32:
5042 case AArch64::MULv4i32:
5043 case AArch64::ANDv8i8:
5044 case AArch64::ANDv16i8:
5045 case AArch64::ORRv8i8:
5046 case AArch64::ORRv16i8:
5047 case AArch64::EORv8i8:
5048 case AArch64::EORv16i8:
5049 // -- SVE instructions --
5050 case AArch64::ADD_ZZZ_B:
5051 case AArch64::ADD_ZZZ_H:
5052 case AArch64::ADD_ZZZ_S:
5053 case AArch64::ADD_ZZZ_D:
5054 case AArch64::MUL_ZZZ_B:
5055 case AArch64::MUL_ZZZ_H:
5056 case AArch64::MUL_ZZZ_S:
5057 case AArch64::MUL_ZZZ_D:
5058 case AArch64::AND_ZZZ:
5059 case AArch64::ORR_ZZZ:
5060 case AArch64::EOR_ZZZ:
5061 return true;
5062
5063 default:
5064 return false;
5065 }
5066}
5067
5068/// Find instructions that can be turned into madd.
5071 unsigned Opc = Root.getOpcode();
5072 MachineBasicBlock &MBB = *Root.getParent();
5073 bool Found = false;
5074
5075 if (!