67#define DEBUG_TYPE "asm-printer"
75 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags =
false;
78 AArch64AsmPrinter(
TargetMachine &
TM, std::unique_ptr<MCStreamer> Streamer)
79 :
AsmPrinter(
TM, std::move(Streamer)), MCInstLowering(OutContext, *
this),
95 const MCSymbol *BranchLabel)
const override;
114 void LowerPATCHABLE_EVENT_CALL(
const MachineInstr &
MI,
bool Typed);
116 typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
117 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
120 void emitHwasanMemaccessSymbols(
Module &M);
126 bool emitPseudoExpansionLowering(
MCStreamer &OutStreamer,
131 void emitFunctionHeaderComment()
override;
144 if (STI->isTargetCOFF()) {
194 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
196 MInstToMCSymbol LOHInstToLabel;
199 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
205void AArch64AsmPrinter::emitStartOfAsmFile(
Module &M) {
208 if (
TT.isOSBinFormatCOFF()) {
211 OutStreamer->beginCOFFSymbolDef(S);
214 OutStreamer->endCOFFSymbolDef();
215 int64_t Feat00Value = 0;
217 if (
M.getModuleFlag(
"cfguard")) {
219 Feat00Value |= COFF::Feat00Flags::GuardCF;
222 if (
M.getModuleFlag(
"ehcontguard")) {
224 Feat00Value |= COFF::Feat00Flags::GuardEHCont;
227 if (
M.getModuleFlag(
"ms-kernel")) {
229 Feat00Value |= COFF::Feat00Flags::Kernel;
233 OutStreamer->emitAssignment(
237 if (!
TT.isOSBinFormatELF())
242 if (
const auto *BTE = mdconst::extract_or_null<ConstantInt>(
243 M.getModuleFlag(
"branch-target-enforcement")))
244 if (BTE->getZExtValue())
247 if (
const auto *Sign = mdconst::extract_or_null<ConstantInt>(
248 M.getModuleFlag(
"sign-return-address")))
249 if (Sign->getZExtValue())
261void AArch64AsmPrinter::emitFunctionHeaderComment() {
264 if (OutlinerString != std::nullopt)
265 OutStreamer->getCommentOS() <<
' ' << OutlinerString;
268void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(
const MachineInstr &
MI)
271 if (
F.hasFnAttribute(
"patchable-function-entry")) {
273 if (
F.getFnAttribute(
"patchable-function-entry")
275 .getAsInteger(10, Num))
281 emitSled(
MI, SledKind::FUNCTION_ENTER);
284void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(
const MachineInstr &
MI) {
285 emitSled(
MI, SledKind::FUNCTION_EXIT);
288void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(
const MachineInstr &
MI) {
289 emitSled(
MI, SledKind::TAIL_CALL);
292void AArch64AsmPrinter::emitSled(
const MachineInstr &
MI, SledKind Kind) {
293 static const int8_t NoopsInSledCount = 7;
314 OutStreamer->emitCodeAlignment(
Align(4), &getSubtargetInfo());
315 auto CurSled = OutContext.createTempSymbol(
"xray_sled_",
true);
316 OutStreamer->emitLabel(CurSled);
317 auto Target = OutContext.createTempSymbol();
322 EmitToStreamer(*OutStreamer,
MCInstBuilder(AArch64::B).addImm(8));
324 for (int8_t
I = 0;
I < NoopsInSledCount;
I++)
325 EmitToStreamer(*OutStreamer,
MCInstBuilder(AArch64::HINT).addImm(0));
327 OutStreamer->emitLabel(
Target);
328 recordSled(CurSled,
MI, Kind, 2);
346void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(
const MachineInstr &
MI,
348 auto &
O = *OutStreamer;
349 MCSymbol *CurSled = OutContext.createTempSymbol(
"xray_sled_",
true);
350 O.emitLabel(CurSled);
361 bool MachO =
TM.getTargetTriple().isOSBinFormatMachO();
363 OutContext.getOrCreateSymbol(
364 Twine(MachO ?
"_" :
"") +
365 (Typed ?
"__xray_TypedEvent" :
"__xray_CustomEvent")),
368 O.AddComment(
"Begin XRay typed event");
380 EmitToStreamer(O, MovX0Op0);
381 EmitToStreamer(O, MovX1Op1);
384 .addReg(AArch64::XZR)
385 .addReg(
MI.getOperand(2).getReg())
392 O.AddComment(
"End XRay typed event");
400 recordSled(CurSled,
MI, SledKind::TYPED_EVENT, 2);
402 O.AddComment(
"Begin XRay custom event");
410 EmitToStreamer(O, MovX0Op0);
411 EmitToStreamer(O, MovX1Op1);
413 O.AddComment(
"End XRay custom event");
421 recordSled(CurSled,
MI, SledKind::CUSTOM_EVENT, 2);
427 assert(std::next(
MI.getIterator())->isCall() &&
428 "KCFI_CHECK not followed by a call instruction");
429 assert(std::next(
MI.getIterator())->getOperand(0).getReg() == AddrReg &&
430 "KCFI_CHECK call target doesn't match call operand");
434 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
435 if (AddrReg == AArch64::XZR) {
441 .addReg(AArch64::XZR)
442 .addReg(AArch64::XZR)
449 for (
auto &Reg : ScratchRegs) {
455 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
456 "Invalid scratch registers for KCFI_CHECK");
460 int64_t PrefixNops = 0;
463 .getFnAttribute(
"patchable-function-prefix")
465 .getAsInteger(10, PrefixNops);
469 .addReg(ScratchRegs[0])
471 .addImm(-(PrefixNops * 4 + 4)));
475 const int64_t
Type =
MI.getOperand(1).getImm();
477 .addReg(ScratchRegs[1])
478 .addReg(ScratchRegs[1])
479 .addImm(
Type & 0xFFFF)
482 .addReg(ScratchRegs[1])
483 .addReg(ScratchRegs[1])
484 .addImm((
Type >> 16) & 0xFFFF)
489 .addReg(AArch64::WZR)
490 .addReg(ScratchRegs[0])
491 .addReg(ScratchRegs[1])
495 EmitToStreamer(*OutStreamer,
505 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
509 AddrIndex = AddrReg - AArch64::X0;
519 assert(AddrIndex < 31 && TypeIndex < 31);
521 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
522 EmitToStreamer(*OutStreamer,
MCInstBuilder(AArch64::BRK).addImm(ESR));
523 OutStreamer->emitLabel(
Pass);
526void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(
const MachineInstr &
MI) {
529 MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
530 uint32_t AccessInfo =
MI.getOperand(1).getImm();
532 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
535 if (!
TM.getTargetTriple().isOSBinFormatELF())
538 std::string SymName =
"__hwasan_check_x" + utostr(Reg - AArch64::X0) +
"_" +
541 SymName +=
"_short_v2";
542 Sym = OutContext.getOrCreateSymbol(SymName);
545 EmitToStreamer(*OutStreamer,
550void AArch64AsmPrinter::emitHwasanMemaccessSymbols(
Module &M) {
551 if (HwasanMemaccessSymbols.empty())
556 std::unique_ptr<MCSubtargetInfo> STI(
557 TM.getTarget().createMCSubtargetInfo(
TT.str(),
"",
""));
558 assert(STI &&
"Unable to create subtarget info");
561 OutContext.getOrCreateSymbol(
"__hwasan_tag_mismatch");
563 OutContext.getOrCreateSymbol(
"__hwasan_tag_mismatch_v2");
570 for (
auto &
P : HwasanMemaccessSymbols) {
571 unsigned Reg = std::get<0>(
P.first);
572 bool IsShort = std::get<1>(
P.first);
573 uint32_t AccessInfo = std::get<2>(
P.first);
575 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
578 bool HasMatchAllTag =
580 uint8_t MatchAllTag =
587 OutStreamer->switchSection(OutContext.getELFSection(
595 OutStreamer->emitLabel(
Sym);
603 OutStreamer->emitInstruction(
606 .
addReg(IsShort ? AArch64::X20 : AArch64::X9)
611 OutStreamer->emitInstruction(
618 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
619 OutStreamer->emitInstruction(
625 MCSymbol *ReturnSym = OutContext.createTempSymbol();
626 OutStreamer->emitLabel(ReturnSym);
627 OutStreamer->emitInstruction(
629 OutStreamer->emitLabel(HandleMismatchOrPartialSym);
631 if (HasMatchAllTag) {
644 OutStreamer->emitInstruction(
658 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
659 OutStreamer->emitInstruction(
665 OutStreamer->emitInstruction(
684 OutStreamer->emitInstruction(
690 OutStreamer->emitInstruction(
701 OutStreamer->emitInstruction(
708 OutStreamer->emitInstruction(
714 OutStreamer->emitLabel(HandleMismatchSym);
731 if (Reg != AArch64::X0)
738 OutStreamer->emitInstruction(
749 OutStreamer->emitInstruction(
755 OutStreamer->emitInstruction(
759 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
762 OutStreamer->emitInstruction(
767 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
770 OutStreamer->emitInstruction(
776void AArch64AsmPrinter::emitEndOfAsmFile(
Module &M) {
777 emitHwasanMemaccessSymbols(M);
780 if (
TT.isOSBinFormatMachO()) {
790 FM.serializeToFaultMapSection();
794void AArch64AsmPrinter::emitLOHs() {
797 for (
const auto &
D : AArch64FI->getLOHContainer()) {
799 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(
MI);
800 assert(LabelIt != LOHInstToLabel.end() &&
801 "Label hasn't been inserted for LOH related instruction");
804 OutStreamer->emitLOHDirective(
D.getKind(), MCArgs);
809void AArch64AsmPrinter::emitFunctionBodyEnd() {
810 if (!AArch64FI->getLOHRelated().empty())
815MCSymbol *AArch64AsmPrinter::GetCPISymbol(
unsigned CPID)
const {
819 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
820 return OutContext.getOrCreateSymbol(
821 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) +
"CPI" +
822 Twine(getFunctionNumber()) +
"_" +
Twine(CPID));
827void AArch64AsmPrinter::printOperand(
const MachineInstr *
MI,
unsigned OpNum,
845 PrintSymbolOperand(MO, O);
856bool AArch64AsmPrinter::printAsmMRegister(
const MachineOperand &MO,
char Mode,
880bool AArch64AsmPrinter::printAsmRegInClass(
const MachineOperand &MO,
883 assert(MO.
isReg() &&
"Should only get here with a register!");
893bool AArch64AsmPrinter::PrintAsmOperand(
const MachineInstr *
MI,
unsigned OpNum,
902 if (ExtraCode && ExtraCode[0]) {
903 if (ExtraCode[1] != 0)
906 switch (ExtraCode[0]) {
914 unsigned Reg = ExtraCode[0] ==
'w' ? AArch64::WZR : AArch64::XZR;
928 switch (ExtraCode[0]) {
930 RC = &AArch64::FPR8RegClass;
933 RC = &AArch64::FPR16RegClass;
936 RC = &AArch64::FPR32RegClass;
939 RC = &AArch64::FPR64RegClass;
942 RC = &AArch64::FPR128RegClass;
945 RC = &AArch64::ZPRRegClass;
950 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
963 if (AArch64::GPR32allRegClass.
contains(Reg) ||
964 AArch64::GPR64allRegClass.
contains(Reg))
968 if (AArch64::GPR64x8ClassRegClass.
contains(Reg))
971 unsigned AltName = AArch64::NoRegAltName;
973 if (AArch64::ZPRRegClass.
contains(Reg)) {
974 RegClass = &AArch64::ZPRRegClass;
975 }
else if (AArch64::PPRRegClass.
contains(Reg)) {
976 RegClass = &AArch64::PPRRegClass;
977 }
else if (AArch64::PNRRegClass.
contains(Reg)) {
978 RegClass = &AArch64::PNRRegClass;
980 RegClass = &AArch64::FPR128RegClass;
981 AltName = AArch64::vreg;
985 return printAsmRegInClass(MO, RegClass, AltName, O);
992bool AArch64AsmPrinter::PrintAsmMemoryOperand(
const MachineInstr *
MI,
994 const char *ExtraCode,
996 if (ExtraCode && ExtraCode[0] && ExtraCode[0] !=
'a')
1000 assert(MO.
isReg() &&
"unexpected inline asm memory operand");
1005void AArch64AsmPrinter::PrintDebugValueComment(
const MachineInstr *
MI,
1007 unsigned NOps =
MI->getNumOperands();
1009 OS <<
'\t' << MAI->getCommentString() <<
"DEBUG_VALUE: ";
1011 OS <<
MI->getDebugVariable()->getName();
1014 assert(
MI->isIndirectDebugValue());
1016 for (
unsigned I = 0,
E = std::distance(
MI->debug_operands().begin(),
1017 MI->debug_operands().end());
1028void AArch64AsmPrinter::emitJumpTableInfo() {
1033 if (
JT.empty())
return;
1037 OutStreamer->switchSection(ReadOnlySec);
1040 for (
unsigned JTI = 0, e =
JT.size(); JTI !=
e; ++JTI) {
1041 const std::vector<MachineBasicBlock*> &JTBBs =
JT[JTI].MBBs;
1044 if (JTBBs.empty())
continue;
1046 unsigned Size = AFI->getJumpTableEntrySize(JTI);
1048 OutStreamer->emitLabel(GetJTISymbol(JTI));
1050 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1053 for (
auto *JTBB : JTBBs) {
1073AArch64AsmPrinter::getCodeViewJumpTableInfo(
int JTI,
1075 const MCSymbol *BranchLabel)
const {
1077 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1079 switch (AFI->getJumpTableEntrySize(JTI)) {
1081 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1084 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1087 EntrySize = codeview::JumpTableEntrySize::Int32;
1092 return std::make_tuple(
Base, 0, BranchLabel, EntrySize);
1095void AArch64AsmPrinter::emitFunctionEntryLabel() {
1097 MF->getFunction().getCallingConv() ==
1119 Register DestReg =
MI.getOperand(0).getReg();
1120 Register ScratchReg =
MI.getOperand(1).getReg();
1122 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
1123 Register TableReg =
MI.getOperand(2).getReg();
1124 Register EntryReg =
MI.getOperand(3).getReg();
1125 int JTIdx =
MI.getOperand(4).getIndex();
1126 int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
1136 Label = MF->getContext().createTempSymbol();
1137 AArch64FI->setJumpTableEntryInfo(JTIdx,
Size, Label);
1144 .addExpr(LabelExpr));
1149 case 1: LdrOpcode = AArch64::LDRBBroX;
break;
1150 case 2: LdrOpcode = AArch64::LDRHHroX;
break;
1151 case 4: LdrOpcode = AArch64::LDRSWroX;
break;
1157 .addReg(
Size == 4 ? ScratchReg : ScratchRegW)
1161 .addImm(
Size == 1 ? 0 : 1));
1169 .addImm(
Size == 4 ? 0 : 2));
1176 assert(STI->hasMTE() ||
Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1178 const auto Ops = [
Opcode]() -> std::array<unsigned, 3> {
1179 if (
Opcode == AArch64::MOPSMemoryCopyPseudo)
1180 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1181 if (
Opcode == AArch64::MOPSMemoryMovePseudo)
1182 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1183 if (
Opcode == AArch64::MOPSMemorySetPseudo)
1184 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1185 if (
Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1186 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1189 const bool IsSet =
Opcode == AArch64::MOPSMemorySetPseudo ||
1190 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1192 for (
auto Op : Ops) {
1196 MCIB.addReg(
MI.getOperand(i++).getReg());
1197 MCIB.addReg(
MI.getOperand(i++).getReg());
1199 MCIB.addReg(
MI.getOperand(i++).getReg());
1201 MCIB.addReg(
MI.getOperand(i++).getReg());
1202 MCIB.addReg(
MI.getOperand(i++).getReg());
1203 MCIB.addReg(
MI.getOperand(i++).getReg());
1205 EmitToStreamer(OutStreamer, MCIB);
1214 MCSymbol *MILabel = Ctx.createTempSymbol();
1218 assert(NumNOPBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
1224 while (NumNOPBytes > 0) {
1225 if (MII ==
MBB.
end() || MII->isCall() ||
1226 MII->getOpcode() == AArch64::DBG_VALUE ||
1227 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1228 MII->getOpcode() == TargetOpcode::STACKMAP)
1235 for (
unsigned i = 0; i < NumNOPBytes; i += 4)
1236 EmitToStreamer(OutStreamer,
MCInstBuilder(AArch64::HINT).addImm(0));
1244 MCSymbol *MILabel = Ctx.createTempSymbol();
1250 int64_t CallTarget = Opers.getCallTarget().getImm();
1251 unsigned EncodedBytes = 0;
1253 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1254 "High 16 bits of call target should be zero.");
1255 Register ScratchReg =
MI.getOperand(Opers.getNextScratchIdx()).getReg();
1260 .addImm((CallTarget >> 32) & 0xFFFF)
1265 .addImm((CallTarget >> 16) & 0xFFFF)
1270 .addImm(CallTarget & 0xFFFF)
1272 EmitToStreamer(OutStreamer,
MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1275 unsigned NumBytes = Opers.getNumPatchBytes();
1276 assert(NumBytes >= EncodedBytes &&
1277 "Patchpoint can't request size less than the length of a call.");
1278 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1279 "Invalid number of NOP bytes requested!");
1280 for (
unsigned i = EncodedBytes; i < NumBytes; i += 4)
1281 EmitToStreamer(OutStreamer,
MCInstBuilder(AArch64::HINT).addImm(0));
1287 if (
unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1288 assert(PatchBytes % 4 == 0 &&
"Invalid number of NOP bytes requested!");
1289 for (
unsigned i = 0; i < PatchBytes; i += 4)
1290 EmitToStreamer(OutStreamer,
MCInstBuilder(AArch64::HINT).addImm(0));
1295 unsigned CallOpcode;
1296 switch (CallTarget.
getType()) {
1299 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1300 CallOpcode = AArch64::BL;
1304 CallOpcode = AArch64::BL;
1308 CallOpcode = AArch64::BLR;
1315 EmitToStreamer(OutStreamer,
1320 MCSymbol *MILabel = Ctx.createTempSymbol();
1325void AArch64AsmPrinter::LowerFAULTING_OP(
const MachineInstr &FaultingMI) {
1334 unsigned OperandsBeginIdx = 4;
1337 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1341 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1352 lowerOperand(MO, Dest);
1353 MI.addOperand(Dest);
1361 Register DestReg =
MI.getOperand(0).getReg();
1362 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
1363 STI->isNeonAvailable()) {
1365 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1366 DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1367 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1368 DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1370 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1373 MOVI.setOpcode(AArch64::MOVID);
1376 EmitToStreamer(*OutStreamer, MOVI);
1379 switch (
MI.getOpcode()) {
1381 case AArch64::FMOVH0:
1382 FMov.
setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
1383 if (!STI->hasFullFP16())
1384 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
1388 case AArch64::FMOVS0:
1393 case AArch64::FMOVD0:
1399 EmitToStreamer(*OutStreamer, FMov);
1405#include "AArch64GenMCPseudoLowering.inc"
1408 AArch64_MC::verifyInstructionPredicates(
MI->getOpcode(), STI->getFeatureBits());
1411 if (emitPseudoExpansionLowering(*OutStreamer,
MI))
1414 if (
MI->getOpcode() == AArch64::ADRP) {
1415 for (
auto &Opd :
MI->operands()) {
1416 if (Opd.isSymbol() &&
StringRef(Opd.getSymbolName()) ==
1417 "swift_async_extendedFramePointerFlags") {
1418 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags =
true;
1423 if (AArch64FI->getLOHRelated().count(
MI)) {
1425 MCSymbol *LOHLabel = createTempSymbol(
"loh");
1427 LOHInstToLabel[
MI] = LOHLabel;
1434 switch (
MI->getOpcode()) {
1437 case AArch64::HINT: {
1442 if (CurrentPatchableFunctionEntrySym &&
1443 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1444 MI == &MF->front().front()) {
1445 int64_t
Imm =
MI->getOperand(0).getImm();
1446 if ((Imm & 32) && (
Imm & 6)) {
1448 MCInstLowering.Lower(
MI, Inst);
1449 EmitToStreamer(*OutStreamer, Inst);
1450 CurrentPatchableFunctionEntrySym = createTempSymbol(
"patch");
1451 OutStreamer->
emitLabel(CurrentPatchableFunctionEntrySym);
1457 case AArch64::MOVMCSym: {
1458 Register DestReg =
MI->getOperand(0).getReg();
1466 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1467 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1474 EmitToStreamer(*OutStreamer, MovZ);
1482 EmitToStreamer(*OutStreamer, MovK);
1485 case AArch64::MOVIv2d_ns:
1493 if (STI->hasZeroCycleZeroingFPWorkaround() &&
1494 MI->getOperand(1).getImm() == 0) {
1496 TmpInst.
setOpcode(AArch64::MOVIv16b_ns);
1499 EmitToStreamer(*OutStreamer, TmpInst);
1504 case AArch64::DBG_VALUE:
1505 case AArch64::DBG_VALUE_LIST:
1509 PrintDebugValueComment(
MI,
OS);
1514 case AArch64::EMITBKEY: {
1516 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1517 ExceptionHandlingType != ExceptionHandling::ARM)
1520 if (getFunctionCFISectionType(*MF) == CFISection::None)
1527 case AArch64::EMITMTETAGGED: {
1529 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1530 ExceptionHandlingType != ExceptionHandling::ARM)
1533 if (getFunctionCFISectionType(*MF) != CFISection::None)
1541 case AArch64::TCRETURNri:
1542 case AArch64::TCRETURNriBTI:
1543 case AArch64::TCRETURNriALL: {
1547 EmitToStreamer(*OutStreamer, TmpInst);
1550 case AArch64::TCRETURNdi: {
1552 MCInstLowering.lowerOperand(
MI->getOperand(0), Dest);
1556 EmitToStreamer(*OutStreamer, TmpInst);
1559 case AArch64::SpeculationBarrierISBDSBEndBB: {
1564 EmitToStreamer(*OutStreamer, TmpInstDSB);
1568 EmitToStreamer(*OutStreamer, TmpInstISB);
1571 case AArch64::SpeculationBarrierSBEndBB: {
1575 EmitToStreamer(*OutStreamer, TmpInstSB);
1578 case AArch64::TLSDESC_CALLSEQ: {
1591 MCInstLowering.lowerOperand(MO_Sym,
Sym);
1592 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1593 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1599 EmitToStreamer(*OutStreamer, Adrp);
1602 if (STI->isTargetILP32()) {
1612 EmitToStreamer(*OutStreamer, Ldr);
1615 if (STI->isTargetILP32()) {
1616 Add.setOpcode(AArch64::ADDWri);
1620 Add.setOpcode(AArch64::ADDXri);
1624 Add.addOperand(SymTLSDescLo12);
1626 EmitToStreamer(*OutStreamer,
Add);
1631 TLSDescCall.
setOpcode(AArch64::TLSDESCCALL);
1633 EmitToStreamer(*OutStreamer, TLSDescCall);
1638 EmitToStreamer(*OutStreamer, Blr);
1643 case AArch64::JumpTableDest32:
1644 case AArch64::JumpTableDest16:
1645 case AArch64::JumpTableDest8:
1646 LowerJumpTableDest(*OutStreamer, *
MI);
1649 case AArch64::FMOVH0:
1650 case AArch64::FMOVS0:
1651 case AArch64::FMOVD0:
1655 case AArch64::MOPSMemoryCopyPseudo:
1656 case AArch64::MOPSMemoryMovePseudo:
1657 case AArch64::MOPSMemorySetPseudo:
1658 case AArch64::MOPSMemorySetTaggingPseudo:
1659 LowerMOPS(*OutStreamer, *
MI);
1662 case TargetOpcode::STACKMAP:
1663 return LowerSTACKMAP(*OutStreamer, SM, *
MI);
1665 case TargetOpcode::PATCHPOINT:
1666 return LowerPATCHPOINT(*OutStreamer, SM, *
MI);
1668 case TargetOpcode::STATEPOINT:
1669 return LowerSTATEPOINT(*OutStreamer, SM, *
MI);
1671 case TargetOpcode::FAULTING_OP:
1672 return LowerFAULTING_OP(*
MI);
1674 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1675 LowerPATCHABLE_FUNCTION_ENTER(*
MI);
1678 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1679 LowerPATCHABLE_FUNCTION_EXIT(*
MI);
1682 case TargetOpcode::PATCHABLE_TAIL_CALL:
1683 LowerPATCHABLE_TAIL_CALL(*
MI);
1685 case TargetOpcode::PATCHABLE_EVENT_CALL:
1686 return LowerPATCHABLE_EVENT_CALL(*
MI,
false);
1687 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
1688 return LowerPATCHABLE_EVENT_CALL(*
MI,
true);
1690 case AArch64::KCFI_CHECK:
1691 LowerKCFI_CHECK(*
MI);
1694 case AArch64::HWASAN_CHECK_MEMACCESS:
1695 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1696 LowerHWASAN_CHECK_MEMACCESS(*
MI);
1699 case AArch64::SEH_StackAlloc:
1703 case AArch64::SEH_SaveFPLR:
1707 case AArch64::SEH_SaveFPLR_X:
1708 assert(
MI->getOperand(0).getImm() < 0 &&
1709 "Pre increment SEH opcode must have a negative offset");
1713 case AArch64::SEH_SaveReg:
1715 MI->getOperand(1).getImm());
1718 case AArch64::SEH_SaveReg_X:
1719 assert(
MI->getOperand(1).getImm() < 0 &&
1720 "Pre increment SEH opcode must have a negative offset");
1722 -
MI->getOperand(1).getImm());
1725 case AArch64::SEH_SaveRegP:
1726 if (
MI->getOperand(1).getImm() == 30 &&
MI->getOperand(0).getImm() >= 19 &&
1727 MI->getOperand(0).getImm() <= 28) {
1728 assert((
MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1729 "Register paired with LR must be odd");
1731 MI->getOperand(2).getImm());
1734 assert((
MI->getOperand(1).getImm() -
MI->getOperand(0).getImm() == 1) &&
1735 "Non-consecutive registers not allowed for save_regp");
1737 MI->getOperand(2).getImm());
1740 case AArch64::SEH_SaveRegP_X:
1741 assert((
MI->getOperand(1).getImm() -
MI->getOperand(0).getImm() == 1) &&
1742 "Non-consecutive registers not allowed for save_regp_x");
1743 assert(
MI->getOperand(2).getImm() < 0 &&
1744 "Pre increment SEH opcode must have a negative offset");
1746 -
MI->getOperand(2).getImm());
1749 case AArch64::SEH_SaveFReg:
1751 MI->getOperand(1).getImm());
1754 case AArch64::SEH_SaveFReg_X:
1755 assert(
MI->getOperand(1).getImm() < 0 &&
1756 "Pre increment SEH opcode must have a negative offset");
1758 -
MI->getOperand(1).getImm());
1761 case AArch64::SEH_SaveFRegP:
1762 assert((
MI->getOperand(1).getImm() -
MI->getOperand(0).getImm() == 1) &&
1763 "Non-consecutive registers not allowed for save_regp");
1765 MI->getOperand(2).getImm());
1768 case AArch64::SEH_SaveFRegP_X:
1769 assert((
MI->getOperand(1).getImm() -
MI->getOperand(0).getImm() == 1) &&
1770 "Non-consecutive registers not allowed for save_regp_x");
1771 assert(
MI->getOperand(2).getImm() < 0 &&
1772 "Pre increment SEH opcode must have a negative offset");
1774 -
MI->getOperand(2).getImm());
1777 case AArch64::SEH_SetFP:
1781 case AArch64::SEH_AddFP:
1785 case AArch64::SEH_Nop:
1789 case AArch64::SEH_PrologEnd:
1793 case AArch64::SEH_EpilogStart:
1797 case AArch64::SEH_EpilogEnd:
1801 case AArch64::SEH_PACSignLR:
1808 MCInstLowering.Lower(
MI, TmpInst);
1809 EmitToStreamer(*OutStreamer, TmpInst);
LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter()
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool printOperand(raw_ostream &OS, const SelectionDAG *G, const SDValue Value)
This file defines the SmallString class.
This file defines the SmallVector class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, char Mode, raw_ostream &O)
static constexpr uint32_t Opcode
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
std::optional< std::string > getOutliningStyle() const
static const char * getRegisterName(MCRegister Reg, unsigned AltIdx=AArch64::NoRegAltName)
static const AArch64MCExpr * create(const MCExpr *Expr, VariantKind Kind, MCContext &Ctx)
AArch64MCInstLower - This class is used to lower an MachineInstr into an MCInst.
bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const
virtual void emitARM64WinCFISaveRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveRegPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFIPACSignLR()
virtual void emitARM64WinCFISaveFRegPX(unsigned Reg, int Offset)
void emitNoteSection(unsigned Flags)
Callback used to implement the .note.gnu.property section.
virtual void emitARM64WinCFISaveRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFIAllocStack(unsigned Size)
virtual void emitARM64WinCFISaveFPLRX(int Offset)
virtual void emitDirectiveVariantPCS(MCSymbol *Symbol)
Callback used to implement the .variant_pcs directive.
virtual void emitARM64WinCFIAddFP(unsigned Size)
virtual void emitARM64WinCFISaveFPLR(int Offset)
virtual void emitARM64WinCFISaveFRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISetFP()
virtual void emitARM64WinCFIEpilogEnd()
virtual void emitARM64WinCFIPrologEnd()
virtual void emitARM64WinCFISaveReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFINop()
virtual void emitARM64WinCFISaveLRPair(unsigned Reg, int Offset)
virtual void emitARM64WinCFIEpilogStart()
Represent the analysis usage information of a pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
This class is intended to be used as a driving class for all asm writers.
virtual void emitInstruction(const MachineInstr *)
Targets should implement this to emit instructions.
void emitXRayTable()
Emit a table with all XRay instrumentation points.
virtual MCSymbol * GetCPISymbol(unsigned CPID) const
Return the symbol for the specified constant pool entry.
virtual void emitJumpTableInfo()
Print assembly representations of the jump tables used by the current function to the current output ...
virtual void SetupMachineFunction(MachineFunction &MF)
This should be called when a new MachineFunction is being processed from runOnMachineFunction.
void emitFunctionBody()
This method emits the body and trailer for a function.
virtual void emitStartOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the start of their fi...
virtual void emitEndOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the end of their file...
void getAnalysisUsage(AnalysisUsage &AU) const override
Record analysis usage.
virtual bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const
bool runOnMachineFunction(MachineFunction &MF) override
Emit the specified function out to the OutStreamer.
virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant as...
virtual void emitFunctionBodyEnd()
Targets can override this to emit stuff after the last basic block in the function.
virtual void emitFunctionEntryLabel()
EmitFunctionEntryLabel - Emit the label that is the entrypoint for the function.
virtual std::tuple< const MCSymbol *, uint64_t, const MCSymbol *, codeview::JumpTableEntrySize > getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr, const MCSymbol *BranchLabel) const
Gets information required to create a CodeView debug symbol for a jump table.
virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant.
This class represents an Operation in the Expression.
bool hasLocalLinkage() const
static const MCBinaryExpr * createLShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Base class for the full range of assembler expressions which are needed for parsing.
MCInstBuilder & addReg(unsigned Reg)
Add a new register operand.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
MCInstBuilder & addExpr(const MCExpr *Val)
Add a new MCExpr operand.
Instances of this class represent a single low-level machine instruction.
void addOperand(const MCOperand Op)
void setOpcode(unsigned Op)
Instances of this class represent operands of the MCInst class.
static MCOperand createReg(unsigned Reg)
static MCOperand createImm(int64_t Val)
uint16_t getEncodingValue(MCRegister RegNo) const
Returns the encoding for RegNo.
Instances of this class represent a uniqued identifier for a section in the current translation unit.
Streaming machine code generation interface.
virtual void emitCFIBKeyFrame()
virtual void beginCOFFSymbolDef(const MCSymbol *Symbol)
Start emitting COFF symbol definition.
virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI)
Emit the given Instruction into the current section.
virtual void emitCOFFSymbolType(int Type)
Emit the type of the symbol.
virtual bool hasRawTextSupport() const
Return true if this asm streamer supports emitting unformatted text to the .s file with EmitRawText.
virtual void endCOFFSymbolDef()
Marks the end of the symbol definition.
MCContext & getContext() const
virtual void AddComment(const Twine &T, bool EOL=true)
Add a textual comment.
virtual void emitCFIMTETaggedFrame()
virtual void emitLabel(MCSymbol *Symbol, SMLoc Loc=SMLoc())
Emit a label for Symbol into the current section.
MCTargetStreamer * getTargetStreamer()
void emitRawText(const Twine &String)
If this file is backed by a assembly streamer, this dumps the specified string in the output ....
virtual void emitCOFFSymbolStorageClass(int StorageClass)
Emit the storage class of the symbol.
Represent a reference to a symbol from inside an expression.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
StringRef getName() const
getName - Get the symbol name.
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
iterator_range< mop_iterator > operands()
const MachineOperand & getOperand(unsigned i) const
const std::vector< MachineJumpTableEntry > & getJumpTables() const
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
const BlockAddress * getBlockAddress() const
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
@ MO_Immediate
Immediate operand.
@ MO_GlobalAddress
Address of a global value.
@ MO_BlockAddress
Address of a basic block.
@ MO_Register
Register operand.
@ MO_ExternalSymbol
Name of external global symbol.
A Module instance is used to store all the information related to an LLVM module.
Pass interface - Implemented by all 'passes'.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
MI-level patchpoint operands.
Wrapper class representing virtual and physical registers.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
void recordStatepoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a statepoint instruction.
void recordPatchPoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a patchpoint instruction.
void recordStackMap(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a stackmap instruction.
MI-level Statepoint operands.
StringRef - Represent a constant reference to a string, i.e.
virtual MCSection * getSectionForJumpTable(const Function &F, const TargetMachine &TM) const
Primary interface to the complete machine description for the target machine.
MCRegister getRegister(unsigned i) const
Return the specified register in the class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM Value Representation.
This class implements an extremely fast bulk output stream that can only output to a stream.
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_S
MO_S - Indicates that the bits of the symbol operand represented by MO_G0 etc are signed.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
SymbolStorageClass
Storage class tells where and what the symbol represents.
@ IMAGE_SYM_CLASS_EXTERNAL
External symbol.
@ IMAGE_SYM_CLASS_STATIC
Static.
@ IMAGE_SYM_DTYPE_NULL
No complex type; simple scalar variable.
@ IMAGE_SYM_DTYPE_FUNCTION
A function that returns a base type.
@ SCT_COMPLEX_TYPE_SHIFT
Type is formed as (base + (derived << SCT_COMPLEX_TYPE_SHIFT))
@ AArch64_VectorCall
Used between AArch64 Advanced SIMD functions.
@ AArch64_SVE_VectorCall
Used between AArch64 SVE functions.
@ GNU_PROPERTY_AARCH64_FEATURE_1_BTI
@ GNU_PROPERTY_AARCH64_FEATURE_1_PAC
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Target & getTheAArch64beTarget()
Target & getTheAArch64leTarget()
static unsigned getXRegFromWReg(unsigned Reg)
static unsigned getXRegFromXRegTuple(unsigned RegTuple)
Target & getTheAArch64_32Target()
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Target & getTheARM64_32Target()
@ MCAF_SubsectionsViaSymbols
.subsections_via_symbols (MachO)
static unsigned getWRegFromXReg(unsigned Reg)
Target & getTheARM64Target()
@ MCSA_Global
.type _foo, @gnu_unique_object
@ MCSA_ELF_TypeFunction
.type _foo, STT_FUNC # aka @function
@ MCSA_Hidden
.hidden (ELF)
This struct is a compact representation of a valid (non-zero power of two) alignment.
RegisterAsmPrinter - Helper template for registering a target specific assembly printer,...