LLVM 18.0.0git
AArch64AsmPrinter.cpp
Go to the documentation of this file.
1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64.h"
15#include "AArch64MCInstLower.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
29#include "llvm/ADT/StringRef.h"
30#include "llvm/ADT/Twine.h"
43#include "llvm/IR/DataLayout.h"
45#include "llvm/MC/MCAsmInfo.h"
46#include "llvm/MC/MCContext.h"
47#include "llvm/MC/MCInst.h"
50#include "llvm/MC/MCStreamer.h"
51#include "llvm/MC/MCSymbol.h"
59#include <algorithm>
60#include <cassert>
61#include <cstdint>
62#include <map>
63#include <memory>
64
65using namespace llvm;
66
67#define DEBUG_TYPE "asm-printer"
68
69namespace {
70
71class AArch64AsmPrinter : public AsmPrinter {
72 AArch64MCInstLower MCInstLowering;
73 FaultMaps FM;
74 const AArch64Subtarget *STI;
75 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
76
77public:
78 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
79 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
80 FM(*this) {}
81
82 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
83
84 /// Wrapper for MCInstLowering.lowerOperand() for the
85 /// tblgen'erated pseudo lowering.
86 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
87 return MCInstLowering.lowerOperand(MO, MCOp);
88 }
89
90 void emitStartOfAsmFile(Module &M) override;
91 void emitJumpTableInfo() override;
92 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
94 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
95 const MCSymbol *BranchLabel) const override;
96
97 void emitFunctionEntryLabel() override;
98
99 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
100
101 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
102
103 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
104 const MachineInstr &MI);
105 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
106 const MachineInstr &MI);
107 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
108 const MachineInstr &MI);
109 void LowerFAULTING_OP(const MachineInstr &MI);
110
111 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
112 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
113 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
114 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
115
116 typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
117 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
118 void LowerKCFI_CHECK(const MachineInstr &MI);
119 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
120 void emitHwasanMemaccessSymbols(Module &M);
121
122 void emitSled(const MachineInstr &MI, SledKind Kind);
123
124 /// tblgen'erated driver function for lowering simple MI->MC
125 /// pseudo instructions.
126 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
127 const MachineInstr *MI);
128
129 void emitInstruction(const MachineInstr *MI) override;
130
131 void emitFunctionHeaderComment() override;
132
133 void getAnalysisUsage(AnalysisUsage &AU) const override {
135 AU.setPreservesAll();
136 }
137
138 bool runOnMachineFunction(MachineFunction &MF) override {
139 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
140 STI = &MF.getSubtarget<AArch64Subtarget>();
141
143
144 if (STI->isTargetCOFF()) {
145 bool Local = MF.getFunction().hasLocalLinkage();
148 int Type =
150
151 OutStreamer->beginCOFFSymbolDef(CurrentFnSym);
152 OutStreamer->emitCOFFSymbolStorageClass(Scl);
153 OutStreamer->emitCOFFSymbolType(Type);
154 OutStreamer->endCOFFSymbolDef();
155 }
156
157 // Emit the rest of the function body.
159
160 // Emit the XRay table for this function.
162
163 // We didn't modify anything.
164 return false;
165 }
166
167private:
168 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
169 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
170 bool printAsmRegInClass(const MachineOperand &MO,
171 const TargetRegisterClass *RC, unsigned AltName,
172 raw_ostream &O);
173
174 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
175 const char *ExtraCode, raw_ostream &O) override;
176 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
177 const char *ExtraCode, raw_ostream &O) override;
178
179 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
180
181 void emitFunctionBodyEnd() override;
182
183 MCSymbol *GetCPISymbol(unsigned CPID) const override;
184 void emitEndOfAsmFile(Module &M) override;
185
186 AArch64FunctionInfo *AArch64FI = nullptr;
187
188 /// Emit the LOHs contained in AArch64FI.
189 void emitLOHs();
190
191 /// Emit instruction to set float register to zero.
192 void emitFMov0(const MachineInstr &MI);
193
194 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
195
196 MInstToMCSymbol LOHInstToLabel;
197
199 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
200 }
201};
202
203} // end anonymous namespace
204
205void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
206 const Triple &TT = TM.getTargetTriple();
207
208 if (TT.isOSBinFormatCOFF()) {
209 // Emit an absolute @feat.00 symbol
210 MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00"));
211 OutStreamer->beginCOFFSymbolDef(S);
212 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
213 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
214 OutStreamer->endCOFFSymbolDef();
215 int64_t Feat00Value = 0;
216
217 if (M.getModuleFlag("cfguard")) {
218 // Object is CFG-aware.
219 Feat00Value |= COFF::Feat00Flags::GuardCF;
220 }
221
222 if (M.getModuleFlag("ehcontguard")) {
223 // Object also has EHCont.
224 Feat00Value |= COFF::Feat00Flags::GuardEHCont;
225 }
226
227 if (M.getModuleFlag("ms-kernel")) {
228 // Object is compiled with /kernel.
229 Feat00Value |= COFF::Feat00Flags::Kernel;
230 }
231
232 OutStreamer->emitSymbolAttribute(S, MCSA_Global);
233 OutStreamer->emitAssignment(
234 S, MCConstantExpr::create(Feat00Value, MMI->getContext()));
235 }
236
237 if (!TT.isOSBinFormatELF())
238 return;
239
240 // Assemble feature flags that may require creation of a note section.
241 unsigned Flags = 0;
242 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
243 M.getModuleFlag("branch-target-enforcement")))
244 if (BTE->getZExtValue())
246
247 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
248 M.getModuleFlag("sign-return-address")))
249 if (Sign->getZExtValue())
251
252 if (Flags == 0)
253 return;
254
255 // Emit a .note.gnu.property section with the flags.
256 auto *TS =
257 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
258 TS->emitNoteSection(Flags);
259}
260
261void AArch64AsmPrinter::emitFunctionHeaderComment() {
262 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
263 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
264 if (OutlinerString != std::nullopt)
265 OutStreamer->getCommentOS() << ' ' << OutlinerString;
266}
267
268void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
269{
270 const Function &F = MF->getFunction();
271 if (F.hasFnAttribute("patchable-function-entry")) {
272 unsigned Num;
273 if (F.getFnAttribute("patchable-function-entry")
274 .getValueAsString()
275 .getAsInteger(10, Num))
276 return;
277 emitNops(Num);
278 return;
279 }
280
281 emitSled(MI, SledKind::FUNCTION_ENTER);
282}
283
284void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
285 emitSled(MI, SledKind::FUNCTION_EXIT);
286}
287
288void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
289 emitSled(MI, SledKind::TAIL_CALL);
290}
291
292void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
293 static const int8_t NoopsInSledCount = 7;
294 // We want to emit the following pattern:
295 //
296 // .Lxray_sled_N:
297 // ALIGN
298 // B #32
299 // ; 7 NOP instructions (28 bytes)
300 // .tmpN
301 //
302 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
303 // over the full 32 bytes (8 instructions) with the following pattern:
304 //
305 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
306 // LDR W17, #12 ; W17 := function ID
307 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
308 // BLR X16 ; call the tracing trampoline
309 // ;DATA: 32 bits of function ID
310 // ;DATA: lower 32 bits of the address of the trampoline
311 // ;DATA: higher 32 bits of the address of the trampoline
312 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
313 //
314 OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
315 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
316 OutStreamer->emitLabel(CurSled);
317 auto Target = OutContext.createTempSymbol();
318
319 // Emit "B #32" instruction, which jumps over the next 28 bytes.
320 // The operand has to be the number of 4-byte instructions to jump over,
321 // including the current instruction.
322 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
323
324 for (int8_t I = 0; I < NoopsInSledCount; I++)
325 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
326
327 OutStreamer->emitLabel(Target);
328 recordSled(CurSled, MI, Kind, 2);
329}
330
331// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
332// (built-in functions __xray_customevent/__xray_typedevent).
333//
334// .Lxray_event_sled_N:
335// b 1f
336// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
337// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
338// bl __xray_CustomEvent or __xray_TypedEvent
339// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
340// 1:
341//
342// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
343//
344// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
345// After patching, b .+N will become a nop.
346void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
347 bool Typed) {
348 auto &O = *OutStreamer;
349 MCSymbol *CurSled = OutContext.createTempSymbol("xray_sled_", true);
350 O.emitLabel(CurSled);
351 MCInst MovX0Op0 = MCInstBuilder(AArch64::ORRXrs)
352 .addReg(AArch64::X0)
353 .addReg(AArch64::XZR)
354 .addReg(MI.getOperand(0).getReg())
355 .addImm(0);
356 MCInst MovX1Op1 = MCInstBuilder(AArch64::ORRXrs)
357 .addReg(AArch64::X1)
358 .addReg(AArch64::XZR)
359 .addReg(MI.getOperand(1).getReg())
360 .addImm(0);
361 bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
363 OutContext.getOrCreateSymbol(
364 Twine(MachO ? "_" : "") +
365 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
366 OutContext);
367 if (Typed) {
368 O.AddComment("Begin XRay typed event");
369 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(9));
370 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
371 .addReg(AArch64::SP)
372 .addReg(AArch64::X0)
373 .addReg(AArch64::X1)
374 .addReg(AArch64::SP)
375 .addImm(-4));
376 EmitToStreamer(O, MCInstBuilder(AArch64::STRXui)
377 .addReg(AArch64::X2)
378 .addReg(AArch64::SP)
379 .addImm(2));
380 EmitToStreamer(O, MovX0Op0);
381 EmitToStreamer(O, MovX1Op1);
382 EmitToStreamer(O, MCInstBuilder(AArch64::ORRXrs)
383 .addReg(AArch64::X2)
384 .addReg(AArch64::XZR)
385 .addReg(MI.getOperand(2).getReg())
386 .addImm(0));
387 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
388 EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui)
389 .addReg(AArch64::X2)
390 .addReg(AArch64::SP)
391 .addImm(2));
392 O.AddComment("End XRay typed event");
393 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
394 .addReg(AArch64::SP)
395 .addReg(AArch64::X0)
396 .addReg(AArch64::X1)
397 .addReg(AArch64::SP)
398 .addImm(4));
399
400 recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
401 } else {
402 O.AddComment("Begin XRay custom event");
403 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(6));
404 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
405 .addReg(AArch64::SP)
406 .addReg(AArch64::X0)
407 .addReg(AArch64::X1)
408 .addReg(AArch64::SP)
409 .addImm(-2));
410 EmitToStreamer(O, MovX0Op0);
411 EmitToStreamer(O, MovX1Op1);
412 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
413 O.AddComment("End XRay custom event");
414 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
415 .addReg(AArch64::SP)
416 .addReg(AArch64::X0)
417 .addReg(AArch64::X1)
418 .addReg(AArch64::SP)
419 .addImm(2));
420
421 recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
422 }
423}
424
425void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
426 Register AddrReg = MI.getOperand(0).getReg();
427 assert(std::next(MI.getIterator())->isCall() &&
428 "KCFI_CHECK not followed by a call instruction");
429 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
430 "KCFI_CHECK call target doesn't match call operand");
431
432 // Default to using the intra-procedure-call temporary registers for
433 // comparing the hashes.
434 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
435 if (AddrReg == AArch64::XZR) {
436 // Checking XZR makes no sense. Instead of emitting a load, zero
437 // ScratchRegs[0] and use it for the ESR AddrIndex below.
438 AddrReg = getXRegFromWReg(ScratchRegs[0]);
439 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
440 .addReg(AddrReg)
441 .addReg(AArch64::XZR)
442 .addReg(AArch64::XZR)
443 .addImm(0));
444 } else {
445 // If one of the scratch registers is used for the call target (e.g.
446 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
447 // temporary register instead (in this case, AArch64::W9) as the check
448 // is immediately followed by the call instruction.
449 for (auto &Reg : ScratchRegs) {
450 if (Reg == getWRegFromXReg(AddrReg)) {
451 Reg = AArch64::W9;
452 break;
453 }
454 }
455 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
456 "Invalid scratch registers for KCFI_CHECK");
457
458 // Adjust the offset for patchable-function-prefix. This assumes that
459 // patchable-function-prefix is the same for all functions.
460 int64_t PrefixNops = 0;
461 (void)MI.getMF()
462 ->getFunction()
463 .getFnAttribute("patchable-function-prefix")
464 .getValueAsString()
465 .getAsInteger(10, PrefixNops);
466
467 // Load the target function type hash.
468 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi)
469 .addReg(ScratchRegs[0])
470 .addReg(AddrReg)
471 .addImm(-(PrefixNops * 4 + 4)));
472 }
473
474 // Load the expected type hash.
475 const int64_t Type = MI.getOperand(1).getImm();
476 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
477 .addReg(ScratchRegs[1])
478 .addReg(ScratchRegs[1])
479 .addImm(Type & 0xFFFF)
480 .addImm(0));
481 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
482 .addReg(ScratchRegs[1])
483 .addReg(ScratchRegs[1])
484 .addImm((Type >> 16) & 0xFFFF)
485 .addImm(16));
486
487 // Compare the hashes and trap if there's a mismatch.
488 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs)
489 .addReg(AArch64::WZR)
490 .addReg(ScratchRegs[0])
491 .addReg(ScratchRegs[1])
492 .addImm(0));
493
494 MCSymbol *Pass = OutContext.createTempSymbol();
495 EmitToStreamer(*OutStreamer,
496 MCInstBuilder(AArch64::Bcc)
497 .addImm(AArch64CC::EQ)
498 .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
499
500 // The base ESR is 0x8000 and the register information is encoded in bits
501 // 0-9 as follows:
502 // - 0-4: n, where the register Xn contains the target address
503 // - 5-9: m, where the register Wm contains the expected type hash
504 // Where n, m are in [0, 30].
505 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
506 unsigned AddrIndex;
507 switch (AddrReg) {
508 default:
509 AddrIndex = AddrReg - AArch64::X0;
510 break;
511 case AArch64::FP:
512 AddrIndex = 29;
513 break;
514 case AArch64::LR:
515 AddrIndex = 30;
516 break;
517 }
518
519 assert(AddrIndex < 31 && TypeIndex < 31);
520
521 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
522 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR));
523 OutStreamer->emitLabel(Pass);
524}
525
526void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
527 Register Reg = MI.getOperand(0).getReg();
528 bool IsShort =
529 MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
530 uint32_t AccessInfo = MI.getOperand(1).getImm();
531 MCSymbol *&Sym =
532 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
533 if (!Sym) {
534 // FIXME: Make this work on non-ELF.
535 if (!TM.getTargetTriple().isOSBinFormatELF())
536 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
537
538 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
539 utostr(AccessInfo);
540 if (IsShort)
541 SymName += "_short_v2";
542 Sym = OutContext.getOrCreateSymbol(SymName);
543 }
544
545 EmitToStreamer(*OutStreamer,
546 MCInstBuilder(AArch64::BL)
547 .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
548}
549
550void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
551 if (HwasanMemaccessSymbols.empty())
552 return;
553
554 const Triple &TT = TM.getTargetTriple();
555 assert(TT.isOSBinFormatELF());
556 std::unique_ptr<MCSubtargetInfo> STI(
557 TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
558 assert(STI && "Unable to create subtarget info");
559
560 MCSymbol *HwasanTagMismatchV1Sym =
561 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
562 MCSymbol *HwasanTagMismatchV2Sym =
563 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
564
565 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
566 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
567 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
568 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
569
570 for (auto &P : HwasanMemaccessSymbols) {
571 unsigned Reg = std::get<0>(P.first);
572 bool IsShort = std::get<1>(P.first);
573 uint32_t AccessInfo = std::get<2>(P.first);
574 const MCSymbolRefExpr *HwasanTagMismatchRef =
575 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
576 MCSymbol *Sym = P.second;
577
578 bool HasMatchAllTag =
579 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
580 uint8_t MatchAllTag =
581 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
582 unsigned Size =
583 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
584 bool CompileKernel =
585 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
586
587 OutStreamer->switchSection(OutContext.getELFSection(
588 ".text.hot", ELF::SHT_PROGBITS,
590 /*IsComdat=*/true));
591
592 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
593 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
594 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
595 OutStreamer->emitLabel(Sym);
596
597 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
598 .addReg(AArch64::X16)
599 .addReg(Reg)
600 .addImm(4)
601 .addImm(55),
602 *STI);
603 OutStreamer->emitInstruction(
604 MCInstBuilder(AArch64::LDRBBroX)
605 .addReg(AArch64::W16)
606 .addReg(IsShort ? AArch64::X20 : AArch64::X9)
607 .addReg(AArch64::X16)
608 .addImm(0)
609 .addImm(0),
610 *STI);
611 OutStreamer->emitInstruction(
612 MCInstBuilder(AArch64::SUBSXrs)
613 .addReg(AArch64::XZR)
614 .addReg(AArch64::X16)
615 .addReg(Reg)
617 *STI);
618 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
619 OutStreamer->emitInstruction(
620 MCInstBuilder(AArch64::Bcc)
622 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
623 OutContext)),
624 *STI);
625 MCSymbol *ReturnSym = OutContext.createTempSymbol();
626 OutStreamer->emitLabel(ReturnSym);
627 OutStreamer->emitInstruction(
628 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
629 OutStreamer->emitLabel(HandleMismatchOrPartialSym);
630
631 if (HasMatchAllTag) {
632 OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
633 .addReg(AArch64::X17)
634 .addReg(Reg)
635 .addImm(56)
636 .addImm(63),
637 *STI);
638 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
639 .addReg(AArch64::XZR)
640 .addReg(AArch64::X17)
641 .addImm(MatchAllTag)
642 .addImm(0),
643 *STI);
644 OutStreamer->emitInstruction(
645 MCInstBuilder(AArch64::Bcc)
647 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
648 *STI);
649 }
650
651 if (IsShort) {
652 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
653 .addReg(AArch64::WZR)
654 .addReg(AArch64::W16)
655 .addImm(15)
656 .addImm(0),
657 *STI);
658 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
659 OutStreamer->emitInstruction(
660 MCInstBuilder(AArch64::Bcc)
662 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
663 *STI);
664
665 OutStreamer->emitInstruction(
666 MCInstBuilder(AArch64::ANDXri)
667 .addReg(AArch64::X17)
668 .addReg(Reg)
670 *STI);
671 if (Size != 1)
672 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
673 .addReg(AArch64::X17)
674 .addReg(AArch64::X17)
675 .addImm(Size - 1)
676 .addImm(0),
677 *STI);
678 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
679 .addReg(AArch64::WZR)
680 .addReg(AArch64::W16)
681 .addReg(AArch64::W17)
682 .addImm(0),
683 *STI);
684 OutStreamer->emitInstruction(
685 MCInstBuilder(AArch64::Bcc)
687 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
688 *STI);
689
690 OutStreamer->emitInstruction(
691 MCInstBuilder(AArch64::ORRXri)
692 .addReg(AArch64::X16)
693 .addReg(Reg)
695 *STI);
696 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
697 .addReg(AArch64::W16)
698 .addReg(AArch64::X16)
699 .addImm(0),
700 *STI);
701 OutStreamer->emitInstruction(
702 MCInstBuilder(AArch64::SUBSXrs)
703 .addReg(AArch64::XZR)
704 .addReg(AArch64::X16)
705 .addReg(Reg)
707 *STI);
708 OutStreamer->emitInstruction(
709 MCInstBuilder(AArch64::Bcc)
711 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
712 *STI);
713
714 OutStreamer->emitLabel(HandleMismatchSym);
715 }
716
717 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
718 .addReg(AArch64::SP)
719 .addReg(AArch64::X0)
720 .addReg(AArch64::X1)
721 .addReg(AArch64::SP)
722 .addImm(-32),
723 *STI);
724 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
725 .addReg(AArch64::FP)
726 .addReg(AArch64::LR)
727 .addReg(AArch64::SP)
728 .addImm(29),
729 *STI);
730
731 if (Reg != AArch64::X0)
732 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
733 .addReg(AArch64::X0)
734 .addReg(AArch64::XZR)
735 .addReg(Reg)
736 .addImm(0),
737 *STI);
738 OutStreamer->emitInstruction(
739 MCInstBuilder(AArch64::MOVZXi)
740 .addReg(AArch64::X1)
742 .addImm(0),
743 *STI);
744
745 if (CompileKernel) {
746 // The Linux kernel's dynamic loader doesn't support GOT relative
747 // relocations, but it doesn't support late binding either, so just call
748 // the function directly.
749 OutStreamer->emitInstruction(
750 MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
751 } else {
752 // Intentionally load the GOT entry and branch to it, rather than possibly
753 // late binding the function, which may clobber the registers before we
754 // have a chance to save them.
755 OutStreamer->emitInstruction(
756 MCInstBuilder(AArch64::ADRP)
757 .addReg(AArch64::X16)
759 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
760 OutContext)),
761 *STI);
762 OutStreamer->emitInstruction(
763 MCInstBuilder(AArch64::LDRXui)
764 .addReg(AArch64::X16)
765 .addReg(AArch64::X16)
767 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
768 OutContext)),
769 *STI);
770 OutStreamer->emitInstruction(
771 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
772 }
773 }
774}
775
776void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
777 emitHwasanMemaccessSymbols(M);
778
779 const Triple &TT = TM.getTargetTriple();
780 if (TT.isOSBinFormatMachO()) {
781 // Funny Darwin hack: This flag tells the linker that no global symbols
782 // contain code that falls through to other global symbols (e.g. the obvious
783 // implementation of multiple entry points). If this doesn't occur, the
784 // linker can safely perform dead code stripping. Since LLVM never
785 // generates code that does this, it is always safe to set.
786 OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
787 }
788
789 // Emit stack and fault map information.
790 FM.serializeToFaultMapSection();
791
792}
793
794void AArch64AsmPrinter::emitLOHs() {
796
797 for (const auto &D : AArch64FI->getLOHContainer()) {
798 for (const MachineInstr *MI : D.getArgs()) {
799 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
800 assert(LabelIt != LOHInstToLabel.end() &&
801 "Label hasn't been inserted for LOH related instruction");
802 MCArgs.push_back(LabelIt->second);
803 }
804 OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
805 MCArgs.clear();
806 }
807}
808
809void AArch64AsmPrinter::emitFunctionBodyEnd() {
810 if (!AArch64FI->getLOHRelated().empty())
811 emitLOHs();
812}
813
814/// GetCPISymbol - Return the symbol for the specified constant pool entry.
815MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
816 // Darwin uses a linker-private symbol name for constant-pools (to
817 // avoid addends on the relocation?), ELF has no such concept and
818 // uses a normal private symbol.
819 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
820 return OutContext.getOrCreateSymbol(
821 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
822 Twine(getFunctionNumber()) + "_" + Twine(CPID));
823
824 return AsmPrinter::GetCPISymbol(CPID);
825}
826
827void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
828 raw_ostream &O) {
829 const MachineOperand &MO = MI->getOperand(OpNum);
830 switch (MO.getType()) {
831 default:
832 llvm_unreachable("<unknown operand type>");
834 Register Reg = MO.getReg();
835 assert(Reg.isPhysical());
836 assert(!MO.getSubReg() && "Subregs should be eliminated!");
838 break;
839 }
841 O << MO.getImm();
842 break;
843 }
845 PrintSymbolOperand(MO, O);
846 break;
847 }
849 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
850 Sym->print(O, MAI);
851 break;
852 }
853 }
854}
855
856bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
857 raw_ostream &O) {
858 Register Reg = MO.getReg();
859 switch (Mode) {
860 default:
861 return true; // Unknown mode.
862 case 'w':
863 Reg = getWRegFromXReg(Reg);
864 break;
865 case 'x':
866 Reg = getXRegFromWReg(Reg);
867 break;
868 case 't':
870 break;
871 }
872
874 return false;
875}
876
877// Prints the register in MO using class RC using the offset in the
878// new register class. This should not be used for cross class
879// printing.
880bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
881 const TargetRegisterClass *RC,
882 unsigned AltName, raw_ostream &O) {
883 assert(MO.isReg() && "Should only get here with a register!");
884 const TargetRegisterInfo *RI = STI->getRegisterInfo();
885 Register Reg = MO.getReg();
886 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
887 if (!RI->regsOverlap(RegToPrint, Reg))
888 return true;
889 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
890 return false;
891}
892
893bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
894 const char *ExtraCode, raw_ostream &O) {
895 const MachineOperand &MO = MI->getOperand(OpNum);
896
897 // First try the generic code, which knows about modifiers like 'c' and 'n'.
898 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
899 return false;
900
901 // Does this asm operand have a single letter operand modifier?
902 if (ExtraCode && ExtraCode[0]) {
903 if (ExtraCode[1] != 0)
904 return true; // Unknown modifier.
905
906 switch (ExtraCode[0]) {
907 default:
908 return true; // Unknown modifier.
909 case 'w': // Print W register
910 case 'x': // Print X register
911 if (MO.isReg())
912 return printAsmMRegister(MO, ExtraCode[0], O);
913 if (MO.isImm() && MO.getImm() == 0) {
914 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
916 return false;
917 }
918 printOperand(MI, OpNum, O);
919 return false;
920 case 'b': // Print B register.
921 case 'h': // Print H register.
922 case 's': // Print S register.
923 case 'd': // Print D register.
924 case 'q': // Print Q register.
925 case 'z': // Print Z register.
926 if (MO.isReg()) {
927 const TargetRegisterClass *RC;
928 switch (ExtraCode[0]) {
929 case 'b':
930 RC = &AArch64::FPR8RegClass;
931 break;
932 case 'h':
933 RC = &AArch64::FPR16RegClass;
934 break;
935 case 's':
936 RC = &AArch64::FPR32RegClass;
937 break;
938 case 'd':
939 RC = &AArch64::FPR64RegClass;
940 break;
941 case 'q':
942 RC = &AArch64::FPR128RegClass;
943 break;
944 case 'z':
945 RC = &AArch64::ZPRRegClass;
946 break;
947 default:
948 return true;
949 }
950 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
951 }
952 printOperand(MI, OpNum, O);
953 return false;
954 }
955 }
956
957 // According to ARM, we should emit x and v registers unless we have a
958 // modifier.
959 if (MO.isReg()) {
960 Register Reg = MO.getReg();
961
962 // If this is a w or x register, print an x register.
963 if (AArch64::GPR32allRegClass.contains(Reg) ||
964 AArch64::GPR64allRegClass.contains(Reg))
965 return printAsmMRegister(MO, 'x', O);
966
967 // If this is an x register tuple, print an x register.
968 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
969 return printAsmMRegister(MO, 't', O);
970
971 unsigned AltName = AArch64::NoRegAltName;
972 const TargetRegisterClass *RegClass;
973 if (AArch64::ZPRRegClass.contains(Reg)) {
974 RegClass = &AArch64::ZPRRegClass;
975 } else if (AArch64::PPRRegClass.contains(Reg)) {
976 RegClass = &AArch64::PPRRegClass;
977 } else if (AArch64::PNRRegClass.contains(Reg)) {
978 RegClass = &AArch64::PNRRegClass;
979 } else {
980 RegClass = &AArch64::FPR128RegClass;
981 AltName = AArch64::vreg;
982 }
983
984 // If this is a b, h, s, d, or q register, print it as a v register.
985 return printAsmRegInClass(MO, RegClass, AltName, O);
986 }
987
988 printOperand(MI, OpNum, O);
989 return false;
990}
991
992bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
993 unsigned OpNum,
994 const char *ExtraCode,
995 raw_ostream &O) {
996 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
997 return true; // Unknown modifier.
998
999 const MachineOperand &MO = MI->getOperand(OpNum);
1000 assert(MO.isReg() && "unexpected inline asm memory operand");
1001 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
1002 return false;
1003}
1004
1005void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1006 raw_ostream &OS) {
1007 unsigned NOps = MI->getNumOperands();
1008 assert(NOps == 4);
1009 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
1010 // cast away const; DIetc do not take const operands for some reason.
1011 OS << MI->getDebugVariable()->getName();
1012 OS << " <- ";
1013 // Frame address. Currently handles register +- offset only.
1014 assert(MI->isIndirectDebugValue());
1015 OS << '[';
1016 for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(),
1017 MI->debug_operands().end());
1018 I < E; ++I) {
1019 if (I != 0)
1020 OS << ", ";
1021 printOperand(MI, I, OS);
1022 }
1023 OS << ']';
1024 OS << "+";
1025 printOperand(MI, NOps - 2, OS);
1026}
1027
1028void AArch64AsmPrinter::emitJumpTableInfo() {
1029 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1030 if (!MJTI) return;
1031
1032 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
1033 if (JT.empty()) return;
1034
1035 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1036 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM);
1037 OutStreamer->switchSection(ReadOnlySec);
1038
1039 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1040 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
1041 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1042
1043 // If this jump table was deleted, ignore it.
1044 if (JTBBs.empty()) continue;
1045
1046 unsigned Size = AFI->getJumpTableEntrySize(JTI);
1047 emitAlignment(Align(Size));
1048 OutStreamer->emitLabel(GetJTISymbol(JTI));
1049
1050 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1051 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
1052
1053 for (auto *JTBB : JTBBs) {
1054 const MCExpr *Value =
1055 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
1056
1057 // Each entry is:
1058 // .byte/.hword (LBB - Lbase)>>2
1059 // or plain:
1060 // .word LBB - Lbase
1061 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
1062 if (Size != 4)
1064 Value, MCConstantExpr::create(2, OutContext), OutContext);
1065
1066 OutStreamer->emitValue(Value, Size);
1067 }
1068 }
1069}
1070
1071std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1073AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1074 const MachineInstr *BranchInstr,
1075 const MCSymbol *BranchLabel) const {
1076 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1077 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1079 switch (AFI->getJumpTableEntrySize(JTI)) {
1080 case 1:
1081 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1082 break;
1083 case 2:
1084 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1085 break;
1086 case 4:
1087 EntrySize = codeview::JumpTableEntrySize::Int32;
1088 break;
1089 default:
1090 llvm_unreachable("Unexpected jump table entry size");
1091 }
1092 return std::make_tuple(Base, 0, BranchLabel, EntrySize);
1093}
1094
1095void AArch64AsmPrinter::emitFunctionEntryLabel() {
1096 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1097 MF->getFunction().getCallingConv() ==
1099 MF->getInfo<AArch64FunctionInfo>()->isSVECC()) {
1100 auto *TS =
1101 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1102 TS->emitDirectiveVariantPCS(CurrentFnSym);
1103 }
1104
1106}
1107
1108/// Small jump tables contain an unsigned byte or half, representing the offset
1109/// from the lowest-addressed possible destination to the desired basic
1110/// block. Since all instructions are 4-byte aligned, this is further compressed
1111/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1112/// materialize the correct destination we need:
1113///
1114/// adr xDest, .LBB0_0
1115/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1116/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1117void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1118 const llvm::MachineInstr &MI) {
1119 Register DestReg = MI.getOperand(0).getReg();
1120 Register ScratchReg = MI.getOperand(1).getReg();
1121 Register ScratchRegW =
1122 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
1123 Register TableReg = MI.getOperand(2).getReg();
1124 Register EntryReg = MI.getOperand(3).getReg();
1125 int JTIdx = MI.getOperand(4).getIndex();
1126 int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
1127
1128 // This has to be first because the compression pass based its reachability
1129 // calculations on the start of the JumpTableDest instruction.
1130 auto Label =
1131 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
1132
1133 // If we don't already have a symbol to use as the base, use the ADR
1134 // instruction itself.
1135 if (!Label) {
1136 Label = MF->getContext().createTempSymbol();
1137 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
1138 OutStreamer.emitLabel(Label);
1139 }
1140
1141 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
1142 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
1143 .addReg(DestReg)
1144 .addExpr(LabelExpr));
1145
1146 // Load the number of instruction-steps to offset from the label.
1147 unsigned LdrOpcode;
1148 switch (Size) {
1149 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1150 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1151 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1152 default:
1153 llvm_unreachable("Unknown jump table size");
1154 }
1155
1156 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
1157 .addReg(Size == 4 ? ScratchReg : ScratchRegW)
1158 .addReg(TableReg)
1159 .addReg(EntryReg)
1160 .addImm(0)
1161 .addImm(Size == 1 ? 0 : 1));
1162
1163 // Add to the already materialized base label address, multiplying by 4 if
1164 // compressed.
1165 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1166 .addReg(DestReg)
1167 .addReg(DestReg)
1168 .addReg(ScratchReg)
1169 .addImm(Size == 4 ? 0 : 2));
1170}
1171
1172void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1173 const llvm::MachineInstr &MI) {
1174 unsigned Opcode = MI.getOpcode();
1175 assert(STI->hasMOPS());
1176 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1177
1178 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1179 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1180 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1181 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1182 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1183 if (Opcode == AArch64::MOPSMemorySetPseudo)
1184 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1185 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1186 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1187 llvm_unreachable("Unhandled memory operation pseudo");
1188 }();
1189 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1190 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1191
1192 for (auto Op : Ops) {
1193 int i = 0;
1194 auto MCIB = MCInstBuilder(Op);
1195 // Destination registers
1196 MCIB.addReg(MI.getOperand(i++).getReg());
1197 MCIB.addReg(MI.getOperand(i++).getReg());
1198 if (!IsSet)
1199 MCIB.addReg(MI.getOperand(i++).getReg());
1200 // Input registers
1201 MCIB.addReg(MI.getOperand(i++).getReg());
1202 MCIB.addReg(MI.getOperand(i++).getReg());
1203 MCIB.addReg(MI.getOperand(i++).getReg());
1204
1205 EmitToStreamer(OutStreamer, MCIB);
1206 }
1207}
1208
1209void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1210 const MachineInstr &MI) {
1211 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1212
1213 auto &Ctx = OutStreamer.getContext();
1214 MCSymbol *MILabel = Ctx.createTempSymbol();
1215 OutStreamer.emitLabel(MILabel);
1216
1217 SM.recordStackMap(*MILabel, MI);
1218 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1219
1220 // Scan ahead to trim the shadow.
1221 const MachineBasicBlock &MBB = *MI.getParent();
1223 ++MII;
1224 while (NumNOPBytes > 0) {
1225 if (MII == MBB.end() || MII->isCall() ||
1226 MII->getOpcode() == AArch64::DBG_VALUE ||
1227 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1228 MII->getOpcode() == TargetOpcode::STACKMAP)
1229 break;
1230 ++MII;
1231 NumNOPBytes -= 4;
1232 }
1233
1234 // Emit nops.
1235 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1236 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1237}
1238
1239// Lower a patchpoint of the form:
1240// [<def>], <id>, <numBytes>, <target>, <numArgs>
1241void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1242 const MachineInstr &MI) {
1243 auto &Ctx = OutStreamer.getContext();
1244 MCSymbol *MILabel = Ctx.createTempSymbol();
1245 OutStreamer.emitLabel(MILabel);
1246 SM.recordPatchPoint(*MILabel, MI);
1247
1248 PatchPointOpers Opers(&MI);
1249
1250 int64_t CallTarget = Opers.getCallTarget().getImm();
1251 unsigned EncodedBytes = 0;
1252 if (CallTarget) {
1253 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1254 "High 16 bits of call target should be zero.");
1255 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
1256 EncodedBytes = 16;
1257 // Materialize the jump address:
1258 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
1259 .addReg(ScratchReg)
1260 .addImm((CallTarget >> 32) & 0xFFFF)
1261 .addImm(32));
1262 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1263 .addReg(ScratchReg)
1264 .addReg(ScratchReg)
1265 .addImm((CallTarget >> 16) & 0xFFFF)
1266 .addImm(16));
1267 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1268 .addReg(ScratchReg)
1269 .addReg(ScratchReg)
1270 .addImm(CallTarget & 0xFFFF)
1271 .addImm(0));
1272 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1273 }
1274 // Emit padding.
1275 unsigned NumBytes = Opers.getNumPatchBytes();
1276 assert(NumBytes >= EncodedBytes &&
1277 "Patchpoint can't request size less than the length of a call.");
1278 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1279 "Invalid number of NOP bytes requested!");
1280 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1281 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1282}
1283
1284void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1285 const MachineInstr &MI) {
1286 StatepointOpers SOpers(&MI);
1287 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1288 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1289 for (unsigned i = 0; i < PatchBytes; i += 4)
1290 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1291 } else {
1292 // Lower call target and choose correct opcode
1293 const MachineOperand &CallTarget = SOpers.getCallTarget();
1294 MCOperand CallTargetMCOp;
1295 unsigned CallOpcode;
1296 switch (CallTarget.getType()) {
1299 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1300 CallOpcode = AArch64::BL;
1301 break;
1303 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1304 CallOpcode = AArch64::BL;
1305 break;
1307 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1308 CallOpcode = AArch64::BLR;
1309 break;
1310 default:
1311 llvm_unreachable("Unsupported operand type in statepoint call target");
1312 break;
1313 }
1314
1315 EmitToStreamer(OutStreamer,
1316 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1317 }
1318
1319 auto &Ctx = OutStreamer.getContext();
1320 MCSymbol *MILabel = Ctx.createTempSymbol();
1321 OutStreamer.emitLabel(MILabel);
1322 SM.recordStatepoint(*MILabel, MI);
1323}
1324
1325void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1326 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1327 // <opcode>, <operands>
1328
1329 Register DefRegister = FaultingMI.getOperand(0).getReg();
1331 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1332 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1333 unsigned Opcode = FaultingMI.getOperand(3).getImm();
1334 unsigned OperandsBeginIdx = 4;
1335
1336 auto &Ctx = OutStreamer->getContext();
1337 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1338 OutStreamer->emitLabel(FaultingLabel);
1339
1340 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1341 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1342
1343 MCInst MI;
1344 MI.setOpcode(Opcode);
1345
1346 if (DefRegister != (Register)0)
1347 MI.addOperand(MCOperand::createReg(DefRegister));
1348
1349 for (const MachineOperand &MO :
1350 llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx)) {
1351 MCOperand Dest;
1352 lowerOperand(MO, Dest);
1353 MI.addOperand(Dest);
1354 }
1355
1356 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1357 OutStreamer->emitInstruction(MI, getSubtargetInfo());
1358}
1359
1360void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1361 Register DestReg = MI.getOperand(0).getReg();
1362 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
1363 STI->isNeonAvailable()) {
1364 // Convert H/S register to corresponding D register
1365 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1366 DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1367 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1368 DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1369 else
1370 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1371
1372 MCInst MOVI;
1373 MOVI.setOpcode(AArch64::MOVID);
1374 MOVI.addOperand(MCOperand::createReg(DestReg));
1375 MOVI.addOperand(MCOperand::createImm(0));
1376 EmitToStreamer(*OutStreamer, MOVI);
1377 } else {
1378 MCInst FMov;
1379 switch (MI.getOpcode()) {
1380 default: llvm_unreachable("Unexpected opcode");
1381 case AArch64::FMOVH0:
1382 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
1383 if (!STI->hasFullFP16())
1384 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
1385 FMov.addOperand(MCOperand::createReg(DestReg));
1386 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1387 break;
1388 case AArch64::FMOVS0:
1389 FMov.setOpcode(AArch64::FMOVWSr);
1390 FMov.addOperand(MCOperand::createReg(DestReg));
1391 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1392 break;
1393 case AArch64::FMOVD0:
1394 FMov.setOpcode(AArch64::FMOVXDr);
1395 FMov.addOperand(MCOperand::createReg(DestReg));
1396 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1397 break;
1398 }
1399 EmitToStreamer(*OutStreamer, FMov);
1400 }
1401}
1402
1403// Simple pseudo-instructions have their lowering (with expansion to real
1404// instructions) auto-generated.
1405#include "AArch64GenMCPseudoLowering.inc"
1406
1407void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1408 AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
1409
1410 // Do any auto-generated pseudo lowerings.
1411 if (emitPseudoExpansionLowering(*OutStreamer, MI))
1412 return;
1413
1414 if (MI->getOpcode() == AArch64::ADRP) {
1415 for (auto &Opd : MI->operands()) {
1416 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
1417 "swift_async_extendedFramePointerFlags") {
1418 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
1419 }
1420 }
1421 }
1422
1423 if (AArch64FI->getLOHRelated().count(MI)) {
1424 // Generate a label for LOH related instruction
1425 MCSymbol *LOHLabel = createTempSymbol("loh");
1426 // Associate the instruction with the label
1427 LOHInstToLabel[MI] = LOHLabel;
1428 OutStreamer->emitLabel(LOHLabel);
1429 }
1430
1432 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1433 // Do any manual lowerings.
1434 switch (MI->getOpcode()) {
1435 default:
1436 break;
1437 case AArch64::HINT: {
1438 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1439 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1440 // non-empty. If MI is the initial BTI, place the
1441 // __patchable_function_entries label after BTI.
1442 if (CurrentPatchableFunctionEntrySym &&
1443 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1444 MI == &MF->front().front()) {
1445 int64_t Imm = MI->getOperand(0).getImm();
1446 if ((Imm & 32) && (Imm & 6)) {
1447 MCInst Inst;
1448 MCInstLowering.Lower(MI, Inst);
1449 EmitToStreamer(*OutStreamer, Inst);
1450 CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1451 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1452 return;
1453 }
1454 }
1455 break;
1456 }
1457 case AArch64::MOVMCSym: {
1458 Register DestReg = MI->getOperand(0).getReg();
1459 const MachineOperand &MO_Sym = MI->getOperand(1);
1460 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1461 MCOperand Hi_MCSym, Lo_MCSym;
1462
1463 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1464 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1465
1466 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1467 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1468
1469 MCInst MovZ;
1470 MovZ.setOpcode(AArch64::MOVZXi);
1471 MovZ.addOperand(MCOperand::createReg(DestReg));
1472 MovZ.addOperand(Hi_MCSym);
1474 EmitToStreamer(*OutStreamer, MovZ);
1475
1476 MCInst MovK;
1477 MovK.setOpcode(AArch64::MOVKXi);
1478 MovK.addOperand(MCOperand::createReg(DestReg));
1479 MovK.addOperand(MCOperand::createReg(DestReg));
1480 MovK.addOperand(Lo_MCSym);
1482 EmitToStreamer(*OutStreamer, MovK);
1483 return;
1484 }
1485 case AArch64::MOVIv2d_ns:
1486 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
1487 // as movi is more efficient across all cores. Newer cores can eliminate
1488 // fmovs early and there is no difference with movi, but this not true for
1489 // all implementations.
1490 //
1491 // The floating-point version doesn't quite work in rare cases on older
1492 // CPUs, so on those targets we lower this instruction to movi.16b instead.
1493 if (STI->hasZeroCycleZeroingFPWorkaround() &&
1494 MI->getOperand(1).getImm() == 0) {
1495 MCInst TmpInst;
1496 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1497 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1498 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1499 EmitToStreamer(*OutStreamer, TmpInst);
1500 return;
1501 }
1502 break;
1503
1504 case AArch64::DBG_VALUE:
1505 case AArch64::DBG_VALUE_LIST:
1506 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1507 SmallString<128> TmpStr;
1508 raw_svector_ostream OS(TmpStr);
1509 PrintDebugValueComment(MI, OS);
1510 OutStreamer->emitRawText(StringRef(OS.str()));
1511 }
1512 return;
1513
1514 case AArch64::EMITBKEY: {
1515 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1516 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1517 ExceptionHandlingType != ExceptionHandling::ARM)
1518 return;
1519
1520 if (getFunctionCFISectionType(*MF) == CFISection::None)
1521 return;
1522
1523 OutStreamer->emitCFIBKeyFrame();
1524 return;
1525 }
1526
1527 case AArch64::EMITMTETAGGED: {
1528 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1529 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1530 ExceptionHandlingType != ExceptionHandling::ARM)
1531 return;
1532
1533 if (getFunctionCFISectionType(*MF) != CFISection::None)
1534 OutStreamer->emitCFIMTETaggedFrame();
1535 return;
1536 }
1537
1538 // Tail calls use pseudo instructions so they have the proper code-gen
1539 // attributes (isCall, isReturn, etc.). We lower them to the real
1540 // instruction here.
1541 case AArch64::TCRETURNri:
1542 case AArch64::TCRETURNriBTI:
1543 case AArch64::TCRETURNriALL: {
1544 MCInst TmpInst;
1545 TmpInst.setOpcode(AArch64::BR);
1546 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1547 EmitToStreamer(*OutStreamer, TmpInst);
1548 return;
1549 }
1550 case AArch64::TCRETURNdi: {
1551 MCOperand Dest;
1552 MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1553 MCInst TmpInst;
1554 TmpInst.setOpcode(AArch64::B);
1555 TmpInst.addOperand(Dest);
1556 EmitToStreamer(*OutStreamer, TmpInst);
1557 return;
1558 }
1559 case AArch64::SpeculationBarrierISBDSBEndBB: {
1560 // Print DSB SYS + ISB
1561 MCInst TmpInstDSB;
1562 TmpInstDSB.setOpcode(AArch64::DSB);
1563 TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1564 EmitToStreamer(*OutStreamer, TmpInstDSB);
1565 MCInst TmpInstISB;
1566 TmpInstISB.setOpcode(AArch64::ISB);
1567 TmpInstISB.addOperand(MCOperand::createImm(0xf));
1568 EmitToStreamer(*OutStreamer, TmpInstISB);
1569 return;
1570 }
1571 case AArch64::SpeculationBarrierSBEndBB: {
1572 // Print SB
1573 MCInst TmpInstSB;
1574 TmpInstSB.setOpcode(AArch64::SB);
1575 EmitToStreamer(*OutStreamer, TmpInstSB);
1576 return;
1577 }
1578 case AArch64::TLSDESC_CALLSEQ: {
1579 /// lower this to:
1580 /// adrp x0, :tlsdesc:var
1581 /// ldr x1, [x0, #:tlsdesc_lo12:var]
1582 /// add x0, x0, #:tlsdesc_lo12:var
1583 /// .tlsdesccall var
1584 /// blr x1
1585 /// (TPIDR_EL0 offset now in x0)
1586 const MachineOperand &MO_Sym = MI->getOperand(0);
1587 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1588 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1589 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1590 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1591 MCInstLowering.lowerOperand(MO_Sym, Sym);
1592 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1593 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1594
1595 MCInst Adrp;
1596 Adrp.setOpcode(AArch64::ADRP);
1597 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1598 Adrp.addOperand(SymTLSDesc);
1599 EmitToStreamer(*OutStreamer, Adrp);
1600
1601 MCInst Ldr;
1602 if (STI->isTargetILP32()) {
1603 Ldr.setOpcode(AArch64::LDRWui);
1604 Ldr.addOperand(MCOperand::createReg(AArch64::W1));
1605 } else {
1606 Ldr.setOpcode(AArch64::LDRXui);
1607 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1608 }
1609 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1610 Ldr.addOperand(SymTLSDescLo12);
1612 EmitToStreamer(*OutStreamer, Ldr);
1613
1614 MCInst Add;
1615 if (STI->isTargetILP32()) {
1616 Add.setOpcode(AArch64::ADDWri);
1617 Add.addOperand(MCOperand::createReg(AArch64::W0));
1618 Add.addOperand(MCOperand::createReg(AArch64::W0));
1619 } else {
1620 Add.setOpcode(AArch64::ADDXri);
1621 Add.addOperand(MCOperand::createReg(AArch64::X0));
1622 Add.addOperand(MCOperand::createReg(AArch64::X0));
1623 }
1624 Add.addOperand(SymTLSDescLo12);
1626 EmitToStreamer(*OutStreamer, Add);
1627
1628 // Emit a relocation-annotation. This expands to no code, but requests
1629 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1630 MCInst TLSDescCall;
1631 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1632 TLSDescCall.addOperand(Sym);
1633 EmitToStreamer(*OutStreamer, TLSDescCall);
1634
1635 MCInst Blr;
1636 Blr.setOpcode(AArch64::BLR);
1637 Blr.addOperand(MCOperand::createReg(AArch64::X1));
1638 EmitToStreamer(*OutStreamer, Blr);
1639
1640 return;
1641 }
1642
1643 case AArch64::JumpTableDest32:
1644 case AArch64::JumpTableDest16:
1645 case AArch64::JumpTableDest8:
1646 LowerJumpTableDest(*OutStreamer, *MI);
1647 return;
1648
1649 case AArch64::FMOVH0:
1650 case AArch64::FMOVS0:
1651 case AArch64::FMOVD0:
1652 emitFMov0(*MI);
1653 return;
1654
1655 case AArch64::MOPSMemoryCopyPseudo:
1656 case AArch64::MOPSMemoryMovePseudo:
1657 case AArch64::MOPSMemorySetPseudo:
1658 case AArch64::MOPSMemorySetTaggingPseudo:
1659 LowerMOPS(*OutStreamer, *MI);
1660 return;
1661
1662 case TargetOpcode::STACKMAP:
1663 return LowerSTACKMAP(*OutStreamer, SM, *MI);
1664
1665 case TargetOpcode::PATCHPOINT:
1666 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1667
1668 case TargetOpcode::STATEPOINT:
1669 return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1670
1671 case TargetOpcode::FAULTING_OP:
1672 return LowerFAULTING_OP(*MI);
1673
1674 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1675 LowerPATCHABLE_FUNCTION_ENTER(*MI);
1676 return;
1677
1678 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1679 LowerPATCHABLE_FUNCTION_EXIT(*MI);
1680 return;
1681
1682 case TargetOpcode::PATCHABLE_TAIL_CALL:
1683 LowerPATCHABLE_TAIL_CALL(*MI);
1684 return;
1685 case TargetOpcode::PATCHABLE_EVENT_CALL:
1686 return LowerPATCHABLE_EVENT_CALL(*MI, false);
1687 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
1688 return LowerPATCHABLE_EVENT_CALL(*MI, true);
1689
1690 case AArch64::KCFI_CHECK:
1691 LowerKCFI_CHECK(*MI);
1692 return;
1693
1694 case AArch64::HWASAN_CHECK_MEMACCESS:
1695 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1696 LowerHWASAN_CHECK_MEMACCESS(*MI);
1697 return;
1698
1699 case AArch64::SEH_StackAlloc:
1700 TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1701 return;
1702
1703 case AArch64::SEH_SaveFPLR:
1704 TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1705 return;
1706
1707 case AArch64::SEH_SaveFPLR_X:
1708 assert(MI->getOperand(0).getImm() < 0 &&
1709 "Pre increment SEH opcode must have a negative offset");
1710 TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1711 return;
1712
1713 case AArch64::SEH_SaveReg:
1714 TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1715 MI->getOperand(1).getImm());
1716 return;
1717
1718 case AArch64::SEH_SaveReg_X:
1719 assert(MI->getOperand(1).getImm() < 0 &&
1720 "Pre increment SEH opcode must have a negative offset");
1721 TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1722 -MI->getOperand(1).getImm());
1723 return;
1724
1725 case AArch64::SEH_SaveRegP:
1726 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1727 MI->getOperand(0).getImm() <= 28) {
1728 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1729 "Register paired with LR must be odd");
1730 TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1731 MI->getOperand(2).getImm());
1732 return;
1733 }
1734 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1735 "Non-consecutive registers not allowed for save_regp");
1736 TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1737 MI->getOperand(2).getImm());
1738 return;
1739
1740 case AArch64::SEH_SaveRegP_X:
1741 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1742 "Non-consecutive registers not allowed for save_regp_x");
1743 assert(MI->getOperand(2).getImm() < 0 &&
1744 "Pre increment SEH opcode must have a negative offset");
1745 TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1746 -MI->getOperand(2).getImm());
1747 return;
1748
1749 case AArch64::SEH_SaveFReg:
1750 TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1751 MI->getOperand(1).getImm());
1752 return;
1753
1754 case AArch64::SEH_SaveFReg_X:
1755 assert(MI->getOperand(1).getImm() < 0 &&
1756 "Pre increment SEH opcode must have a negative offset");
1757 TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1758 -MI->getOperand(1).getImm());
1759 return;
1760
1761 case AArch64::SEH_SaveFRegP:
1762 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1763 "Non-consecutive registers not allowed for save_regp");
1764 TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1765 MI->getOperand(2).getImm());
1766 return;
1767
1768 case AArch64::SEH_SaveFRegP_X:
1769 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1770 "Non-consecutive registers not allowed for save_regp_x");
1771 assert(MI->getOperand(2).getImm() < 0 &&
1772 "Pre increment SEH opcode must have a negative offset");
1773 TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1774 -MI->getOperand(2).getImm());
1775 return;
1776
1777 case AArch64::SEH_SetFP:
1779 return;
1780
1781 case AArch64::SEH_AddFP:
1782 TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1783 return;
1784
1785 case AArch64::SEH_Nop:
1786 TS->emitARM64WinCFINop();
1787 return;
1788
1789 case AArch64::SEH_PrologEnd:
1791 return;
1792
1793 case AArch64::SEH_EpilogStart:
1795 return;
1796
1797 case AArch64::SEH_EpilogEnd:
1799 return;
1800
1801 case AArch64::SEH_PACSignLR:
1803 return;
1804 }
1805
1806 // Finally, do the automated lowerings for everything else.
1807 MCInst TmpInst;
1808 MCInstLowering.Lower(MI, TmpInst);
1809 EmitToStreamer(*OutStreamer, TmpInst);
1810}
1811
1812// Force static initialization.
1819}
LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter()
MachineBasicBlock & MBB
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
Definition: Compiler.h:135
uint64_t Size
Symbol * Sym
Definition: ELF_riscv.cpp:477
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
static bool printOperand(raw_ostream &OS, const SelectionDAG *G, const SDValue Value)
This file defines the SmallString class.
This file defines the SmallVector class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition: Value.cpp:469
static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, char Mode, raw_ostream &O)
static constexpr uint32_t Opcode
Definition: aarch32.h:200
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
std::optional< std::string > getOutliningStyle() const
static const char * getRegisterName(MCRegister Reg, unsigned AltIdx=AArch64::NoRegAltName)
static const AArch64MCExpr * create(const MCExpr *Expr, VariantKind Kind, MCContext &Ctx)
AArch64MCInstLower - This class is used to lower an MachineInstr into an MCInst.
bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const
virtual void emitARM64WinCFISaveRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveRegPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegPX(unsigned Reg, int Offset)
void emitNoteSection(unsigned Flags)
Callback used to implement the .note.gnu.property section.
virtual void emitARM64WinCFISaveRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFIAllocStack(unsigned Size)
virtual void emitARM64WinCFISaveFPLRX(int Offset)
virtual void emitDirectiveVariantPCS(MCSymbol *Symbol)
Callback used to implement the .variant_pcs directive.
virtual void emitARM64WinCFIAddFP(unsigned Size)
virtual void emitARM64WinCFISaveFPLR(int Offset)
virtual void emitARM64WinCFISaveFRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveLRPair(unsigned Reg, int Offset)
Represent the analysis usage information of a pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
This class is intended to be used as a driving class for all asm writers.
Definition: AsmPrinter.h:84
virtual void emitInstruction(const MachineInstr *)
Targets should implement this to emit instructions.
Definition: AsmPrinter.h:571
void emitXRayTable()
Emit a table with all XRay instrumentation points.
virtual MCSymbol * GetCPISymbol(unsigned CPID) const
Return the symbol for the specified constant pool entry.
virtual void emitJumpTableInfo()
Print assembly representations of the jump tables used by the current function to the current output ...
virtual void SetupMachineFunction(MachineFunction &MF)
This should be called when a new MachineFunction is being processed from runOnMachineFunction.
void emitFunctionBody()
This method emits the body and trailer for a function.
virtual void emitStartOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the start of their fi...
Definition: AsmPrinter.h:547
virtual void emitEndOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the end of their file...
Definition: AsmPrinter.h:551
void getAnalysisUsage(AnalysisUsage &AU) const override
Record analysis usage.
Definition: AsmPrinter.cpp:423
virtual bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const
Definition: AsmPrinter.h:891
bool runOnMachineFunction(MachineFunction &MF) override
Emit the specified function out to the OutStreamer.
Definition: AsmPrinter.h:399
virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant as...
virtual void emitFunctionBodyEnd()
Targets can override this to emit stuff after the last basic block in the function.
Definition: AsmPrinter.h:559
virtual void emitFunctionEntryLabel()
EmitFunctionEntryLabel - Emit the label that is the entrypoint for the function.
virtual std::tuple< const MCSymbol *, uint64_t, const MCSymbol *, codeview::JumpTableEntrySize > getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr, const MCSymbol *BranchLabel) const
Gets information required to create a CodeView debug symbol for a jump table.
virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant.
This class represents an Operation in the Expression.
bool hasLocalLinkage() const
Definition: GlobalValue.h:523
static const MCBinaryExpr * createLShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:608
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:613
static const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Definition: MCExpr.cpp:194
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:35
MCInstBuilder & addReg(unsigned Reg)
Add a new register operand.
Definition: MCInstBuilder.h:31
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Definition: MCInstBuilder.h:37
MCInstBuilder & addExpr(const MCExpr *Val)
Add a new MCExpr operand.
Definition: MCInstBuilder.h:55
Instances of this class represent a single low-level machine instruction.
Definition: MCInst.h:184
void addOperand(const MCOperand Op)
Definition: MCInst.h:210
void setOpcode(unsigned Op)
Definition: MCInst.h:197
Instances of this class represent operands of the MCInst class.
Definition: MCInst.h:36
static MCOperand createReg(unsigned Reg)
Definition: MCInst.h:134
static MCOperand createImm(int64_t Val)
Definition: MCInst.h:141
uint16_t getEncodingValue(MCRegister RegNo) const
Returns the encoding for RegNo.
Instances of this class represent a uniqued identifier for a section in the current translation unit.
Definition: MCSection.h:39
Streaming machine code generation interface.
Definition: MCStreamer.h:212
virtual void emitCFIBKeyFrame()
Definition: MCStreamer.cpp:249
virtual void beginCOFFSymbolDef(const MCSymbol *Symbol)
Start emitting COFF symbol definition.
virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI)
Emit the given Instruction into the current section.
virtual void emitCOFFSymbolType(int Type)
Emit the type of the symbol.
virtual bool hasRawTextSupport() const
Return true if this asm streamer supports emitting unformatted text to the .s file with EmitRawText.
Definition: MCStreamer.h:340
virtual void endCOFFSymbolDef()
Marks the end of the symbol definition.
MCContext & getContext() const
Definition: MCStreamer.h:297
virtual void AddComment(const Twine &T, bool EOL=true)
Add a textual comment.
Definition: MCStreamer.h:359
virtual void emitCFIMTETaggedFrame()
Definition: MCStreamer.cpp:256
virtual void emitLabel(MCSymbol *Symbol, SMLoc Loc=SMLoc())
Emit a label for Symbol into the current section.
Definition: MCStreamer.cpp:424
MCTargetStreamer * getTargetStreamer()
Definition: MCStreamer.h:304
void emitRawText(const Twine &String)
If this file is backed by a assembly streamer, this dumps the specified string in the output ....
virtual void emitCOFFSymbolStorageClass(int StorageClass)
Emit the storage class of the symbol.
Represent a reference to a symbol from inside an expression.
Definition: MCExpr.h:192
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:389
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
StringRef getName() const
getName - Get the symbol name.
Definition: MCSymbol.h:205
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
Definition: MachineInstr.h:68
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:659
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:553
const std::vector< MachineJumpTableEntry > & getJumpTables() const
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
const BlockAddress * getBlockAddress() const
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
@ MO_Immediate
Immediate operand.
@ MO_GlobalAddress
Address of a global value.
@ MO_BlockAddress
Address of a basic block.
@ MO_Register
Register operand.
@ MO_ExternalSymbol
Name of external global symbol.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
MI-level patchpoint operands.
Definition: StackMaps.h:76
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
MI-level stackmap operands.
Definition: StackMaps.h:35
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
Definition: StackMaps.h:50
void recordStatepoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a statepoint instruction.
Definition: StackMaps.cpp:575
void recordPatchPoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a patchpoint instruction.
Definition: StackMaps.cpp:554
void recordStackMap(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a stackmap instruction.
Definition: StackMaps.cpp:544
MI-level Statepoint operands.
Definition: StackMaps.h:158
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
virtual MCSection * getSectionForJumpTable(const Function &F, const TargetMachine &TM) const
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
MCRegister getRegister(unsigned i) const
Return the specified register in the class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an SmallVector or SmallString.
Definition: raw_ostream.h:672
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_S
MO_S - Indicates that the bits of the symbol operand represented by MO_G0 etc are signed.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
SymbolStorageClass
Storage class tells where and what the symbol represents.
Definition: COFF.h:217
@ IMAGE_SYM_CLASS_EXTERNAL
External symbol.
Definition: COFF.h:223
@ IMAGE_SYM_CLASS_STATIC
Static.
Definition: COFF.h:224
@ IMAGE_SYM_DTYPE_NULL
No complex type; simple scalar variable.
Definition: COFF.h:273
@ IMAGE_SYM_DTYPE_FUNCTION
A function that returns a base type.
Definition: COFF.h:275
@ SCT_COMPLEX_TYPE_SHIFT
Type is formed as (base + (derived << SCT_COMPLEX_TYPE_SHIFT))
Definition: COFF.h:279
@ AArch64_VectorCall
Used between AArch64 Advanced SIMD functions.
Definition: CallingConv.h:218
@ AArch64_SVE_VectorCall
Used between AArch64 SVE functions.
Definition: CallingConv.h:221
@ SHT_PROGBITS
Definition: ELF.h:1004
@ GNU_PROPERTY_AARCH64_FEATURE_1_BTI
Definition: ELF.h:1686
@ GNU_PROPERTY_AARCH64_FEATURE_1_PAC
Definition: ELF.h:1687
@ SHF_ALLOC
Definition: ELF.h:1093
@ SHF_GROUP
Definition: ELF.h:1115
@ SHF_EXECINSTR
Definition: ELF.h:1096
Reg
All possible values of the reg field in the ModR/M byte.
constexpr double e
Definition: MathExtras.h:31
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
ExceptionHandling
Target & getTheAArch64beTarget()
Target & getTheAArch64leTarget()
static unsigned getXRegFromWReg(unsigned Reg)
static unsigned getXRegFromXRegTuple(unsigned RegTuple)
Target & getTheAArch64_32Target()
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
Target & getTheARM64_32Target()
@ MCAF_SubsectionsViaSymbols
.subsections_via_symbols (MachO)
Definition: MCDirectives.h:55
@ Add
Sum of integers.
static unsigned getWRegFromXReg(unsigned Reg)
Target & getTheARM64Target()
@ MCSA_Weak
.weak
Definition: MCDirectives.h:45
@ MCSA_Global
.type _foo, @gnu_unique_object
Definition: MCDirectives.h:30
@ MCSA_ELF_TypeFunction
.type _foo, STT_FUNC # aka @function
Definition: MCDirectives.h:23
@ MCSA_Hidden
.hidden (ELF)
Definition: MCDirectives.h:33
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
RegisterAsmPrinter - Helper template for registering a target specific assembly printer,...