LLVM 19.0.0git
AArch64AsmPrinter.cpp
Go to the documentation of this file.
1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64.h"
15#include "AArch64MCInstLower.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
29#include "llvm/ADT/StringRef.h"
30#include "llvm/ADT/Twine.h"
44#include "llvm/IR/DataLayout.h"
46#include "llvm/MC/MCAsmInfo.h"
47#include "llvm/MC/MCContext.h"
48#include "llvm/MC/MCInst.h"
52#include "llvm/MC/MCStreamer.h"
53#include "llvm/MC/MCSymbol.h"
62#include <algorithm>
63#include <cassert>
64#include <cstdint>
65#include <map>
66#include <memory>
67
68using namespace llvm;
69
70#define DEBUG_TYPE "asm-printer"
71
72namespace {
73
74class AArch64AsmPrinter : public AsmPrinter {
75 AArch64MCInstLower MCInstLowering;
76 FaultMaps FM;
77 const AArch64Subtarget *STI;
78 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
79
80public:
81 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
82 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
83 FM(*this) {}
84
85 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
86
87 /// Wrapper for MCInstLowering.lowerOperand() for the
88 /// tblgen'erated pseudo lowering.
89 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
90 return MCInstLowering.lowerOperand(MO, MCOp);
91 }
92
93 void emitStartOfAsmFile(Module &M) override;
94 void emitJumpTableInfo() override;
95 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
97 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
98 const MCSymbol *BranchLabel) const override;
99
100 void emitFunctionEntryLabel() override;
101
102 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
103
104 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
105
106 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
107 const MachineInstr &MI);
108 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
109 const MachineInstr &MI);
110 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
111 const MachineInstr &MI);
112 void LowerFAULTING_OP(const MachineInstr &MI);
113
114 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
115 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
116 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
117 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
118
119 typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
120 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
121 void LowerKCFI_CHECK(const MachineInstr &MI);
122 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
123 void emitHwasanMemaccessSymbols(Module &M);
124
125 void emitSled(const MachineInstr &MI, SledKind Kind);
126
127 /// tblgen'erated driver function for lowering simple MI->MC
128 /// pseudo instructions.
129 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
130 const MachineInstr *MI);
131
132 void emitInstruction(const MachineInstr *MI) override;
133
134 void emitFunctionHeaderComment() override;
135
136 void getAnalysisUsage(AnalysisUsage &AU) const override {
138 AU.setPreservesAll();
139 }
140
141 bool runOnMachineFunction(MachineFunction &MF) override {
142 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
143 STI = &MF.getSubtarget<AArch64Subtarget>();
144
146
147 if (STI->isTargetCOFF()) {
148 bool Local = MF.getFunction().hasLocalLinkage();
151 int Type =
153
154 OutStreamer->beginCOFFSymbolDef(CurrentFnSym);
155 OutStreamer->emitCOFFSymbolStorageClass(Scl);
156 OutStreamer->emitCOFFSymbolType(Type);
157 OutStreamer->endCOFFSymbolDef();
158 }
159
160 // Emit the rest of the function body.
162
163 // Emit the XRay table for this function.
165
166 // We didn't modify anything.
167 return false;
168 }
169
170 const MCExpr *lowerConstant(const Constant *CV) override;
171
172private:
173 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
174 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
175 bool printAsmRegInClass(const MachineOperand &MO,
176 const TargetRegisterClass *RC, unsigned AltName,
177 raw_ostream &O);
178
179 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
180 const char *ExtraCode, raw_ostream &O) override;
181 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
182 const char *ExtraCode, raw_ostream &O) override;
183
184 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
185
186 void emitFunctionBodyEnd() override;
187
188 MCSymbol *GetCPISymbol(unsigned CPID) const override;
189 void emitEndOfAsmFile(Module &M) override;
190
191 AArch64FunctionInfo *AArch64FI = nullptr;
192
193 /// Emit the LOHs contained in AArch64FI.
194 void emitLOHs();
195
196 /// Emit instruction to set float register to zero.
197 void emitFMov0(const MachineInstr &MI);
198
199 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
200
201 MInstToMCSymbol LOHInstToLabel;
202
204 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
205 }
206
207 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
208 assert(STI);
209 return STI;
210 }
211 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
212 MCSymbol *LazyPointer) override;
214 MCSymbol *LazyPointer) override;
215};
216
217} // end anonymous namespace
218
219void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
220 const Triple &TT = TM.getTargetTriple();
221
222 if (TT.isOSBinFormatCOFF()) {
223 // Emit an absolute @feat.00 symbol
224 MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00"));
225 OutStreamer->beginCOFFSymbolDef(S);
226 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
227 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
228 OutStreamer->endCOFFSymbolDef();
229 int64_t Feat00Value = 0;
230
231 if (M.getModuleFlag("cfguard")) {
232 // Object is CFG-aware.
233 Feat00Value |= COFF::Feat00Flags::GuardCF;
234 }
235
236 if (M.getModuleFlag("ehcontguard")) {
237 // Object also has EHCont.
238 Feat00Value |= COFF::Feat00Flags::GuardEHCont;
239 }
240
241 if (M.getModuleFlag("ms-kernel")) {
242 // Object is compiled with /kernel.
243 Feat00Value |= COFF::Feat00Flags::Kernel;
244 }
245
246 OutStreamer->emitSymbolAttribute(S, MCSA_Global);
247 OutStreamer->emitAssignment(
248 S, MCConstantExpr::create(Feat00Value, MMI->getContext()));
249 }
250
251 if (!TT.isOSBinFormatELF())
252 return;
253
254 // Assemble feature flags that may require creation of a note section.
255 unsigned Flags = 0;
256 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
257 M.getModuleFlag("branch-target-enforcement")))
258 if (BTE->getZExtValue())
260
261 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
262 M.getModuleFlag("guarded-control-stack")))
263 if (GCS->getZExtValue())
265
266 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
267 M.getModuleFlag("sign-return-address")))
268 if (Sign->getZExtValue())
270
271 if (Flags == 0)
272 return;
273
274 // Emit a .note.gnu.property section with the flags.
275 auto *TS =
276 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
277 TS->emitNoteSection(Flags);
278}
279
280void AArch64AsmPrinter::emitFunctionHeaderComment() {
281 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
282 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
283 if (OutlinerString != std::nullopt)
284 OutStreamer->getCommentOS() << ' ' << OutlinerString;
285}
286
287void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
288{
289 const Function &F = MF->getFunction();
290 if (F.hasFnAttribute("patchable-function-entry")) {
291 unsigned Num;
292 if (F.getFnAttribute("patchable-function-entry")
293 .getValueAsString()
294 .getAsInteger(10, Num))
295 return;
296 emitNops(Num);
297 return;
298 }
299
300 emitSled(MI, SledKind::FUNCTION_ENTER);
301}
302
303void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
304 emitSled(MI, SledKind::FUNCTION_EXIT);
305}
306
307void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
308 emitSled(MI, SledKind::TAIL_CALL);
309}
310
311void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
312 static const int8_t NoopsInSledCount = 7;
313 // We want to emit the following pattern:
314 //
315 // .Lxray_sled_N:
316 // ALIGN
317 // B #32
318 // ; 7 NOP instructions (28 bytes)
319 // .tmpN
320 //
321 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
322 // over the full 32 bytes (8 instructions) with the following pattern:
323 //
324 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
325 // LDR W17, #12 ; W17 := function ID
326 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
327 // BLR X16 ; call the tracing trampoline
328 // ;DATA: 32 bits of function ID
329 // ;DATA: lower 32 bits of the address of the trampoline
330 // ;DATA: higher 32 bits of the address of the trampoline
331 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
332 //
333 OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
334 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
335 OutStreamer->emitLabel(CurSled);
336 auto Target = OutContext.createTempSymbol();
337
338 // Emit "B #32" instruction, which jumps over the next 28 bytes.
339 // The operand has to be the number of 4-byte instructions to jump over,
340 // including the current instruction.
341 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
342
343 for (int8_t I = 0; I < NoopsInSledCount; I++)
344 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
345
346 OutStreamer->emitLabel(Target);
347 recordSled(CurSled, MI, Kind, 2);
348}
349
350// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
351// (built-in functions __xray_customevent/__xray_typedevent).
352//
353// .Lxray_event_sled_N:
354// b 1f
355// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
356// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
357// bl __xray_CustomEvent or __xray_TypedEvent
358// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
359// 1:
360//
361// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
362//
363// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
364// After patching, b .+N will become a nop.
365void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
366 bool Typed) {
367 auto &O = *OutStreamer;
368 MCSymbol *CurSled = OutContext.createTempSymbol("xray_sled_", true);
369 O.emitLabel(CurSled);
370 MCInst MovX0Op0 = MCInstBuilder(AArch64::ORRXrs)
371 .addReg(AArch64::X0)
372 .addReg(AArch64::XZR)
373 .addReg(MI.getOperand(0).getReg())
374 .addImm(0);
375 MCInst MovX1Op1 = MCInstBuilder(AArch64::ORRXrs)
376 .addReg(AArch64::X1)
377 .addReg(AArch64::XZR)
378 .addReg(MI.getOperand(1).getReg())
379 .addImm(0);
380 bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
382 OutContext.getOrCreateSymbol(
383 Twine(MachO ? "_" : "") +
384 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
385 OutContext);
386 if (Typed) {
387 O.AddComment("Begin XRay typed event");
388 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(9));
389 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
390 .addReg(AArch64::SP)
391 .addReg(AArch64::X0)
392 .addReg(AArch64::X1)
393 .addReg(AArch64::SP)
394 .addImm(-4));
395 EmitToStreamer(O, MCInstBuilder(AArch64::STRXui)
396 .addReg(AArch64::X2)
397 .addReg(AArch64::SP)
398 .addImm(2));
399 EmitToStreamer(O, MovX0Op0);
400 EmitToStreamer(O, MovX1Op1);
401 EmitToStreamer(O, MCInstBuilder(AArch64::ORRXrs)
402 .addReg(AArch64::X2)
403 .addReg(AArch64::XZR)
404 .addReg(MI.getOperand(2).getReg())
405 .addImm(0));
406 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
407 EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui)
408 .addReg(AArch64::X2)
409 .addReg(AArch64::SP)
410 .addImm(2));
411 O.AddComment("End XRay typed event");
412 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
413 .addReg(AArch64::SP)
414 .addReg(AArch64::X0)
415 .addReg(AArch64::X1)
416 .addReg(AArch64::SP)
417 .addImm(4));
418
419 recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
420 } else {
421 O.AddComment("Begin XRay custom event");
422 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(6));
423 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
424 .addReg(AArch64::SP)
425 .addReg(AArch64::X0)
426 .addReg(AArch64::X1)
427 .addReg(AArch64::SP)
428 .addImm(-2));
429 EmitToStreamer(O, MovX0Op0);
430 EmitToStreamer(O, MovX1Op1);
431 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
432 O.AddComment("End XRay custom event");
433 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
434 .addReg(AArch64::SP)
435 .addReg(AArch64::X0)
436 .addReg(AArch64::X1)
437 .addReg(AArch64::SP)
438 .addImm(2));
439
440 recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
441 }
442}
443
444void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
445 Register AddrReg = MI.getOperand(0).getReg();
446 assert(std::next(MI.getIterator())->isCall() &&
447 "KCFI_CHECK not followed by a call instruction");
448 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
449 "KCFI_CHECK call target doesn't match call operand");
450
451 // Default to using the intra-procedure-call temporary registers for
452 // comparing the hashes.
453 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
454 if (AddrReg == AArch64::XZR) {
455 // Checking XZR makes no sense. Instead of emitting a load, zero
456 // ScratchRegs[0] and use it for the ESR AddrIndex below.
457 AddrReg = getXRegFromWReg(ScratchRegs[0]);
458 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
459 .addReg(AddrReg)
460 .addReg(AArch64::XZR)
461 .addReg(AArch64::XZR)
462 .addImm(0));
463 } else {
464 // If one of the scratch registers is used for the call target (e.g.
465 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
466 // temporary register instead (in this case, AArch64::W9) as the check
467 // is immediately followed by the call instruction.
468 for (auto &Reg : ScratchRegs) {
469 if (Reg == getWRegFromXReg(AddrReg)) {
470 Reg = AArch64::W9;
471 break;
472 }
473 }
474 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
475 "Invalid scratch registers for KCFI_CHECK");
476
477 // Adjust the offset for patchable-function-prefix. This assumes that
478 // patchable-function-prefix is the same for all functions.
479 int64_t PrefixNops = 0;
480 (void)MI.getMF()
481 ->getFunction()
482 .getFnAttribute("patchable-function-prefix")
483 .getValueAsString()
484 .getAsInteger(10, PrefixNops);
485
486 // Load the target function type hash.
487 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi)
488 .addReg(ScratchRegs[0])
489 .addReg(AddrReg)
490 .addImm(-(PrefixNops * 4 + 4)));
491 }
492
493 // Load the expected type hash.
494 const int64_t Type = MI.getOperand(1).getImm();
495 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
496 .addReg(ScratchRegs[1])
497 .addReg(ScratchRegs[1])
498 .addImm(Type & 0xFFFF)
499 .addImm(0));
500 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
501 .addReg(ScratchRegs[1])
502 .addReg(ScratchRegs[1])
503 .addImm((Type >> 16) & 0xFFFF)
504 .addImm(16));
505
506 // Compare the hashes and trap if there's a mismatch.
507 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs)
508 .addReg(AArch64::WZR)
509 .addReg(ScratchRegs[0])
510 .addReg(ScratchRegs[1])
511 .addImm(0));
512
513 MCSymbol *Pass = OutContext.createTempSymbol();
514 EmitToStreamer(*OutStreamer,
515 MCInstBuilder(AArch64::Bcc)
516 .addImm(AArch64CC::EQ)
517 .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
518
519 // The base ESR is 0x8000 and the register information is encoded in bits
520 // 0-9 as follows:
521 // - 0-4: n, where the register Xn contains the target address
522 // - 5-9: m, where the register Wm contains the expected type hash
523 // Where n, m are in [0, 30].
524 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
525 unsigned AddrIndex;
526 switch (AddrReg) {
527 default:
528 AddrIndex = AddrReg - AArch64::X0;
529 break;
530 case AArch64::FP:
531 AddrIndex = 29;
532 break;
533 case AArch64::LR:
534 AddrIndex = 30;
535 break;
536 }
537
538 assert(AddrIndex < 31 && TypeIndex < 31);
539
540 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
541 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR));
542 OutStreamer->emitLabel(Pass);
543}
544
545void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
546 Register Reg = MI.getOperand(0).getReg();
547 bool IsShort =
548 MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
549 uint32_t AccessInfo = MI.getOperand(1).getImm();
550 MCSymbol *&Sym =
551 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
552 if (!Sym) {
553 // FIXME: Make this work on non-ELF.
554 if (!TM.getTargetTriple().isOSBinFormatELF())
555 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
556
557 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
558 utostr(AccessInfo);
559 if (IsShort)
560 SymName += "_short_v2";
561 Sym = OutContext.getOrCreateSymbol(SymName);
562 }
563
564 EmitToStreamer(*OutStreamer,
565 MCInstBuilder(AArch64::BL)
566 .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
567}
568
569void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
570 if (HwasanMemaccessSymbols.empty())
571 return;
572
573 const Triple &TT = TM.getTargetTriple();
574 assert(TT.isOSBinFormatELF());
575 std::unique_ptr<MCSubtargetInfo> STI(
576 TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
577 assert(STI && "Unable to create subtarget info");
578
579 MCSymbol *HwasanTagMismatchV1Sym =
580 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
581 MCSymbol *HwasanTagMismatchV2Sym =
582 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
583
584 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
585 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
586 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
587 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
588
589 for (auto &P : HwasanMemaccessSymbols) {
590 unsigned Reg = std::get<0>(P.first);
591 bool IsShort = std::get<1>(P.first);
592 uint32_t AccessInfo = std::get<2>(P.first);
593 const MCSymbolRefExpr *HwasanTagMismatchRef =
594 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
595 MCSymbol *Sym = P.second;
596
597 bool HasMatchAllTag =
598 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
599 uint8_t MatchAllTag =
600 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
601 unsigned Size =
602 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
603 bool CompileKernel =
604 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
605
606 OutStreamer->switchSection(OutContext.getELFSection(
607 ".text.hot", ELF::SHT_PROGBITS,
609 /*IsComdat=*/true));
610
611 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
612 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
613 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
614 OutStreamer->emitLabel(Sym);
615
616 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
617 .addReg(AArch64::X16)
618 .addReg(Reg)
619 .addImm(4)
620 .addImm(55),
621 *STI);
622 OutStreamer->emitInstruction(
623 MCInstBuilder(AArch64::LDRBBroX)
624 .addReg(AArch64::W16)
625 .addReg(IsShort ? AArch64::X20 : AArch64::X9)
626 .addReg(AArch64::X16)
627 .addImm(0)
628 .addImm(0),
629 *STI);
630 OutStreamer->emitInstruction(
631 MCInstBuilder(AArch64::SUBSXrs)
632 .addReg(AArch64::XZR)
633 .addReg(AArch64::X16)
634 .addReg(Reg)
636 *STI);
637 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
638 OutStreamer->emitInstruction(
639 MCInstBuilder(AArch64::Bcc)
641 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
642 OutContext)),
643 *STI);
644 MCSymbol *ReturnSym = OutContext.createTempSymbol();
645 OutStreamer->emitLabel(ReturnSym);
646 OutStreamer->emitInstruction(
647 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
648 OutStreamer->emitLabel(HandleMismatchOrPartialSym);
649
650 if (HasMatchAllTag) {
651 OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
652 .addReg(AArch64::X17)
653 .addReg(Reg)
654 .addImm(56)
655 .addImm(63),
656 *STI);
657 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
658 .addReg(AArch64::XZR)
659 .addReg(AArch64::X17)
660 .addImm(MatchAllTag)
661 .addImm(0),
662 *STI);
663 OutStreamer->emitInstruction(
664 MCInstBuilder(AArch64::Bcc)
666 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
667 *STI);
668 }
669
670 if (IsShort) {
671 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
672 .addReg(AArch64::WZR)
673 .addReg(AArch64::W16)
674 .addImm(15)
675 .addImm(0),
676 *STI);
677 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
678 OutStreamer->emitInstruction(
679 MCInstBuilder(AArch64::Bcc)
681 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
682 *STI);
683
684 OutStreamer->emitInstruction(
685 MCInstBuilder(AArch64::ANDXri)
686 .addReg(AArch64::X17)
687 .addReg(Reg)
689 *STI);
690 if (Size != 1)
691 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
692 .addReg(AArch64::X17)
693 .addReg(AArch64::X17)
694 .addImm(Size - 1)
695 .addImm(0),
696 *STI);
697 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
698 .addReg(AArch64::WZR)
699 .addReg(AArch64::W16)
700 .addReg(AArch64::W17)
701 .addImm(0),
702 *STI);
703 OutStreamer->emitInstruction(
704 MCInstBuilder(AArch64::Bcc)
706 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
707 *STI);
708
709 OutStreamer->emitInstruction(
710 MCInstBuilder(AArch64::ORRXri)
711 .addReg(AArch64::X16)
712 .addReg(Reg)
714 *STI);
715 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
716 .addReg(AArch64::W16)
717 .addReg(AArch64::X16)
718 .addImm(0),
719 *STI);
720 OutStreamer->emitInstruction(
721 MCInstBuilder(AArch64::SUBSXrs)
722 .addReg(AArch64::XZR)
723 .addReg(AArch64::X16)
724 .addReg(Reg)
726 *STI);
727 OutStreamer->emitInstruction(
728 MCInstBuilder(AArch64::Bcc)
730 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
731 *STI);
732
733 OutStreamer->emitLabel(HandleMismatchSym);
734 }
735
736 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
737 .addReg(AArch64::SP)
738 .addReg(AArch64::X0)
739 .addReg(AArch64::X1)
740 .addReg(AArch64::SP)
741 .addImm(-32),
742 *STI);
743 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
744 .addReg(AArch64::FP)
745 .addReg(AArch64::LR)
746 .addReg(AArch64::SP)
747 .addImm(29),
748 *STI);
749
750 if (Reg != AArch64::X0)
751 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
752 .addReg(AArch64::X0)
753 .addReg(AArch64::XZR)
754 .addReg(Reg)
755 .addImm(0),
756 *STI);
757 OutStreamer->emitInstruction(
758 MCInstBuilder(AArch64::MOVZXi)
759 .addReg(AArch64::X1)
761 .addImm(0),
762 *STI);
763
764 if (CompileKernel) {
765 // The Linux kernel's dynamic loader doesn't support GOT relative
766 // relocations, but it doesn't support late binding either, so just call
767 // the function directly.
768 OutStreamer->emitInstruction(
769 MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
770 } else {
771 // Intentionally load the GOT entry and branch to it, rather than possibly
772 // late binding the function, which may clobber the registers before we
773 // have a chance to save them.
774 OutStreamer->emitInstruction(
775 MCInstBuilder(AArch64::ADRP)
776 .addReg(AArch64::X16)
778 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
779 OutContext)),
780 *STI);
781 OutStreamer->emitInstruction(
782 MCInstBuilder(AArch64::LDRXui)
783 .addReg(AArch64::X16)
784 .addReg(AArch64::X16)
786 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
787 OutContext)),
788 *STI);
789 OutStreamer->emitInstruction(
790 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
791 }
792 }
793}
794
795void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
796 emitHwasanMemaccessSymbols(M);
797
798 const Triple &TT = TM.getTargetTriple();
799 if (TT.isOSBinFormatMachO()) {
800 // Funny Darwin hack: This flag tells the linker that no global symbols
801 // contain code that falls through to other global symbols (e.g. the obvious
802 // implementation of multiple entry points). If this doesn't occur, the
803 // linker can safely perform dead code stripping. Since LLVM never
804 // generates code that does this, it is always safe to set.
805 OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
806 }
807
808 // Emit stack and fault map information.
809 FM.serializeToFaultMapSection();
810
811}
812
813void AArch64AsmPrinter::emitLOHs() {
815
816 for (const auto &D : AArch64FI->getLOHContainer()) {
817 for (const MachineInstr *MI : D.getArgs()) {
818 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
819 assert(LabelIt != LOHInstToLabel.end() &&
820 "Label hasn't been inserted for LOH related instruction");
821 MCArgs.push_back(LabelIt->second);
822 }
823 OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
824 MCArgs.clear();
825 }
826}
827
828void AArch64AsmPrinter::emitFunctionBodyEnd() {
829 if (!AArch64FI->getLOHRelated().empty())
830 emitLOHs();
831}
832
833/// GetCPISymbol - Return the symbol for the specified constant pool entry.
834MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
835 // Darwin uses a linker-private symbol name for constant-pools (to
836 // avoid addends on the relocation?), ELF has no such concept and
837 // uses a normal private symbol.
838 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
839 return OutContext.getOrCreateSymbol(
840 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
841 Twine(getFunctionNumber()) + "_" + Twine(CPID));
842
843 return AsmPrinter::GetCPISymbol(CPID);
844}
845
846void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
847 raw_ostream &O) {
848 const MachineOperand &MO = MI->getOperand(OpNum);
849 switch (MO.getType()) {
850 default:
851 llvm_unreachable("<unknown operand type>");
853 Register Reg = MO.getReg();
854 assert(Reg.isPhysical());
855 assert(!MO.getSubReg() && "Subregs should be eliminated!");
857 break;
858 }
860 O << MO.getImm();
861 break;
862 }
864 PrintSymbolOperand(MO, O);
865 break;
866 }
868 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
869 Sym->print(O, MAI);
870 break;
871 }
872 }
873}
874
875bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
876 raw_ostream &O) {
877 Register Reg = MO.getReg();
878 switch (Mode) {
879 default:
880 return true; // Unknown mode.
881 case 'w':
882 Reg = getWRegFromXReg(Reg);
883 break;
884 case 'x':
885 Reg = getXRegFromWReg(Reg);
886 break;
887 case 't':
889 break;
890 }
891
893 return false;
894}
895
896// Prints the register in MO using class RC using the offset in the
897// new register class. This should not be used for cross class
898// printing.
899bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
900 const TargetRegisterClass *RC,
901 unsigned AltName, raw_ostream &O) {
902 assert(MO.isReg() && "Should only get here with a register!");
903 const TargetRegisterInfo *RI = STI->getRegisterInfo();
904 Register Reg = MO.getReg();
905 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
906 if (!RI->regsOverlap(RegToPrint, Reg))
907 return true;
908 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
909 return false;
910}
911
912bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
913 const char *ExtraCode, raw_ostream &O) {
914 const MachineOperand &MO = MI->getOperand(OpNum);
915
916 // First try the generic code, which knows about modifiers like 'c' and 'n'.
917 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
918 return false;
919
920 // Does this asm operand have a single letter operand modifier?
921 if (ExtraCode && ExtraCode[0]) {
922 if (ExtraCode[1] != 0)
923 return true; // Unknown modifier.
924
925 switch (ExtraCode[0]) {
926 default:
927 return true; // Unknown modifier.
928 case 'w': // Print W register
929 case 'x': // Print X register
930 if (MO.isReg())
931 return printAsmMRegister(MO, ExtraCode[0], O);
932 if (MO.isImm() && MO.getImm() == 0) {
933 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
935 return false;
936 }
937 printOperand(MI, OpNum, O);
938 return false;
939 case 'b': // Print B register.
940 case 'h': // Print H register.
941 case 's': // Print S register.
942 case 'd': // Print D register.
943 case 'q': // Print Q register.
944 case 'z': // Print Z register.
945 if (MO.isReg()) {
946 const TargetRegisterClass *RC;
947 switch (ExtraCode[0]) {
948 case 'b':
949 RC = &AArch64::FPR8RegClass;
950 break;
951 case 'h':
952 RC = &AArch64::FPR16RegClass;
953 break;
954 case 's':
955 RC = &AArch64::FPR32RegClass;
956 break;
957 case 'd':
958 RC = &AArch64::FPR64RegClass;
959 break;
960 case 'q':
961 RC = &AArch64::FPR128RegClass;
962 break;
963 case 'z':
964 RC = &AArch64::ZPRRegClass;
965 break;
966 default:
967 return true;
968 }
969 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
970 }
971 printOperand(MI, OpNum, O);
972 return false;
973 }
974 }
975
976 // According to ARM, we should emit x and v registers unless we have a
977 // modifier.
978 if (MO.isReg()) {
979 Register Reg = MO.getReg();
980
981 // If this is a w or x register, print an x register.
982 if (AArch64::GPR32allRegClass.contains(Reg) ||
983 AArch64::GPR64allRegClass.contains(Reg))
984 return printAsmMRegister(MO, 'x', O);
985
986 // If this is an x register tuple, print an x register.
987 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
988 return printAsmMRegister(MO, 't', O);
989
990 unsigned AltName = AArch64::NoRegAltName;
991 const TargetRegisterClass *RegClass;
992 if (AArch64::ZPRRegClass.contains(Reg)) {
993 RegClass = &AArch64::ZPRRegClass;
994 } else if (AArch64::PPRRegClass.contains(Reg)) {
995 RegClass = &AArch64::PPRRegClass;
996 } else if (AArch64::PNRRegClass.contains(Reg)) {
997 RegClass = &AArch64::PNRRegClass;
998 } else {
999 RegClass = &AArch64::FPR128RegClass;
1000 AltName = AArch64::vreg;
1001 }
1002
1003 // If this is a b, h, s, d, or q register, print it as a v register.
1004 return printAsmRegInClass(MO, RegClass, AltName, O);
1005 }
1006
1007 printOperand(MI, OpNum, O);
1008 return false;
1009}
1010
1011bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1012 unsigned OpNum,
1013 const char *ExtraCode,
1014 raw_ostream &O) {
1015 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1016 return true; // Unknown modifier.
1017
1018 const MachineOperand &MO = MI->getOperand(OpNum);
1019 assert(MO.isReg() && "unexpected inline asm memory operand");
1020 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
1021 return false;
1022}
1023
1024void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1025 raw_ostream &OS) {
1026 unsigned NOps = MI->getNumOperands();
1027 assert(NOps == 4);
1028 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
1029 // cast away const; DIetc do not take const operands for some reason.
1030 OS << MI->getDebugVariable()->getName();
1031 OS << " <- ";
1032 // Frame address. Currently handles register +- offset only.
1033 assert(MI->isIndirectDebugValue());
1034 OS << '[';
1035 for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(),
1036 MI->debug_operands().end());
1037 I < E; ++I) {
1038 if (I != 0)
1039 OS << ", ";
1040 printOperand(MI, I, OS);
1041 }
1042 OS << ']';
1043 OS << "+";
1044 printOperand(MI, NOps - 2, OS);
1045}
1046
1047void AArch64AsmPrinter::emitJumpTableInfo() {
1048 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1049 if (!MJTI) return;
1050
1051 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
1052 if (JT.empty()) return;
1053
1054 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1055 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM);
1056 OutStreamer->switchSection(ReadOnlySec);
1057
1058 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1059 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
1060 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1061
1062 // If this jump table was deleted, ignore it.
1063 if (JTBBs.empty()) continue;
1064
1065 unsigned Size = AFI->getJumpTableEntrySize(JTI);
1066 emitAlignment(Align(Size));
1067 OutStreamer->emitLabel(GetJTISymbol(JTI));
1068
1069 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1070 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
1071
1072 for (auto *JTBB : JTBBs) {
1073 const MCExpr *Value =
1074 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
1075
1076 // Each entry is:
1077 // .byte/.hword (LBB - Lbase)>>2
1078 // or plain:
1079 // .word LBB - Lbase
1080 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
1081 if (Size != 4)
1083 Value, MCConstantExpr::create(2, OutContext), OutContext);
1084
1085 OutStreamer->emitValue(Value, Size);
1086 }
1087 }
1088}
1089
1090std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1092AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1093 const MachineInstr *BranchInstr,
1094 const MCSymbol *BranchLabel) const {
1095 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1096 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1098 switch (AFI->getJumpTableEntrySize(JTI)) {
1099 case 1:
1100 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1101 break;
1102 case 2:
1103 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1104 break;
1105 case 4:
1106 EntrySize = codeview::JumpTableEntrySize::Int32;
1107 break;
1108 default:
1109 llvm_unreachable("Unexpected jump table entry size");
1110 }
1111 return std::make_tuple(Base, 0, BranchLabel, EntrySize);
1112}
1113
1114void AArch64AsmPrinter::emitFunctionEntryLabel() {
1115 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1116 MF->getFunction().getCallingConv() ==
1118 MF->getInfo<AArch64FunctionInfo>()->isSVECC()) {
1119 auto *TS =
1120 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1121 TS->emitDirectiveVariantPCS(CurrentFnSym);
1122 }
1123
1124 if (TM.getTargetTriple().isWindowsArm64EC() &&
1125 !MF->getFunction().hasLocalLinkage()) {
1126 // For ARM64EC targets, a function definition's name is mangled differently
1127 // from the normal symbol. We emit the alias from the unmangled symbol to
1128 // mangled symbol name here.
1129 if (MDNode *Unmangled =
1130 MF->getFunction().getMetadata("arm64ec_unmangled_name")) {
1132
1133 if (MDNode *ECMangled =
1134 MF->getFunction().getMetadata("arm64ec_ecmangled_name")) {
1135 StringRef UnmangledStr =
1136 cast<MDString>(Unmangled->getOperand(0))->getString();
1137 MCSymbol *UnmangledSym =
1138 MMI->getContext().getOrCreateSymbol(UnmangledStr);
1139 StringRef ECMangledStr =
1140 cast<MDString>(ECMangled->getOperand(0))->getString();
1141 MCSymbol *ECMangledSym =
1142 MMI->getContext().getOrCreateSymbol(ECMangledStr);
1143 OutStreamer->emitSymbolAttribute(UnmangledSym, MCSA_WeakAntiDep);
1144 OutStreamer->emitAssignment(
1145 UnmangledSym,
1147 MMI->getContext()));
1148 OutStreamer->emitSymbolAttribute(ECMangledSym, MCSA_WeakAntiDep);
1149 OutStreamer->emitAssignment(
1150 ECMangledSym,
1152 MMI->getContext()));
1153 return;
1154 } else {
1155 StringRef UnmangledStr =
1156 cast<MDString>(Unmangled->getOperand(0))->getString();
1157 MCSymbol *UnmangledSym =
1158 MMI->getContext().getOrCreateSymbol(UnmangledStr);
1159 OutStreamer->emitSymbolAttribute(UnmangledSym, MCSA_WeakAntiDep);
1160 OutStreamer->emitAssignment(
1161 UnmangledSym,
1163 MMI->getContext()));
1164 return;
1165 }
1166 }
1167 }
1168
1170}
1171
1172/// Small jump tables contain an unsigned byte or half, representing the offset
1173/// from the lowest-addressed possible destination to the desired basic
1174/// block. Since all instructions are 4-byte aligned, this is further compressed
1175/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1176/// materialize the correct destination we need:
1177///
1178/// adr xDest, .LBB0_0
1179/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1180/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1181void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1182 const llvm::MachineInstr &MI) {
1183 Register DestReg = MI.getOperand(0).getReg();
1184 Register ScratchReg = MI.getOperand(1).getReg();
1185 Register ScratchRegW =
1186 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
1187 Register TableReg = MI.getOperand(2).getReg();
1188 Register EntryReg = MI.getOperand(3).getReg();
1189 int JTIdx = MI.getOperand(4).getIndex();
1190 int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
1191
1192 // This has to be first because the compression pass based its reachability
1193 // calculations on the start of the JumpTableDest instruction.
1194 auto Label =
1195 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
1196
1197 // If we don't already have a symbol to use as the base, use the ADR
1198 // instruction itself.
1199 if (!Label) {
1200 Label = MF->getContext().createTempSymbol();
1201 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
1202 OutStreamer.emitLabel(Label);
1203 }
1204
1205 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
1206 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
1207 .addReg(DestReg)
1208 .addExpr(LabelExpr));
1209
1210 // Load the number of instruction-steps to offset from the label.
1211 unsigned LdrOpcode;
1212 switch (Size) {
1213 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1214 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1215 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1216 default:
1217 llvm_unreachable("Unknown jump table size");
1218 }
1219
1220 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
1221 .addReg(Size == 4 ? ScratchReg : ScratchRegW)
1222 .addReg(TableReg)
1223 .addReg(EntryReg)
1224 .addImm(0)
1225 .addImm(Size == 1 ? 0 : 1));
1226
1227 // Add to the already materialized base label address, multiplying by 4 if
1228 // compressed.
1229 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1230 .addReg(DestReg)
1231 .addReg(DestReg)
1232 .addReg(ScratchReg)
1233 .addImm(Size == 4 ? 0 : 2));
1234}
1235
1236void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1237 const llvm::MachineInstr &MI) {
1238 unsigned Opcode = MI.getOpcode();
1239 assert(STI->hasMOPS());
1240 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1241
1242 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1243 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1244 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1245 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1246 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1247 if (Opcode == AArch64::MOPSMemorySetPseudo)
1248 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1249 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1250 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1251 llvm_unreachable("Unhandled memory operation pseudo");
1252 }();
1253 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1254 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1255
1256 for (auto Op : Ops) {
1257 int i = 0;
1258 auto MCIB = MCInstBuilder(Op);
1259 // Destination registers
1260 MCIB.addReg(MI.getOperand(i++).getReg());
1261 MCIB.addReg(MI.getOperand(i++).getReg());
1262 if (!IsSet)
1263 MCIB.addReg(MI.getOperand(i++).getReg());
1264 // Input registers
1265 MCIB.addReg(MI.getOperand(i++).getReg());
1266 MCIB.addReg(MI.getOperand(i++).getReg());
1267 MCIB.addReg(MI.getOperand(i++).getReg());
1268
1269 EmitToStreamer(OutStreamer, MCIB);
1270 }
1271}
1272
1273void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1274 const MachineInstr &MI) {
1275 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1276
1277 auto &Ctx = OutStreamer.getContext();
1278 MCSymbol *MILabel = Ctx.createTempSymbol();
1279 OutStreamer.emitLabel(MILabel);
1280
1281 SM.recordStackMap(*MILabel, MI);
1282 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1283
1284 // Scan ahead to trim the shadow.
1285 const MachineBasicBlock &MBB = *MI.getParent();
1287 ++MII;
1288 while (NumNOPBytes > 0) {
1289 if (MII == MBB.end() || MII->isCall() ||
1290 MII->getOpcode() == AArch64::DBG_VALUE ||
1291 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1292 MII->getOpcode() == TargetOpcode::STACKMAP)
1293 break;
1294 ++MII;
1295 NumNOPBytes -= 4;
1296 }
1297
1298 // Emit nops.
1299 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1300 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1301}
1302
1303// Lower a patchpoint of the form:
1304// [<def>], <id>, <numBytes>, <target>, <numArgs>
1305void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1306 const MachineInstr &MI) {
1307 auto &Ctx = OutStreamer.getContext();
1308 MCSymbol *MILabel = Ctx.createTempSymbol();
1309 OutStreamer.emitLabel(MILabel);
1310 SM.recordPatchPoint(*MILabel, MI);
1311
1312 PatchPointOpers Opers(&MI);
1313
1314 int64_t CallTarget = Opers.getCallTarget().getImm();
1315 unsigned EncodedBytes = 0;
1316 if (CallTarget) {
1317 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1318 "High 16 bits of call target should be zero.");
1319 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
1320 EncodedBytes = 16;
1321 // Materialize the jump address:
1322 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
1323 .addReg(ScratchReg)
1324 .addImm((CallTarget >> 32) & 0xFFFF)
1325 .addImm(32));
1326 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1327 .addReg(ScratchReg)
1328 .addReg(ScratchReg)
1329 .addImm((CallTarget >> 16) & 0xFFFF)
1330 .addImm(16));
1331 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1332 .addReg(ScratchReg)
1333 .addReg(ScratchReg)
1334 .addImm(CallTarget & 0xFFFF)
1335 .addImm(0));
1336 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1337 }
1338 // Emit padding.
1339 unsigned NumBytes = Opers.getNumPatchBytes();
1340 assert(NumBytes >= EncodedBytes &&
1341 "Patchpoint can't request size less than the length of a call.");
1342 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1343 "Invalid number of NOP bytes requested!");
1344 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1345 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1346}
1347
1348void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1349 const MachineInstr &MI) {
1350 StatepointOpers SOpers(&MI);
1351 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1352 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1353 for (unsigned i = 0; i < PatchBytes; i += 4)
1354 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1355 } else {
1356 // Lower call target and choose correct opcode
1357 const MachineOperand &CallTarget = SOpers.getCallTarget();
1358 MCOperand CallTargetMCOp;
1359 unsigned CallOpcode;
1360 switch (CallTarget.getType()) {
1363 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1364 CallOpcode = AArch64::BL;
1365 break;
1367 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1368 CallOpcode = AArch64::BL;
1369 break;
1371 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1372 CallOpcode = AArch64::BLR;
1373 break;
1374 default:
1375 llvm_unreachable("Unsupported operand type in statepoint call target");
1376 break;
1377 }
1378
1379 EmitToStreamer(OutStreamer,
1380 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1381 }
1382
1383 auto &Ctx = OutStreamer.getContext();
1384 MCSymbol *MILabel = Ctx.createTempSymbol();
1385 OutStreamer.emitLabel(MILabel);
1386 SM.recordStatepoint(*MILabel, MI);
1387}
1388
1389void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1390 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1391 // <opcode>, <operands>
1392
1393 Register DefRegister = FaultingMI.getOperand(0).getReg();
1395 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1396 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1397 unsigned Opcode = FaultingMI.getOperand(3).getImm();
1398 unsigned OperandsBeginIdx = 4;
1399
1400 auto &Ctx = OutStreamer->getContext();
1401 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1402 OutStreamer->emitLabel(FaultingLabel);
1403
1404 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1405 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1406
1407 MCInst MI;
1408 MI.setOpcode(Opcode);
1409
1410 if (DefRegister != (Register)0)
1411 MI.addOperand(MCOperand::createReg(DefRegister));
1412
1413 for (const MachineOperand &MO :
1414 llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx)) {
1415 MCOperand Dest;
1416 lowerOperand(MO, Dest);
1417 MI.addOperand(Dest);
1418 }
1419
1420 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1421 OutStreamer->emitInstruction(MI, getSubtargetInfo());
1422}
1423
1424void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1425 Register DestReg = MI.getOperand(0).getReg();
1426 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
1427 STI->isNeonAvailable()) {
1428 // Convert H/S register to corresponding D register
1429 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1430 DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1431 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1432 DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1433 else
1434 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1435
1436 MCInst MOVI;
1437 MOVI.setOpcode(AArch64::MOVID);
1438 MOVI.addOperand(MCOperand::createReg(DestReg));
1439 MOVI.addOperand(MCOperand::createImm(0));
1440 EmitToStreamer(*OutStreamer, MOVI);
1441 } else {
1442 MCInst FMov;
1443 switch (MI.getOpcode()) {
1444 default: llvm_unreachable("Unexpected opcode");
1445 case AArch64::FMOVH0:
1446 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
1447 if (!STI->hasFullFP16())
1448 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
1449 FMov.addOperand(MCOperand::createReg(DestReg));
1450 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1451 break;
1452 case AArch64::FMOVS0:
1453 FMov.setOpcode(AArch64::FMOVWSr);
1454 FMov.addOperand(MCOperand::createReg(DestReg));
1455 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1456 break;
1457 case AArch64::FMOVD0:
1458 FMov.setOpcode(AArch64::FMOVXDr);
1459 FMov.addOperand(MCOperand::createReg(DestReg));
1460 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1461 break;
1462 }
1463 EmitToStreamer(*OutStreamer, FMov);
1464 }
1465}
1466
1467// Simple pseudo-instructions have their lowering (with expansion to real
1468// instructions) auto-generated.
1469#include "AArch64GenMCPseudoLowering.inc"
1470
1471void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1472 AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
1473
1474 // Do any auto-generated pseudo lowerings.
1475 if (emitPseudoExpansionLowering(*OutStreamer, MI))
1476 return;
1477
1478 if (MI->getOpcode() == AArch64::ADRP) {
1479 for (auto &Opd : MI->operands()) {
1480 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
1481 "swift_async_extendedFramePointerFlags") {
1482 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
1483 }
1484 }
1485 }
1486
1487 if (AArch64FI->getLOHRelated().count(MI)) {
1488 // Generate a label for LOH related instruction
1489 MCSymbol *LOHLabel = createTempSymbol("loh");
1490 // Associate the instruction with the label
1491 LOHInstToLabel[MI] = LOHLabel;
1492 OutStreamer->emitLabel(LOHLabel);
1493 }
1494
1496 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1497 // Do any manual lowerings.
1498 switch (MI->getOpcode()) {
1499 default:
1500 break;
1501 case AArch64::HINT: {
1502 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1503 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1504 // non-empty. If MI is the initial BTI, place the
1505 // __patchable_function_entries label after BTI.
1506 if (CurrentPatchableFunctionEntrySym &&
1507 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1508 MI == &MF->front().front()) {
1509 int64_t Imm = MI->getOperand(0).getImm();
1510 if ((Imm & 32) && (Imm & 6)) {
1511 MCInst Inst;
1512 MCInstLowering.Lower(MI, Inst);
1513 EmitToStreamer(*OutStreamer, Inst);
1514 CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1515 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1516 return;
1517 }
1518 }
1519 break;
1520 }
1521 case AArch64::MOVMCSym: {
1522 Register DestReg = MI->getOperand(0).getReg();
1523 const MachineOperand &MO_Sym = MI->getOperand(1);
1524 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1525 MCOperand Hi_MCSym, Lo_MCSym;
1526
1527 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1528 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1529
1530 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1531 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1532
1533 MCInst MovZ;
1534 MovZ.setOpcode(AArch64::MOVZXi);
1535 MovZ.addOperand(MCOperand::createReg(DestReg));
1536 MovZ.addOperand(Hi_MCSym);
1538 EmitToStreamer(*OutStreamer, MovZ);
1539
1540 MCInst MovK;
1541 MovK.setOpcode(AArch64::MOVKXi);
1542 MovK.addOperand(MCOperand::createReg(DestReg));
1543 MovK.addOperand(MCOperand::createReg(DestReg));
1544 MovK.addOperand(Lo_MCSym);
1546 EmitToStreamer(*OutStreamer, MovK);
1547 return;
1548 }
1549 case AArch64::MOVIv2d_ns:
1550 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
1551 // as movi is more efficient across all cores. Newer cores can eliminate
1552 // fmovs early and there is no difference with movi, but this not true for
1553 // all implementations.
1554 //
1555 // The floating-point version doesn't quite work in rare cases on older
1556 // CPUs, so on those targets we lower this instruction to movi.16b instead.
1557 if (STI->hasZeroCycleZeroingFPWorkaround() &&
1558 MI->getOperand(1).getImm() == 0) {
1559 MCInst TmpInst;
1560 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1561 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1562 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1563 EmitToStreamer(*OutStreamer, TmpInst);
1564 return;
1565 }
1566 break;
1567
1568 case AArch64::DBG_VALUE:
1569 case AArch64::DBG_VALUE_LIST:
1570 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1571 SmallString<128> TmpStr;
1572 raw_svector_ostream OS(TmpStr);
1573 PrintDebugValueComment(MI, OS);
1574 OutStreamer->emitRawText(StringRef(OS.str()));
1575 }
1576 return;
1577
1578 case AArch64::EMITBKEY: {
1579 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1580 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1581 ExceptionHandlingType != ExceptionHandling::ARM)
1582 return;
1583
1584 if (getFunctionCFISectionType(*MF) == CFISection::None)
1585 return;
1586
1587 OutStreamer->emitCFIBKeyFrame();
1588 return;
1589 }
1590
1591 case AArch64::EMITMTETAGGED: {
1592 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1593 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1594 ExceptionHandlingType != ExceptionHandling::ARM)
1595 return;
1596
1597 if (getFunctionCFISectionType(*MF) != CFISection::None)
1598 OutStreamer->emitCFIMTETaggedFrame();
1599 return;
1600 }
1601
1602 // Tail calls use pseudo instructions so they have the proper code-gen
1603 // attributes (isCall, isReturn, etc.). We lower them to the real
1604 // instruction here.
1605 case AArch64::TCRETURNri:
1606 case AArch64::TCRETURNrix16x17:
1607 case AArch64::TCRETURNrix17:
1608 case AArch64::TCRETURNrinotx16:
1609 case AArch64::TCRETURNriALL: {
1610 MCInst TmpInst;
1611 TmpInst.setOpcode(AArch64::BR);
1612 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1613 EmitToStreamer(*OutStreamer, TmpInst);
1614 return;
1615 }
1616 case AArch64::TCRETURNdi: {
1617 MCOperand Dest;
1618 MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1619 MCInst TmpInst;
1620 TmpInst.setOpcode(AArch64::B);
1621 TmpInst.addOperand(Dest);
1622 EmitToStreamer(*OutStreamer, TmpInst);
1623 return;
1624 }
1625 case AArch64::SpeculationBarrierISBDSBEndBB: {
1626 // Print DSB SYS + ISB
1627 MCInst TmpInstDSB;
1628 TmpInstDSB.setOpcode(AArch64::DSB);
1629 TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1630 EmitToStreamer(*OutStreamer, TmpInstDSB);
1631 MCInst TmpInstISB;
1632 TmpInstISB.setOpcode(AArch64::ISB);
1633 TmpInstISB.addOperand(MCOperand::createImm(0xf));
1634 EmitToStreamer(*OutStreamer, TmpInstISB);
1635 return;
1636 }
1637 case AArch64::SpeculationBarrierSBEndBB: {
1638 // Print SB
1639 MCInst TmpInstSB;
1640 TmpInstSB.setOpcode(AArch64::SB);
1641 EmitToStreamer(*OutStreamer, TmpInstSB);
1642 return;
1643 }
1644 case AArch64::TLSDESC_CALLSEQ: {
1645 /// lower this to:
1646 /// adrp x0, :tlsdesc:var
1647 /// ldr x1, [x0, #:tlsdesc_lo12:var]
1648 /// add x0, x0, #:tlsdesc_lo12:var
1649 /// .tlsdesccall var
1650 /// blr x1
1651 /// (TPIDR_EL0 offset now in x0)
1652 const MachineOperand &MO_Sym = MI->getOperand(0);
1653 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1654 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1655 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1656 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1657 MCInstLowering.lowerOperand(MO_Sym, Sym);
1658 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1659 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1660
1661 MCInst Adrp;
1662 Adrp.setOpcode(AArch64::ADRP);
1663 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1664 Adrp.addOperand(SymTLSDesc);
1665 EmitToStreamer(*OutStreamer, Adrp);
1666
1667 MCInst Ldr;
1668 if (STI->isTargetILP32()) {
1669 Ldr.setOpcode(AArch64::LDRWui);
1670 Ldr.addOperand(MCOperand::createReg(AArch64::W1));
1671 } else {
1672 Ldr.setOpcode(AArch64::LDRXui);
1673 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1674 }
1675 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1676 Ldr.addOperand(SymTLSDescLo12);
1678 EmitToStreamer(*OutStreamer, Ldr);
1679
1680 MCInst Add;
1681 if (STI->isTargetILP32()) {
1682 Add.setOpcode(AArch64::ADDWri);
1683 Add.addOperand(MCOperand::createReg(AArch64::W0));
1684 Add.addOperand(MCOperand::createReg(AArch64::W0));
1685 } else {
1686 Add.setOpcode(AArch64::ADDXri);
1687 Add.addOperand(MCOperand::createReg(AArch64::X0));
1688 Add.addOperand(MCOperand::createReg(AArch64::X0));
1689 }
1690 Add.addOperand(SymTLSDescLo12);
1692 EmitToStreamer(*OutStreamer, Add);
1693
1694 // Emit a relocation-annotation. This expands to no code, but requests
1695 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1696 MCInst TLSDescCall;
1697 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1698 TLSDescCall.addOperand(Sym);
1699 EmitToStreamer(*OutStreamer, TLSDescCall);
1700
1701 MCInst Blr;
1702 Blr.setOpcode(AArch64::BLR);
1703 Blr.addOperand(MCOperand::createReg(AArch64::X1));
1704 EmitToStreamer(*OutStreamer, Blr);
1705
1706 return;
1707 }
1708
1709 case AArch64::JumpTableDest32:
1710 case AArch64::JumpTableDest16:
1711 case AArch64::JumpTableDest8:
1712 LowerJumpTableDest(*OutStreamer, *MI);
1713 return;
1714
1715 case AArch64::FMOVH0:
1716 case AArch64::FMOVS0:
1717 case AArch64::FMOVD0:
1718 emitFMov0(*MI);
1719 return;
1720
1721 case AArch64::MOPSMemoryCopyPseudo:
1722 case AArch64::MOPSMemoryMovePseudo:
1723 case AArch64::MOPSMemorySetPseudo:
1724 case AArch64::MOPSMemorySetTaggingPseudo:
1725 LowerMOPS(*OutStreamer, *MI);
1726 return;
1727
1728 case TargetOpcode::STACKMAP:
1729 return LowerSTACKMAP(*OutStreamer, SM, *MI);
1730
1731 case TargetOpcode::PATCHPOINT:
1732 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1733
1734 case TargetOpcode::STATEPOINT:
1735 return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1736
1737 case TargetOpcode::FAULTING_OP:
1738 return LowerFAULTING_OP(*MI);
1739
1740 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1741 LowerPATCHABLE_FUNCTION_ENTER(*MI);
1742 return;
1743
1744 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1745 LowerPATCHABLE_FUNCTION_EXIT(*MI);
1746 return;
1747
1748 case TargetOpcode::PATCHABLE_TAIL_CALL:
1749 LowerPATCHABLE_TAIL_CALL(*MI);
1750 return;
1751 case TargetOpcode::PATCHABLE_EVENT_CALL:
1752 return LowerPATCHABLE_EVENT_CALL(*MI, false);
1753 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
1754 return LowerPATCHABLE_EVENT_CALL(*MI, true);
1755
1756 case AArch64::KCFI_CHECK:
1757 LowerKCFI_CHECK(*MI);
1758 return;
1759
1760 case AArch64::HWASAN_CHECK_MEMACCESS:
1761 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1762 LowerHWASAN_CHECK_MEMACCESS(*MI);
1763 return;
1764
1765 case AArch64::SEH_StackAlloc:
1766 TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1767 return;
1768
1769 case AArch64::SEH_SaveFPLR:
1770 TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1771 return;
1772
1773 case AArch64::SEH_SaveFPLR_X:
1774 assert(MI->getOperand(0).getImm() < 0 &&
1775 "Pre increment SEH opcode must have a negative offset");
1776 TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1777 return;
1778
1779 case AArch64::SEH_SaveReg:
1780 TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1781 MI->getOperand(1).getImm());
1782 return;
1783
1784 case AArch64::SEH_SaveReg_X:
1785 assert(MI->getOperand(1).getImm() < 0 &&
1786 "Pre increment SEH opcode must have a negative offset");
1787 TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1788 -MI->getOperand(1).getImm());
1789 return;
1790
1791 case AArch64::SEH_SaveRegP:
1792 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1793 MI->getOperand(0).getImm() <= 28) {
1794 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1795 "Register paired with LR must be odd");
1796 TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1797 MI->getOperand(2).getImm());
1798 return;
1799 }
1800 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1801 "Non-consecutive registers not allowed for save_regp");
1802 TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1803 MI->getOperand(2).getImm());
1804 return;
1805
1806 case AArch64::SEH_SaveRegP_X:
1807 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1808 "Non-consecutive registers not allowed for save_regp_x");
1809 assert(MI->getOperand(2).getImm() < 0 &&
1810 "Pre increment SEH opcode must have a negative offset");
1811 TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1812 -MI->getOperand(2).getImm());
1813 return;
1814
1815 case AArch64::SEH_SaveFReg:
1816 TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1817 MI->getOperand(1).getImm());
1818 return;
1819
1820 case AArch64::SEH_SaveFReg_X:
1821 assert(MI->getOperand(1).getImm() < 0 &&
1822 "Pre increment SEH opcode must have a negative offset");
1823 TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1824 -MI->getOperand(1).getImm());
1825 return;
1826
1827 case AArch64::SEH_SaveFRegP:
1828 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1829 "Non-consecutive registers not allowed for save_regp");
1830 TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1831 MI->getOperand(2).getImm());
1832 return;
1833
1834 case AArch64::SEH_SaveFRegP_X:
1835 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1836 "Non-consecutive registers not allowed for save_regp_x");
1837 assert(MI->getOperand(2).getImm() < 0 &&
1838 "Pre increment SEH opcode must have a negative offset");
1839 TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1840 -MI->getOperand(2).getImm());
1841 return;
1842
1843 case AArch64::SEH_SetFP:
1845 return;
1846
1847 case AArch64::SEH_AddFP:
1848 TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1849 return;
1850
1851 case AArch64::SEH_Nop:
1852 TS->emitARM64WinCFINop();
1853 return;
1854
1855 case AArch64::SEH_PrologEnd:
1857 return;
1858
1859 case AArch64::SEH_EpilogStart:
1861 return;
1862
1863 case AArch64::SEH_EpilogEnd:
1865 return;
1866
1867 case AArch64::SEH_PACSignLR:
1869 return;
1870
1871 case AArch64::SEH_SaveAnyRegQP:
1872 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
1873 "Non-consecutive registers not allowed for save_any_reg");
1874 assert(MI->getOperand(2).getImm() >= 0 &&
1875 "SaveAnyRegQP SEH opcode offset must be non-negative");
1876 assert(MI->getOperand(2).getImm() <= 1008 &&
1877 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
1878 TS->emitARM64WinCFISaveAnyRegQP(MI->getOperand(0).getImm(),
1879 MI->getOperand(2).getImm());
1880 return;
1881
1882 case AArch64::SEH_SaveAnyRegQPX:
1883 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
1884 "Non-consecutive registers not allowed for save_any_reg");
1885 assert(MI->getOperand(2).getImm() < 0 &&
1886 "SaveAnyRegQPX SEH opcode offset must be negative");
1887 assert(MI->getOperand(2).getImm() >= -1008 &&
1888 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
1889 TS->emitARM64WinCFISaveAnyRegQPX(MI->getOperand(0).getImm(),
1890 -MI->getOperand(2).getImm());
1891 return;
1892 }
1893
1894 // Finally, do the automated lowerings for everything else.
1895 MCInst TmpInst;
1896 MCInstLowering.Lower(MI, TmpInst);
1897 EmitToStreamer(*OutStreamer, TmpInst);
1898}
1899
1900void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
1901 MCSymbol *LazyPointer) {
1902 // _ifunc:
1903 // adrp x16, lazy_pointer@GOTPAGE
1904 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
1905 // ldr x16, [x16]
1906 // br x16
1907
1908 {
1909 MCInst Adrp;
1910 Adrp.setOpcode(AArch64::ADRP);
1911 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
1912 MCOperand SymPage;
1913 MCInstLowering.lowerOperand(
1916 SymPage);
1917 Adrp.addOperand(SymPage);
1918 OutStreamer->emitInstruction(Adrp, *STI);
1919 }
1920
1921 {
1922 MCInst Ldr;
1923 Ldr.setOpcode(AArch64::LDRXui);
1924 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
1925 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
1926 MCOperand SymPageOff;
1927 MCInstLowering.lowerOperand(
1930 SymPageOff);
1931 Ldr.addOperand(SymPageOff);
1933 OutStreamer->emitInstruction(Ldr, *STI);
1934 }
1935
1936 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui)
1937 .addReg(AArch64::X16)
1938 .addReg(AArch64::X16)
1939 .addImm(0),
1940 *STI);
1941
1942 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
1943 ? AArch64::BRAAZ
1944 : AArch64::BR)
1945 .addReg(AArch64::X16),
1946 *STI);
1947}
1948
1949void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
1950 const GlobalIFunc &GI,
1951 MCSymbol *LazyPointer) {
1952 // These stub helpers are only ever called once, so here we're optimizing for
1953 // minimum size by using the pre-indexed store variants, which saves a few
1954 // bytes of instructions to bump & restore sp.
1955
1956 // _ifunc.stub_helper:
1957 // stp fp, lr, [sp, #-16]!
1958 // mov fp, sp
1959 // stp x1, x0, [sp, #-16]!
1960 // stp x3, x2, [sp, #-16]!
1961 // stp x5, x4, [sp, #-16]!
1962 // stp x7, x6, [sp, #-16]!
1963 // stp d1, d0, [sp, #-16]!
1964 // stp d3, d2, [sp, #-16]!
1965 // stp d5, d4, [sp, #-16]!
1966 // stp d7, d6, [sp, #-16]!
1967 // bl _resolver
1968 // adrp x16, lazy_pointer@GOTPAGE
1969 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
1970 // str x0, [x16]
1971 // mov x16, x0
1972 // ldp d7, d6, [sp], #16
1973 // ldp d5, d4, [sp], #16
1974 // ldp d3, d2, [sp], #16
1975 // ldp d1, d0, [sp], #16
1976 // ldp x7, x6, [sp], #16
1977 // ldp x5, x4, [sp], #16
1978 // ldp x3, x2, [sp], #16
1979 // ldp x1, x0, [sp], #16
1980 // ldp fp, lr, [sp], #16
1981 // br x16
1982
1983 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
1984 .addReg(AArch64::SP)
1985 .addReg(AArch64::FP)
1986 .addReg(AArch64::LR)
1987 .addReg(AArch64::SP)
1988 .addImm(-2),
1989 *STI);
1990
1991 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
1992 .addReg(AArch64::FP)
1993 .addReg(AArch64::SP)
1994 .addImm(0)
1995 .addImm(0),
1996 *STI);
1997
1998 for (int I = 0; I != 4; ++I)
1999 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
2000 .addReg(AArch64::SP)
2001 .addReg(AArch64::X1 + 2 * I)
2002 .addReg(AArch64::X0 + 2 * I)
2003 .addReg(AArch64::SP)
2004 .addImm(-2),
2005 *STI);
2006
2007 for (int I = 0; I != 4; ++I)
2008 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre)
2009 .addReg(AArch64::SP)
2010 .addReg(AArch64::D1 + 2 * I)
2011 .addReg(AArch64::D0 + 2 * I)
2012 .addReg(AArch64::SP)
2013 .addImm(-2),
2014 *STI);
2015
2016 OutStreamer->emitInstruction(
2017 MCInstBuilder(AArch64::BL)
2019 *STI);
2020
2021 {
2022 MCInst Adrp;
2023 Adrp.setOpcode(AArch64::ADRP);
2024 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
2025 MCOperand SymPage;
2026 MCInstLowering.lowerOperand(
2027 MachineOperand::CreateES(LazyPointer->getName().data() + 1,
2029 SymPage);
2030 Adrp.addOperand(SymPage);
2031 OutStreamer->emitInstruction(Adrp, *STI);
2032 }
2033
2034 {
2035 MCInst Ldr;
2036 Ldr.setOpcode(AArch64::LDRXui);
2037 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
2038 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
2039 MCOperand SymPageOff;
2040 MCInstLowering.lowerOperand(
2041 MachineOperand::CreateES(LazyPointer->getName().data() + 1,
2043 SymPageOff);
2044 Ldr.addOperand(SymPageOff);
2046 OutStreamer->emitInstruction(Ldr, *STI);
2047 }
2048
2049 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STRXui)
2050 .addReg(AArch64::X0)
2051 .addReg(AArch64::X16)
2052 .addImm(0),
2053 *STI);
2054
2055 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
2056 .addReg(AArch64::X16)
2057 .addReg(AArch64::X0)
2058 .addImm(0)
2059 .addImm(0),
2060 *STI);
2061
2062 for (int I = 3; I != -1; --I)
2063 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost)
2064 .addReg(AArch64::SP)
2065 .addReg(AArch64::D1 + 2 * I)
2066 .addReg(AArch64::D0 + 2 * I)
2067 .addReg(AArch64::SP)
2068 .addImm(2),
2069 *STI);
2070
2071 for (int I = 3; I != -1; --I)
2072 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
2073 .addReg(AArch64::SP)
2074 .addReg(AArch64::X1 + 2 * I)
2075 .addReg(AArch64::X0 + 2 * I)
2076 .addReg(AArch64::SP)
2077 .addImm(2),
2078 *STI);
2079
2080 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
2081 .addReg(AArch64::SP)
2082 .addReg(AArch64::FP)
2083 .addReg(AArch64::LR)
2084 .addReg(AArch64::SP)
2085 .addImm(2),
2086 *STI);
2087
2088 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
2089 ? AArch64::BRAAZ
2090 : AArch64::BR)
2091 .addReg(AArch64::X16),
2092 *STI);
2093}
2094
2095const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV) {
2096 if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
2097 return MCSymbolRefExpr::create(MCInstLowering.GetGlobalValueSymbol(GV, 0),
2098 OutContext);
2099 }
2100
2101 return AsmPrinter::lowerConstant(CV);
2102}
2103
2104// Force static initialization.
2111}
LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter()
MachineBasicBlock & MBB
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_EXTERNAL_VISIBILITY
Definition: Compiler.h:135
uint64_t Size
Symbol * Sym
Definition: ELF_riscv.cpp:479
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
const char LLVMTargetMachineRef TM
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
static bool printOperand(raw_ostream &OS, const SelectionDAG *G, const SDValue Value)
This file defines the SmallString class.
This file defines the SmallVector class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition: Value.cpp:469
static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, char Mode, raw_ostream &O)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
std::optional< std::string > getOutliningStyle() const
static const char * getRegisterName(MCRegister Reg, unsigned AltIdx=AArch64::NoRegAltName)
static const AArch64MCExpr * create(const MCExpr *Expr, VariantKind Kind, MCContext &Ctx)
AArch64MCInstLower - This class is used to lower an MachineInstr into an MCInst.
bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const
virtual void emitARM64WinCFISaveRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveRegPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegPX(unsigned Reg, int Offset)
void emitNoteSection(unsigned Flags)
Callback used to implement the .note.gnu.property section.
virtual void emitARM64WinCFISaveRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFIAllocStack(unsigned Size)
virtual void emitARM64WinCFISaveFPLRX(int Offset)
virtual void emitDirectiveVariantPCS(MCSymbol *Symbol)
Callback used to implement the .variant_pcs directive.
virtual void emitARM64WinCFIAddFP(unsigned Size)
virtual void emitARM64WinCFISaveFPLR(int Offset)
virtual void emitARM64WinCFISaveFRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveLRPair(unsigned Reg, int Offset)
Represent the analysis usage information of a pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
This class is intended to be used as a driving class for all asm writers.
Definition: AsmPrinter.h:84
virtual void emitInstruction(const MachineInstr *)
Targets should implement this to emit instructions.
Definition: AsmPrinter.h:567
void emitXRayTable()
Emit a table with all XRay instrumentation points.
virtual MCSymbol * GetCPISymbol(unsigned CPID) const
Return the symbol for the specified constant pool entry.
virtual void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI, MCSymbol *LazyPointer)
Definition: AsmPrinter.h:612
virtual void emitJumpTableInfo()
Print assembly representations of the jump tables used by the current function to the current output ...
virtual void SetupMachineFunction(MachineFunction &MF)
This should be called when a new MachineFunction is being processed from runOnMachineFunction.
void emitFunctionBody()
This method emits the body and trailer for a function.
virtual void emitStartOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the start of their fi...
Definition: AsmPrinter.h:543
virtual void emitEndOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the end of their file...
Definition: AsmPrinter.h:547
virtual void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, MCSymbol *LazyPointer)
Definition: AsmPrinter.h:606
void getAnalysisUsage(AnalysisUsage &AU) const override
Record analysis usage.
Definition: AsmPrinter.cpp:440
virtual bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const
Definition: AsmPrinter.h:908
virtual const MCSubtargetInfo * getIFuncMCSubtargetInfo() const
getSubtargetInfo() cannot be used where this is needed because we don't have a MachineFunction when w...
Definition: AsmPrinter.h:602
bool runOnMachineFunction(MachineFunction &MF) override
Emit the specified function out to the OutStreamer.
Definition: AsmPrinter.h:395
virtual const MCExpr * lowerConstant(const Constant *CV)
Lower the specified LLVM Constant to an MCExpr.
virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant as...
virtual void emitFunctionBodyEnd()
Targets can override this to emit stuff after the last basic block in the function.
Definition: AsmPrinter.h:555
virtual void emitFunctionEntryLabel()
EmitFunctionEntryLabel - Emit the label that is the entrypoint for the function.
virtual std::tuple< const MCSymbol *, uint64_t, const MCSymbol *, codeview::JumpTableEntrySize > getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr, const MCSymbol *BranchLabel) const
Gets information required to create a CodeView debug symbol for a jump table.
virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant.
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
const Constant * getResolver() const
Definition: GlobalIFunc.h:70
bool hasLocalLinkage() const
Definition: GlobalValue.h:527
static const MCBinaryExpr * createLShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:616
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:621
static const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Definition: MCExpr.cpp:194
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:35
MCInstBuilder & addReg(unsigned Reg)
Add a new register operand.
Definition: MCInstBuilder.h:37
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Definition: MCInstBuilder.h:43
MCInstBuilder & addExpr(const MCExpr *Val)
Add a new MCExpr operand.
Definition: MCInstBuilder.h:61
Instances of this class represent a single low-level machine instruction.
Definition: MCInst.h:184
void addOperand(const MCOperand Op)
Definition: MCInst.h:210
void setOpcode(unsigned Op)
Definition: MCInst.h:197
Instances of this class represent operands of the MCInst class.
Definition: MCInst.h:36
static MCOperand createReg(unsigned Reg)
Definition: MCInst.h:134
static MCOperand createExpr(const MCExpr *Val)
Definition: MCInst.h:162
static MCOperand createImm(int64_t Val)
Definition: MCInst.h:141
uint16_t getEncodingValue(MCRegister RegNo) const
Returns the encoding for RegNo.
Instances of this class represent a uniqued identifier for a section in the current translation unit.
Definition: MCSection.h:39
Streaming machine code generation interface.
Definition: MCStreamer.h:212
virtual void emitCFIBKeyFrame()
Definition: MCStreamer.cpp:249
virtual void beginCOFFSymbolDef(const MCSymbol *Symbol)
Start emitting COFF symbol definition.
virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI)
Emit the given Instruction into the current section.
virtual void emitCOFFSymbolType(int Type)
Emit the type of the symbol.
virtual bool hasRawTextSupport() const
Return true if this asm streamer supports emitting unformatted text to the .s file with EmitRawText.
Definition: MCStreamer.h:340
virtual void endCOFFSymbolDef()
Marks the end of the symbol definition.
MCContext & getContext() const
Definition: MCStreamer.h:297
virtual void AddComment(const Twine &T, bool EOL=true)
Add a textual comment.
Definition: MCStreamer.h:359
virtual void emitCFIMTETaggedFrame()
Definition: MCStreamer.cpp:256
virtual void emitLabel(MCSymbol *Symbol, SMLoc Loc=SMLoc())
Emit a label for Symbol into the current section.
Definition: MCStreamer.cpp:424
MCTargetStreamer * getTargetStreamer()
Definition: MCStreamer.h:304
void emitRawText(const Twine &String)
If this file is backed by a assembly streamer, this dumps the specified string in the output ....
virtual void emitCOFFSymbolStorageClass(int StorageClass)
Emit the storage class of the symbol.
Generic base class for all target subtargets.
Represent a reference to a symbol from inside an expression.
Definition: MCExpr.h:192
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:397
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
StringRef getName() const
getName - Get the symbol name.
Definition: MCSymbol.h:205
Metadata node.
Definition: Metadata.h:1067
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
Definition: MachineInstr.h:69
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:662
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:556
const std::vector< MachineJumpTableEntry > & getJumpTables() const
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
static MachineOperand CreateMCSymbol(MCSymbol *Sym, unsigned TargetFlags=0)
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
const BlockAddress * getBlockAddress() const
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
@ MO_Immediate
Immediate operand.
@ MO_GlobalAddress
Address of a global value.
@ MO_BlockAddress
Address of a basic block.
@ MO_Register
Register operand.
@ MO_ExternalSymbol
Name of external global symbol.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
MI-level patchpoint operands.
Definition: StackMaps.h:76
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
MI-level stackmap operands.
Definition: StackMaps.h:35
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
Definition: StackMaps.h:50
void recordStatepoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a statepoint instruction.
Definition: StackMaps.cpp:569
void recordPatchPoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a patchpoint instruction.
Definition: StackMaps.cpp:548
void recordStackMap(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a stackmap instruction.
Definition: StackMaps.cpp:538
MI-level Statepoint operands.
Definition: StackMaps.h:158
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
virtual MCSection * getSectionForJumpTable(const Function &F, const TargetMachine &TM) const
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
MCRegister getRegister(unsigned i) const
Return the specified register in the class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an SmallVector or SmallString.
Definition: raw_ostream.h:690
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_S
MO_S - Indicates that the bits of the symbol operand represented by MO_G0 etc are signed.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
SymbolStorageClass
Storage class tells where and what the symbol represents.
Definition: COFF.h:217
@ IMAGE_SYM_CLASS_EXTERNAL
External symbol.
Definition: COFF.h:223
@ IMAGE_SYM_CLASS_STATIC
Static.
Definition: COFF.h:224
@ IMAGE_SYM_DTYPE_NULL
No complex type; simple scalar variable.
Definition: COFF.h:273
@ IMAGE_SYM_DTYPE_FUNCTION
A function that returns a base type.
Definition: COFF.h:275
@ SCT_COMPLEX_TYPE_SHIFT
Type is formed as (base + (derived << SCT_COMPLEX_TYPE_SHIFT))
Definition: COFF.h:279
@ AArch64_VectorCall
Used between AArch64 Advanced SIMD functions.
Definition: CallingConv.h:221
@ AArch64_SVE_VectorCall
Used between AArch64 SVE functions.
Definition: CallingConv.h:224
@ SHF_ALLOC
Definition: ELF.h:1157
@ SHF_GROUP
Definition: ELF.h:1179
@ SHF_EXECINSTR
Definition: ELF.h:1160
@ SHT_PROGBITS
Definition: ELF.h:1063
@ GNU_PROPERTY_AARCH64_FEATURE_1_BTI
Definition: ELF.h:1756
@ GNU_PROPERTY_AARCH64_FEATURE_1_PAC
Definition: ELF.h:1757
@ GNU_PROPERTY_AARCH64_FEATURE_1_GCS
Definition: ELF.h:1758
Reg
All possible values of the reg field in the ModR/M byte.
constexpr double e
Definition: MathExtras.h:31
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
ExceptionHandling
Target & getTheAArch64beTarget()
Target & getTheAArch64leTarget()
static unsigned getXRegFromWReg(unsigned Reg)
static unsigned getXRegFromXRegTuple(unsigned RegTuple)
Target & getTheAArch64_32Target()
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
Target & getTheARM64_32Target()
@ MCAF_SubsectionsViaSymbols
.subsections_via_symbols (MachO)
Definition: MCDirectives.h:55
@ Add
Sum of integers.
static unsigned getWRegFromXReg(unsigned Reg)
Target & getTheARM64Target()
@ MCSA_Weak
.weak
Definition: MCDirectives.h:45
@ MCSA_Global
.type _foo, @gnu_unique_object
Definition: MCDirectives.h:30
@ MCSA_WeakAntiDep
.weak_anti_dep (COFF)
Definition: MCDirectives.h:49
@ MCSA_ELF_TypeFunction
.type _foo, STT_FUNC # aka @function
Definition: MCDirectives.h:23
@ MCSA_Hidden
.hidden (ELF)
Definition: MCDirectives.h:33
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
RegisterAsmPrinter - Helper template for registering a target specific assembly printer,...