LLVM 20.0.0git
AArch64AsmPrinter.cpp
Go to the documentation of this file.
1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64.h"
15#include "AArch64MCInstLower.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
27#include "llvm/ADT/ScopeExit.h"
30#include "llvm/ADT/StringRef.h"
31#include "llvm/ADT/Twine.h"
45#include "llvm/IR/DataLayout.h"
47#include "llvm/IR/Module.h"
48#include "llvm/MC/MCAsmInfo.h"
49#include "llvm/MC/MCContext.h"
50#include "llvm/MC/MCInst.h"
54#include "llvm/MC/MCStreamer.h"
55#include "llvm/MC/MCSymbol.h"
64#include <cassert>
65#include <cstdint>
66#include <map>
67#include <memory>
68
69using namespace llvm;
70
73 "aarch64-ptrauth-auth-checks", cl::Hidden,
74 cl::values(clEnumValN(Unchecked, "none", "don't test for failure"),
75 clEnumValN(Poison, "poison", "poison on failure"),
76 clEnumValN(Trap, "trap", "trap on failure")),
77 cl::desc("Check pointer authentication auth/resign failures"),
79
80#define DEBUG_TYPE "asm-printer"
81
82namespace {
83
84class AArch64AsmPrinter : public AsmPrinter {
85 AArch64MCInstLower MCInstLowering;
86 FaultMaps FM;
87 const AArch64Subtarget *STI;
88 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
89#ifndef NDEBUG
90 unsigned InstsEmitted;
91#endif
92
93public:
94 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
95 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
96 FM(*this) {}
97
98 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
99
100 /// Wrapper for MCInstLowering.lowerOperand() for the
101 /// tblgen'erated pseudo lowering.
102 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
103 return MCInstLowering.lowerOperand(MO, MCOp);
104 }
105
106 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
107
108 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
109
110 void emitStartOfAsmFile(Module &M) override;
111 void emitJumpTableInfo() override;
112 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
114 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
115 const MCSymbol *BranchLabel) const override;
116
117 void emitFunctionEntryLabel() override;
118
119 void emitXXStructor(const DataLayout &DL, const Constant *CV) override;
120
121 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
122
123 void LowerHardenedBRJumpTable(const MachineInstr &MI);
124
125 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
126
127 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
128 const MachineInstr &MI);
129 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
130 const MachineInstr &MI);
131 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
132 const MachineInstr &MI);
133 void LowerFAULTING_OP(const MachineInstr &MI);
134
135 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
136 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
137 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
138 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
139
140 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
141 HwasanMemaccessTuple;
142 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
143 void LowerKCFI_CHECK(const MachineInstr &MI);
144 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
145 void emitHwasanMemaccessSymbols(Module &M);
146
147 void emitSled(const MachineInstr &MI, SledKind Kind);
148
149 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
150 void emitPtrauthBranch(const MachineInstr *MI);
151
152 void emitPtrauthCheckAuthenticatedValue(Register TestedReg,
153 Register ScratchReg,
156 bool ShouldTrap,
157 const MCSymbol *OnFailure);
158
159 // Check authenticated LR before tail calling.
160 void emitPtrauthTailCallHardening(const MachineInstr *TC);
161
162 // Emit the sequence for AUT or AUTPAC.
163 void emitPtrauthAuthResign(const MachineInstr *MI);
164
165 // Emit the sequence to compute the discriminator.
166 //
167 // ScratchReg should be x16/x17.
168 //
169 // The returned register is either unmodified AddrDisc or x16/x17.
170 //
171 // If the expanded pseudo is allowed to clobber AddrDisc register, setting
172 // MayUseAddrAsScratch may save one MOV instruction, provided the address
173 // is already in x16/x17 (i.e. return x16/x17 which is the *modified* AddrDisc
174 // register at the same time):
175 //
176 // mov x17, x16
177 // movk x17, #1234, lsl #48
178 // ; x16 is not used anymore
179 //
180 // can be replaced by
181 //
182 // movk x16, #1234, lsl #48
183 Register emitPtrauthDiscriminator(uint16_t Disc, Register AddrDisc,
184 Register ScratchReg,
185 bool MayUseAddrAsScratch = false);
186
187 // Emit the sequence for LOADauthptrstatic
188 void LowerLOADauthptrstatic(const MachineInstr &MI);
189
190 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
191 // adrp-add followed by PAC sign)
192 void LowerMOVaddrPAC(const MachineInstr &MI);
193
194 // Emit the sequence for LOADgotAUTH (load signed pointer from signed ELF GOT
195 // and authenticate it with, if FPAC bit is not set, check+trap sequence after
196 // authenticating)
197 void LowerLOADgotAUTH(const MachineInstr &MI);
198
199 /// tblgen'erated driver function for lowering simple MI->MC
200 /// pseudo instructions.
201 bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
202
203 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
204 void EmitToStreamer(const MCInst &Inst) {
205 EmitToStreamer(*OutStreamer, Inst);
206 }
207
208 void emitInstruction(const MachineInstr *MI) override;
209
210 void emitFunctionHeaderComment() override;
211
212 void getAnalysisUsage(AnalysisUsage &AU) const override {
214 AU.setPreservesAll();
215 }
216
217 bool runOnMachineFunction(MachineFunction &MF) override {
218 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
219 STI = &MF.getSubtarget<AArch64Subtarget>();
220
222
223 if (STI->isTargetCOFF()) {
224 bool Local = MF.getFunction().hasLocalLinkage();
227 int Type =
229
230 OutStreamer->beginCOFFSymbolDef(CurrentFnSym);
231 OutStreamer->emitCOFFSymbolStorageClass(Scl);
232 OutStreamer->emitCOFFSymbolType(Type);
233 OutStreamer->endCOFFSymbolDef();
234 }
235
236 // Emit the rest of the function body.
238
239 // Emit the XRay table for this function.
241
242 // We didn't modify anything.
243 return false;
244 }
245
246 const MCExpr *lowerConstant(const Constant *CV) override;
247
248private:
249 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
250 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
251 bool printAsmRegInClass(const MachineOperand &MO,
252 const TargetRegisterClass *RC, unsigned AltName,
253 raw_ostream &O);
254
255 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
256 const char *ExtraCode, raw_ostream &O) override;
257 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
258 const char *ExtraCode, raw_ostream &O) override;
259
260 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
261
262 void emitFunctionBodyEnd() override;
263 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
264
265 MCSymbol *GetCPISymbol(unsigned CPID) const override;
266 void emitEndOfAsmFile(Module &M) override;
267
268 AArch64FunctionInfo *AArch64FI = nullptr;
269
270 /// Emit the LOHs contained in AArch64FI.
271 void emitLOHs();
272
273 void emitMovXReg(Register Dest, Register Src);
274 void emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift);
275 void emitMOVK(Register Dest, uint64_t Imm, unsigned Shift);
276
277 /// Emit instruction to set float register to zero.
278 void emitFMov0(const MachineInstr &MI);
279
280 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
281
282 MInstToMCSymbol LOHInstToLabel;
283
285 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
286 }
287
288 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
289 assert(STI);
290 return STI;
291 }
292 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
293 MCSymbol *LazyPointer) override;
295 MCSymbol *LazyPointer) override;
296};
297
298} // end anonymous namespace
299
300void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
301 const Triple &TT = TM.getTargetTriple();
302
303 if (TT.isOSBinFormatCOFF()) {
304 // Emit an absolute @feat.00 symbol
305 MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00"));
306 OutStreamer->beginCOFFSymbolDef(S);
307 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
308 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
309 OutStreamer->endCOFFSymbolDef();
310 int64_t Feat00Value = 0;
311
312 if (M.getModuleFlag("cfguard")) {
313 // Object is CFG-aware.
314 Feat00Value |= COFF::Feat00Flags::GuardCF;
315 }
316
317 if (M.getModuleFlag("ehcontguard")) {
318 // Object also has EHCont.
319 Feat00Value |= COFF::Feat00Flags::GuardEHCont;
320 }
321
322 if (M.getModuleFlag("ms-kernel")) {
323 // Object is compiled with /kernel.
324 Feat00Value |= COFF::Feat00Flags::Kernel;
325 }
326
327 OutStreamer->emitSymbolAttribute(S, MCSA_Global);
328 OutStreamer->emitAssignment(
329 S, MCConstantExpr::create(Feat00Value, MMI->getContext()));
330 }
331
332 if (!TT.isOSBinFormatELF())
333 return;
334
335 // Assemble feature flags that may require creation of a note section.
336 unsigned Flags = 0;
337 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
338 M.getModuleFlag("branch-target-enforcement")))
339 if (!BTE->isZero())
341
342 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
343 M.getModuleFlag("guarded-control-stack")))
344 if (!GCS->isZero())
346
347 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
348 M.getModuleFlag("sign-return-address")))
349 if (!Sign->isZero())
351
352 uint64_t PAuthABIPlatform = -1;
353 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
354 M.getModuleFlag("aarch64-elf-pauthabi-platform")))
355 PAuthABIPlatform = PAP->getZExtValue();
356 uint64_t PAuthABIVersion = -1;
357 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
358 M.getModuleFlag("aarch64-elf-pauthabi-version")))
359 PAuthABIVersion = PAV->getZExtValue();
360
361 // Emit a .note.gnu.property section with the flags.
362 auto *TS =
363 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
364 TS->emitNoteSection(Flags, PAuthABIPlatform, PAuthABIVersion);
365}
366
367void AArch64AsmPrinter::emitFunctionHeaderComment() {
368 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
369 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
370 if (OutlinerString != std::nullopt)
371 OutStreamer->getCommentOS() << ' ' << OutlinerString;
372}
373
374void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
375{
376 const Function &F = MF->getFunction();
377 if (F.hasFnAttribute("patchable-function-entry")) {
378 unsigned Num;
379 if (F.getFnAttribute("patchable-function-entry")
380 .getValueAsString()
381 .getAsInteger(10, Num))
382 return;
383 emitNops(Num);
384 return;
385 }
386
387 emitSled(MI, SledKind::FUNCTION_ENTER);
388}
389
390void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
391 emitSled(MI, SledKind::FUNCTION_EXIT);
392}
393
394void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
395 emitSled(MI, SledKind::TAIL_CALL);
396}
397
398void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
399 static const int8_t NoopsInSledCount = 7;
400 // We want to emit the following pattern:
401 //
402 // .Lxray_sled_N:
403 // ALIGN
404 // B #32
405 // ; 7 NOP instructions (28 bytes)
406 // .tmpN
407 //
408 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
409 // over the full 32 bytes (8 instructions) with the following pattern:
410 //
411 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
412 // LDR W17, #12 ; W17 := function ID
413 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
414 // BLR X16 ; call the tracing trampoline
415 // ;DATA: 32 bits of function ID
416 // ;DATA: lower 32 bits of the address of the trampoline
417 // ;DATA: higher 32 bits of the address of the trampoline
418 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
419 //
420 OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
421 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
422 OutStreamer->emitLabel(CurSled);
423 auto Target = OutContext.createTempSymbol();
424
425 // Emit "B #32" instruction, which jumps over the next 28 bytes.
426 // The operand has to be the number of 4-byte instructions to jump over,
427 // including the current instruction.
428 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
429
430 for (int8_t I = 0; I < NoopsInSledCount; I++)
431 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
432
433 OutStreamer->emitLabel(Target);
434 recordSled(CurSled, MI, Kind, 2);
435}
436
437// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
438// (built-in functions __xray_customevent/__xray_typedevent).
439//
440// .Lxray_event_sled_N:
441// b 1f
442// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
443// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
444// bl __xray_CustomEvent or __xray_TypedEvent
445// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
446// 1:
447//
448// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
449//
450// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
451// After patching, b .+N will become a nop.
452void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
453 bool Typed) {
454 auto &O = *OutStreamer;
455 MCSymbol *CurSled = OutContext.createTempSymbol("xray_sled_", true);
456 O.emitLabel(CurSled);
457 bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
459 OutContext.getOrCreateSymbol(
460 Twine(MachO ? "_" : "") +
461 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
462 OutContext);
463 if (Typed) {
464 O.AddComment("Begin XRay typed event");
465 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(9));
466 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
467 .addReg(AArch64::SP)
468 .addReg(AArch64::X0)
469 .addReg(AArch64::X1)
470 .addReg(AArch64::SP)
471 .addImm(-4));
472 EmitToStreamer(O, MCInstBuilder(AArch64::STRXui)
473 .addReg(AArch64::X2)
474 .addReg(AArch64::SP)
475 .addImm(2));
476 emitMovXReg(AArch64::X0, MI.getOperand(0).getReg());
477 emitMovXReg(AArch64::X1, MI.getOperand(1).getReg());
478 emitMovXReg(AArch64::X2, MI.getOperand(2).getReg());
479 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
480 EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui)
481 .addReg(AArch64::X2)
482 .addReg(AArch64::SP)
483 .addImm(2));
484 O.AddComment("End XRay typed event");
485 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
486 .addReg(AArch64::SP)
487 .addReg(AArch64::X0)
488 .addReg(AArch64::X1)
489 .addReg(AArch64::SP)
490 .addImm(4));
491
492 recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
493 } else {
494 O.AddComment("Begin XRay custom event");
495 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(6));
496 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
497 .addReg(AArch64::SP)
498 .addReg(AArch64::X0)
499 .addReg(AArch64::X1)
500 .addReg(AArch64::SP)
501 .addImm(-2));
502 emitMovXReg(AArch64::X0, MI.getOperand(0).getReg());
503 emitMovXReg(AArch64::X1, MI.getOperand(1).getReg());
504 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
505 O.AddComment("End XRay custom event");
506 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
507 .addReg(AArch64::SP)
508 .addReg(AArch64::X0)
509 .addReg(AArch64::X1)
510 .addReg(AArch64::SP)
511 .addImm(2));
512
513 recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
514 }
515}
516
517void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
518 Register AddrReg = MI.getOperand(0).getReg();
519 assert(std::next(MI.getIterator())->isCall() &&
520 "KCFI_CHECK not followed by a call instruction");
521 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
522 "KCFI_CHECK call target doesn't match call operand");
523
524 // Default to using the intra-procedure-call temporary registers for
525 // comparing the hashes.
526 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
527 if (AddrReg == AArch64::XZR) {
528 // Checking XZR makes no sense. Instead of emitting a load, zero
529 // ScratchRegs[0] and use it for the ESR AddrIndex below.
530 AddrReg = getXRegFromWReg(ScratchRegs[0]);
531 emitMovXReg(AddrReg, AArch64::XZR);
532 } else {
533 // If one of the scratch registers is used for the call target (e.g.
534 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
535 // temporary register instead (in this case, AArch64::W9) as the check
536 // is immediately followed by the call instruction.
537 for (auto &Reg : ScratchRegs) {
538 if (Reg == getWRegFromXReg(AddrReg)) {
539 Reg = AArch64::W9;
540 break;
541 }
542 }
543 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
544 "Invalid scratch registers for KCFI_CHECK");
545
546 // Adjust the offset for patchable-function-prefix. This assumes that
547 // patchable-function-prefix is the same for all functions.
548 int64_t PrefixNops = 0;
549 (void)MI.getMF()
550 ->getFunction()
551 .getFnAttribute("patchable-function-prefix")
552 .getValueAsString()
553 .getAsInteger(10, PrefixNops);
554
555 // Load the target function type hash.
556 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi)
557 .addReg(ScratchRegs[0])
558 .addReg(AddrReg)
559 .addImm(-(PrefixNops * 4 + 4)));
560 }
561
562 // Load the expected type hash.
563 const int64_t Type = MI.getOperand(1).getImm();
564 emitMOVK(ScratchRegs[1], Type & 0xFFFF, 0);
565 emitMOVK(ScratchRegs[1], (Type >> 16) & 0xFFFF, 16);
566
567 // Compare the hashes and trap if there's a mismatch.
568 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs)
569 .addReg(AArch64::WZR)
570 .addReg(ScratchRegs[0])
571 .addReg(ScratchRegs[1])
572 .addImm(0));
573
574 MCSymbol *Pass = OutContext.createTempSymbol();
575 EmitToStreamer(*OutStreamer,
576 MCInstBuilder(AArch64::Bcc)
577 .addImm(AArch64CC::EQ)
578 .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
579
580 // The base ESR is 0x8000 and the register information is encoded in bits
581 // 0-9 as follows:
582 // - 0-4: n, where the register Xn contains the target address
583 // - 5-9: m, where the register Wm contains the expected type hash
584 // Where n, m are in [0, 30].
585 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
586 unsigned AddrIndex;
587 switch (AddrReg) {
588 default:
589 AddrIndex = AddrReg - AArch64::X0;
590 break;
591 case AArch64::FP:
592 AddrIndex = 29;
593 break;
594 case AArch64::LR:
595 AddrIndex = 30;
596 break;
597 }
598
599 assert(AddrIndex < 31 && TypeIndex < 31);
600
601 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
602 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR));
603 OutStreamer->emitLabel(Pass);
604}
605
606void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
607 Register Reg = MI.getOperand(0).getReg();
608 bool IsShort =
609 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
610 (MI.getOpcode() ==
611 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
612 uint32_t AccessInfo = MI.getOperand(1).getImm();
613 bool IsFixedShadow =
614 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
615 (MI.getOpcode() ==
616 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
617 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(2).getImm() : 0;
618
619 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
620 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
621 if (!Sym) {
622 // FIXME: Make this work on non-ELF.
623 if (!TM.getTargetTriple().isOSBinFormatELF())
624 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
625
626 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
627 utostr(AccessInfo);
628 if (IsFixedShadow)
629 SymName += "_fixed_" + utostr(FixedShadowOffset);
630 if (IsShort)
631 SymName += "_short_v2";
632 Sym = OutContext.getOrCreateSymbol(SymName);
633 }
634
635 EmitToStreamer(*OutStreamer,
636 MCInstBuilder(AArch64::BL)
637 .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
638}
639
640void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
641 if (HwasanMemaccessSymbols.empty())
642 return;
643
644 const Triple &TT = TM.getTargetTriple();
645 assert(TT.isOSBinFormatELF());
646 std::unique_ptr<MCSubtargetInfo> STI(
647 TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
648 assert(STI && "Unable to create subtarget info");
649 this->STI = static_cast<const AArch64Subtarget *>(&*STI);
650
651 MCSymbol *HwasanTagMismatchV1Sym =
652 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
653 MCSymbol *HwasanTagMismatchV2Sym =
654 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
655
656 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
657 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
658 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
659 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
660
661 for (auto &P : HwasanMemaccessSymbols) {
662 unsigned Reg = std::get<0>(P.first);
663 bool IsShort = std::get<1>(P.first);
664 uint32_t AccessInfo = std::get<2>(P.first);
665 bool IsFixedShadow = std::get<3>(P.first);
666 uint64_t FixedShadowOffset = std::get<4>(P.first);
667 const MCSymbolRefExpr *HwasanTagMismatchRef =
668 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
669 MCSymbol *Sym = P.second;
670
671 bool HasMatchAllTag =
672 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
673 uint8_t MatchAllTag =
674 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
675 unsigned Size =
676 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
677 bool CompileKernel =
678 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
679
680 OutStreamer->switchSection(OutContext.getELFSection(
681 ".text.hot", ELF::SHT_PROGBITS,
683 /*IsComdat=*/true));
684
685 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
686 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
687 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
688 OutStreamer->emitLabel(Sym);
689
690 EmitToStreamer(MCInstBuilder(AArch64::SBFMXri)
691 .addReg(AArch64::X16)
692 .addReg(Reg)
693 .addImm(4)
694 .addImm(55));
695
696 if (IsFixedShadow) {
697 // Aarch64 makes it difficult to embed large constants in the code.
698 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
699 // left-shift option in the MOV instruction. Combined with the 16-bit
700 // immediate, this is enough to represent any offset up to 2**48.
701 emitMOVZ(AArch64::X17, FixedShadowOffset >> 32, 32);
702 EmitToStreamer(MCInstBuilder(AArch64::LDRBBroX)
703 .addReg(AArch64::W16)
704 .addReg(AArch64::X17)
705 .addReg(AArch64::X16)
706 .addImm(0)
707 .addImm(0));
708 } else {
709 EmitToStreamer(MCInstBuilder(AArch64::LDRBBroX)
710 .addReg(AArch64::W16)
711 .addReg(IsShort ? AArch64::X20 : AArch64::X9)
712 .addReg(AArch64::X16)
713 .addImm(0)
714 .addImm(0));
715 }
716
717 EmitToStreamer(MCInstBuilder(AArch64::SUBSXrs)
718 .addReg(AArch64::XZR)
719 .addReg(AArch64::X16)
720 .addReg(Reg)
722 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
723 EmitToStreamer(MCInstBuilder(AArch64::Bcc)
724 .addImm(AArch64CC::NE)
726 HandleMismatchOrPartialSym, OutContext)));
727 MCSymbol *ReturnSym = OutContext.createTempSymbol();
728 OutStreamer->emitLabel(ReturnSym);
729 EmitToStreamer(MCInstBuilder(AArch64::RET).addReg(AArch64::LR));
730 OutStreamer->emitLabel(HandleMismatchOrPartialSym);
731
732 if (HasMatchAllTag) {
733 EmitToStreamer(MCInstBuilder(AArch64::UBFMXri)
734 .addReg(AArch64::X17)
735 .addReg(Reg)
736 .addImm(56)
737 .addImm(63));
738 EmitToStreamer(MCInstBuilder(AArch64::SUBSXri)
739 .addReg(AArch64::XZR)
740 .addReg(AArch64::X17)
741 .addImm(MatchAllTag)
742 .addImm(0));
743 EmitToStreamer(
744 MCInstBuilder(AArch64::Bcc)
745 .addImm(AArch64CC::EQ)
746 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)));
747 }
748
749 if (IsShort) {
750 EmitToStreamer(MCInstBuilder(AArch64::SUBSWri)
751 .addReg(AArch64::WZR)
752 .addReg(AArch64::W16)
753 .addImm(15)
754 .addImm(0));
755 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
756 EmitToStreamer(
757 MCInstBuilder(AArch64::Bcc)
758 .addImm(AArch64CC::HI)
759 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)));
760
761 EmitToStreamer(MCInstBuilder(AArch64::ANDXri)
762 .addReg(AArch64::X17)
763 .addReg(Reg)
764 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)));
765 if (Size != 1)
766 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
767 .addReg(AArch64::X17)
768 .addReg(AArch64::X17)
769 .addImm(Size - 1)
770 .addImm(0));
771 EmitToStreamer(MCInstBuilder(AArch64::SUBSWrs)
772 .addReg(AArch64::WZR)
773 .addReg(AArch64::W16)
774 .addReg(AArch64::W17)
775 .addImm(0));
776 EmitToStreamer(
777 MCInstBuilder(AArch64::Bcc)
778 .addImm(AArch64CC::LS)
779 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)));
780
781 EmitToStreamer(MCInstBuilder(AArch64::ORRXri)
782 .addReg(AArch64::X16)
783 .addReg(Reg)
784 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)));
785 EmitToStreamer(MCInstBuilder(AArch64::LDRBBui)
786 .addReg(AArch64::W16)
787 .addReg(AArch64::X16)
788 .addImm(0));
789 EmitToStreamer(
790 MCInstBuilder(AArch64::SUBSXrs)
791 .addReg(AArch64::XZR)
792 .addReg(AArch64::X16)
793 .addReg(Reg)
795 EmitToStreamer(
796 MCInstBuilder(AArch64::Bcc)
797 .addImm(AArch64CC::EQ)
798 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)));
799
800 OutStreamer->emitLabel(HandleMismatchSym);
801 }
802
803 EmitToStreamer(MCInstBuilder(AArch64::STPXpre)
804 .addReg(AArch64::SP)
805 .addReg(AArch64::X0)
806 .addReg(AArch64::X1)
807 .addReg(AArch64::SP)
808 .addImm(-32));
809 EmitToStreamer(MCInstBuilder(AArch64::STPXi)
810 .addReg(AArch64::FP)
811 .addReg(AArch64::LR)
812 .addReg(AArch64::SP)
813 .addImm(29));
814
815 if (Reg != AArch64::X0)
816 emitMovXReg(AArch64::X0, Reg);
817 emitMOVZ(AArch64::X1, AccessInfo & HWASanAccessInfo::RuntimeMask, 0);
818
819 if (CompileKernel) {
820 // The Linux kernel's dynamic loader doesn't support GOT relative
821 // relocations, but it doesn't support late binding either, so just call
822 // the function directly.
823 EmitToStreamer(MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef));
824 } else {
825 // Intentionally load the GOT entry and branch to it, rather than possibly
826 // late binding the function, which may clobber the registers before we
827 // have a chance to save them.
828 EmitToStreamer(
829 MCInstBuilder(AArch64::ADRP)
830 .addReg(AArch64::X16)
831 .addExpr(AArch64MCExpr::create(
832 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
833 OutContext)));
834 EmitToStreamer(
835 MCInstBuilder(AArch64::LDRXui)
836 .addReg(AArch64::X16)
837 .addReg(AArch64::X16)
838 .addExpr(AArch64MCExpr::create(
839 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
840 OutContext)));
841 EmitToStreamer(MCInstBuilder(AArch64::BR).addReg(AArch64::X16));
842 }
843 }
844 this->STI = nullptr;
845}
846
847static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
848 MCSymbol *StubLabel,
849 const MCExpr *StubAuthPtrRef) {
850 // sym$auth_ptr$key$disc:
851 OutStreamer.emitLabel(StubLabel);
852 OutStreamer.emitValue(StubAuthPtrRef, /*size=*/8);
853}
854
855void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
856 emitHwasanMemaccessSymbols(M);
857
858 const Triple &TT = TM.getTargetTriple();
859 if (TT.isOSBinFormatMachO()) {
860 // Output authenticated pointers as indirect symbols, if we have any.
861 MachineModuleInfoMachO &MMIMacho =
862 MMI->getObjFileInfo<MachineModuleInfoMachO>();
863
864 auto Stubs = MMIMacho.getAuthGVStubList();
865
866 if (!Stubs.empty()) {
867 // Switch to the "__auth_ptr" section.
868 OutStreamer->switchSection(
869 OutContext.getMachOSection("__DATA", "__auth_ptr", MachO::S_REGULAR,
871 emitAlignment(Align(8));
872
873 for (const auto &Stub : Stubs)
874 emitAuthenticatedPointer(*OutStreamer, Stub.first, Stub.second);
875
876 OutStreamer->addBlankLine();
877 }
878
879 // Funny Darwin hack: This flag tells the linker that no global symbols
880 // contain code that falls through to other global symbols (e.g. the obvious
881 // implementation of multiple entry points). If this doesn't occur, the
882 // linker can safely perform dead code stripping. Since LLVM never
883 // generates code that does this, it is always safe to set.
884 OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
885 }
886
887 if (TT.isOSBinFormatELF()) {
888 // Output authenticated pointers as indirect symbols, if we have any.
889 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
890
891 auto Stubs = MMIELF.getAuthGVStubList();
892
893 if (!Stubs.empty()) {
894 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
895 OutStreamer->switchSection(TLOF.getDataSection());
896 emitAlignment(Align(8));
897
898 for (const auto &Stub : Stubs)
899 emitAuthenticatedPointer(*OutStreamer, Stub.first, Stub.second);
900
901 OutStreamer->addBlankLine();
902 }
903
904 // With signed ELF GOT enabled, the linker looks at the symbol type to
905 // choose between keys IA (for STT_FUNC) and DA (for other types). Symbols
906 // for functions not defined in the module have STT_NOTYPE type by default.
907 // This makes linker to emit signing schema with DA key (instead of IA) for
908 // corresponding R_AARCH64_AUTH_GLOB_DAT dynamic reloc. To avoid that, force
909 // all function symbols used in the module to have STT_FUNC type. See
910 // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#default-signing-schema
911 const auto *PtrAuthELFGOTFlag = mdconst::extract_or_null<ConstantInt>(
912 M.getModuleFlag("ptrauth-elf-got"));
913 if (PtrAuthELFGOTFlag && PtrAuthELFGOTFlag->getZExtValue() == 1)
914 for (const GlobalValue &GV : M.global_values())
915 if (!GV.use_empty() && isa<Function>(GV) &&
916 !GV.getName().starts_with("llvm."))
917 OutStreamer->emitSymbolAttribute(getSymbol(&GV),
919 }
920
921 // Emit stack and fault map information.
922 FM.serializeToFaultMapSection();
923
924}
925
926void AArch64AsmPrinter::emitLOHs() {
928
929 for (const auto &D : AArch64FI->getLOHContainer()) {
930 for (const MachineInstr *MI : D.getArgs()) {
931 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
932 assert(LabelIt != LOHInstToLabel.end() &&
933 "Label hasn't been inserted for LOH related instruction");
934 MCArgs.push_back(LabelIt->second);
935 }
936 OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
937 MCArgs.clear();
938 }
939}
940
941void AArch64AsmPrinter::emitFunctionBodyEnd() {
942 if (!AArch64FI->getLOHRelated().empty())
943 emitLOHs();
944}
945
946/// GetCPISymbol - Return the symbol for the specified constant pool entry.
947MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
948 // Darwin uses a linker-private symbol name for constant-pools (to
949 // avoid addends on the relocation?), ELF has no such concept and
950 // uses a normal private symbol.
951 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
952 return OutContext.getOrCreateSymbol(
953 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
954 Twine(getFunctionNumber()) + "_" + Twine(CPID));
955
956 return AsmPrinter::GetCPISymbol(CPID);
957}
958
959void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
960 raw_ostream &O) {
961 const MachineOperand &MO = MI->getOperand(OpNum);
962 switch (MO.getType()) {
963 default:
964 llvm_unreachable("<unknown operand type>");
966 Register Reg = MO.getReg();
967 assert(Reg.isPhysical());
968 assert(!MO.getSubReg() && "Subregs should be eliminated!");
970 break;
971 }
973 O << MO.getImm();
974 break;
975 }
977 PrintSymbolOperand(MO, O);
978 break;
979 }
981 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
982 Sym->print(O, MAI);
983 break;
984 }
985 }
986}
987
988bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
989 raw_ostream &O) {
990 Register Reg = MO.getReg();
991 switch (Mode) {
992 default:
993 return true; // Unknown mode.
994 case 'w':
995 Reg = getWRegFromXReg(Reg);
996 break;
997 case 'x':
998 Reg = getXRegFromWReg(Reg);
999 break;
1000 case 't':
1002 break;
1003 }
1004
1006 return false;
1007}
1008
1009// Prints the register in MO using class RC using the offset in the
1010// new register class. This should not be used for cross class
1011// printing.
1012bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1013 const TargetRegisterClass *RC,
1014 unsigned AltName, raw_ostream &O) {
1015 assert(MO.isReg() && "Should only get here with a register!");
1016 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1017 Register Reg = MO.getReg();
1018 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
1019 if (!RI->regsOverlap(RegToPrint, Reg))
1020 return true;
1021 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
1022 return false;
1023}
1024
1025bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1026 const char *ExtraCode, raw_ostream &O) {
1027 const MachineOperand &MO = MI->getOperand(OpNum);
1028
1029 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1030 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
1031 return false;
1032
1033 // Does this asm operand have a single letter operand modifier?
1034 if (ExtraCode && ExtraCode[0]) {
1035 if (ExtraCode[1] != 0)
1036 return true; // Unknown modifier.
1037
1038 switch (ExtraCode[0]) {
1039 default:
1040 return true; // Unknown modifier.
1041 case 'w': // Print W register
1042 case 'x': // Print X register
1043 if (MO.isReg())
1044 return printAsmMRegister(MO, ExtraCode[0], O);
1045 if (MO.isImm() && MO.getImm() == 0) {
1046 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1048 return false;
1049 }
1050 printOperand(MI, OpNum, O);
1051 return false;
1052 case 'b': // Print B register.
1053 case 'h': // Print H register.
1054 case 's': // Print S register.
1055 case 'd': // Print D register.
1056 case 'q': // Print Q register.
1057 case 'z': // Print Z register.
1058 if (MO.isReg()) {
1059 const TargetRegisterClass *RC;
1060 switch (ExtraCode[0]) {
1061 case 'b':
1062 RC = &AArch64::FPR8RegClass;
1063 break;
1064 case 'h':
1065 RC = &AArch64::FPR16RegClass;
1066 break;
1067 case 's':
1068 RC = &AArch64::FPR32RegClass;
1069 break;
1070 case 'd':
1071 RC = &AArch64::FPR64RegClass;
1072 break;
1073 case 'q':
1074 RC = &AArch64::FPR128RegClass;
1075 break;
1076 case 'z':
1077 RC = &AArch64::ZPRRegClass;
1078 break;
1079 default:
1080 return true;
1081 }
1082 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
1083 }
1084 printOperand(MI, OpNum, O);
1085 return false;
1086 }
1087 }
1088
1089 // According to ARM, we should emit x and v registers unless we have a
1090 // modifier.
1091 if (MO.isReg()) {
1092 Register Reg = MO.getReg();
1093
1094 // If this is a w or x register, print an x register.
1095 if (AArch64::GPR32allRegClass.contains(Reg) ||
1096 AArch64::GPR64allRegClass.contains(Reg))
1097 return printAsmMRegister(MO, 'x', O);
1098
1099 // If this is an x register tuple, print an x register.
1100 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1101 return printAsmMRegister(MO, 't', O);
1102
1103 unsigned AltName = AArch64::NoRegAltName;
1104 const TargetRegisterClass *RegClass;
1105 if (AArch64::ZPRRegClass.contains(Reg)) {
1106 RegClass = &AArch64::ZPRRegClass;
1107 } else if (AArch64::PPRRegClass.contains(Reg)) {
1108 RegClass = &AArch64::PPRRegClass;
1109 } else if (AArch64::PNRRegClass.contains(Reg)) {
1110 RegClass = &AArch64::PNRRegClass;
1111 } else {
1112 RegClass = &AArch64::FPR128RegClass;
1113 AltName = AArch64::vreg;
1114 }
1115
1116 // If this is a b, h, s, d, or q register, print it as a v register.
1117 return printAsmRegInClass(MO, RegClass, AltName, O);
1118 }
1119
1120 printOperand(MI, OpNum, O);
1121 return false;
1122}
1123
1124bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1125 unsigned OpNum,
1126 const char *ExtraCode,
1127 raw_ostream &O) {
1128 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1129 return true; // Unknown modifier.
1130
1131 const MachineOperand &MO = MI->getOperand(OpNum);
1132 assert(MO.isReg() && "unexpected inline asm memory operand");
1133 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
1134 return false;
1135}
1136
1137void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1138 raw_ostream &OS) {
1139 unsigned NOps = MI->getNumOperands();
1140 assert(NOps == 4);
1141 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
1142 // cast away const; DIetc do not take const operands for some reason.
1143 OS << MI->getDebugVariable()->getName();
1144 OS << " <- ";
1145 // Frame address. Currently handles register +- offset only.
1146 assert(MI->isIndirectDebugValue());
1147 OS << '[';
1148 for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(),
1149 MI->debug_operands().end());
1150 I < E; ++I) {
1151 if (I != 0)
1152 OS << ", ";
1153 printOperand(MI, I, OS);
1154 }
1155 OS << ']';
1156 OS << "+";
1157 printOperand(MI, NOps - 2, OS);
1158}
1159
1160void AArch64AsmPrinter::emitJumpTableInfo() {
1161 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1162 if (!MJTI) return;
1163
1164 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
1165 if (JT.empty()) return;
1166
1167 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1168 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM);
1169 OutStreamer->switchSection(ReadOnlySec);
1170
1171 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1172 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
1173 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1174
1175 // If this jump table was deleted, ignore it.
1176 if (JTBBs.empty()) continue;
1177
1178 unsigned Size = AFI->getJumpTableEntrySize(JTI);
1179 emitAlignment(Align(Size));
1180 OutStreamer->emitLabel(GetJTISymbol(JTI));
1181
1182 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1183 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
1184
1185 for (auto *JTBB : JTBBs) {
1186 const MCExpr *Value =
1187 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
1188
1189 // Each entry is:
1190 // .byte/.hword (LBB - Lbase)>>2
1191 // or plain:
1192 // .word LBB - Lbase
1193 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
1194 if (Size != 4)
1196 Value, MCConstantExpr::create(2, OutContext), OutContext);
1197
1198 OutStreamer->emitValue(Value, Size);
1199 }
1200 }
1201}
1202
1203std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1205AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1206 const MachineInstr *BranchInstr,
1207 const MCSymbol *BranchLabel) const {
1208 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1209 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1211 switch (AFI->getJumpTableEntrySize(JTI)) {
1212 case 1:
1213 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1214 break;
1215 case 2:
1216 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1217 break;
1218 case 4:
1219 EntrySize = codeview::JumpTableEntrySize::Int32;
1220 break;
1221 default:
1222 llvm_unreachable("Unexpected jump table entry size");
1223 }
1224 return std::make_tuple(Base, 0, BranchLabel, EntrySize);
1225}
1226
1227void AArch64AsmPrinter::emitFunctionEntryLabel() {
1228 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1229 MF->getFunction().getCallingConv() ==
1231 MF->getInfo<AArch64FunctionInfo>()->isSVECC()) {
1232 auto *TS =
1233 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1234 TS->emitDirectiveVariantPCS(CurrentFnSym);
1235 }
1236
1238
1239 if (TM.getTargetTriple().isWindowsArm64EC() &&
1240 !MF->getFunction().hasLocalLinkage()) {
1241 // For ARM64EC targets, a function definition's name is mangled differently
1242 // from the normal symbol, emit required aliases here.
1243 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1244 OutStreamer->emitSymbolAttribute(Src, MCSA_WeakAntiDep);
1245 OutStreamer->emitAssignment(
1247 MMI->getContext()));
1248 };
1249
1250 auto getSymbolFromMetadata = [&](StringRef Name) {
1251 MCSymbol *Sym = nullptr;
1252 if (MDNode *Node = MF->getFunction().getMetadata(Name)) {
1253 StringRef NameStr = cast<MDString>(Node->getOperand(0))->getString();
1254 Sym = MMI->getContext().getOrCreateSymbol(NameStr);
1255 }
1256 return Sym;
1257 };
1258
1259 if (MCSymbol *UnmangledSym =
1260 getSymbolFromMetadata("arm64ec_unmangled_name")) {
1261 MCSymbol *ECMangledSym = getSymbolFromMetadata("arm64ec_ecmangled_name");
1262
1263 if (ECMangledSym) {
1264 // An external function, emit the alias from the unmangled symbol to
1265 // mangled symbol name and the alias from the mangled symbol to guest
1266 // exit thunk.
1267 emitFunctionAlias(UnmangledSym, ECMangledSym);
1268 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1269 } else {
1270 // A function implementation, emit the alias from the unmangled symbol
1271 // to mangled symbol name.
1272 emitFunctionAlias(UnmangledSym, CurrentFnSym);
1273 }
1274 }
1275 }
1276}
1277
1278void AArch64AsmPrinter::emitXXStructor(const DataLayout &DL,
1279 const Constant *CV) {
1280 if (const auto *CPA = dyn_cast<ConstantPtrAuth>(CV))
1281 if (CPA->hasAddressDiscriminator() &&
1282 !CPA->hasSpecialAddressDiscriminator(
1285 "unexpected address discrimination value for ctors/dtors entry, only "
1286 "'ptr inttoptr (i64 1 to ptr)' is allowed");
1287 // If we have signed pointers in xxstructors list, they'll be lowered to @AUTH
1288 // MCExpr's via AArch64AsmPrinter::lowerConstantPtrAuth. It does not look at
1289 // actual address discrimination value and only checks
1290 // hasAddressDiscriminator(), so it's OK to leave special address
1291 // discrimination value here.
1293}
1294
1295void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1296 const GlobalAlias &GA) {
1297 if (auto F = dyn_cast_or_null<Function>(GA.getAliasee())) {
1298 // Global aliases must point to a definition, but unmangled patchable
1299 // symbols are special and need to point to an undefined symbol with "EXP+"
1300 // prefix. Such undefined symbol is resolved by the linker by creating
1301 // x86 thunk that jumps back to the actual EC target.
1302 if (MDNode *Node = F->getMetadata("arm64ec_exp_name")) {
1303 StringRef ExpStr = cast<MDString>(Node->getOperand(0))->getString();
1304 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(ExpStr);
1305 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(GA.getName());
1306
1307 OutStreamer->beginCOFFSymbolDef(ExpSym);
1308 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_EXTERNAL);
1309 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION
1311 OutStreamer->endCOFFSymbolDef();
1312
1313 OutStreamer->beginCOFFSymbolDef(Sym);
1314 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_EXTERNAL);
1315 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION
1317 OutStreamer->endCOFFSymbolDef();
1318 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
1319 OutStreamer->emitAssignment(
1321 MMI->getContext()));
1322 return;
1323 }
1324 }
1326}
1327
1328/// Small jump tables contain an unsigned byte or half, representing the offset
1329/// from the lowest-addressed possible destination to the desired basic
1330/// block. Since all instructions are 4-byte aligned, this is further compressed
1331/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1332/// materialize the correct destination we need:
1333///
1334/// adr xDest, .LBB0_0
1335/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1336/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1337void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1338 const llvm::MachineInstr &MI) {
1339 Register DestReg = MI.getOperand(0).getReg();
1340 Register ScratchReg = MI.getOperand(1).getReg();
1341 Register ScratchRegW =
1342 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
1343 Register TableReg = MI.getOperand(2).getReg();
1344 Register EntryReg = MI.getOperand(3).getReg();
1345 int JTIdx = MI.getOperand(4).getIndex();
1346 int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
1347
1348 // This has to be first because the compression pass based its reachability
1349 // calculations on the start of the JumpTableDest instruction.
1350 auto Label =
1351 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
1352
1353 // If we don't already have a symbol to use as the base, use the ADR
1354 // instruction itself.
1355 if (!Label) {
1356 Label = MF->getContext().createTempSymbol();
1357 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
1358 OutStreamer.emitLabel(Label);
1359 }
1360
1361 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
1362 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
1363 .addReg(DestReg)
1364 .addExpr(LabelExpr));
1365
1366 // Load the number of instruction-steps to offset from the label.
1367 unsigned LdrOpcode;
1368 switch (Size) {
1369 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1370 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1371 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1372 default:
1373 llvm_unreachable("Unknown jump table size");
1374 }
1375
1376 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
1377 .addReg(Size == 4 ? ScratchReg : ScratchRegW)
1378 .addReg(TableReg)
1379 .addReg(EntryReg)
1380 .addImm(0)
1381 .addImm(Size == 1 ? 0 : 1));
1382
1383 // Add to the already materialized base label address, multiplying by 4 if
1384 // compressed.
1385 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1386 .addReg(DestReg)
1387 .addReg(DestReg)
1388 .addReg(ScratchReg)
1389 .addImm(Size == 4 ? 0 : 2));
1390}
1391
1392void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1393 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1394 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1395
1396 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1397 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1398
1399 // Emit:
1400 // mov x17, #<size of table> ; depending on table size, with MOVKs
1401 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1402 // csel x16, x16, xzr, ls ; check for index overflow
1403 //
1404 // adrp x17, Ltable@PAGE ; materialize table address
1405 // add x17, Ltable@PAGEOFF
1406 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1407 //
1408 // Lanchor:
1409 // adr x17, Lanchor ; compute target address
1410 // add x16, x17, x16
1411 // br x16 ; branch to target
1412
1413 MachineOperand JTOp = MI.getOperand(0);
1414
1415 unsigned JTI = JTOp.getIndex();
1416 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1417 "unsupported compressed jump table");
1418
1419 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1420
1421 // cmp only supports a 12-bit immediate. If we need more, materialize the
1422 // immediate, using x17 as a scratch register.
1423 uint64_t MaxTableEntry = NumTableEntries - 1;
1424 if (isUInt<12>(MaxTableEntry)) {
1425 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXri)
1426 .addReg(AArch64::XZR)
1427 .addReg(AArch64::X16)
1428 .addImm(MaxTableEntry)
1429 .addImm(0));
1430 } else {
1431 emitMOVZ(AArch64::X17, static_cast<uint16_t>(MaxTableEntry), 0);
1432 // It's sad that we have to manually materialize instructions, but we can't
1433 // trivially reuse the main pseudo expansion logic.
1434 // A MOVK sequence is easy enough to generate and handles the general case.
1435 for (int Offset = 16; Offset < 64; Offset += 16) {
1436 if ((MaxTableEntry >> Offset) == 0)
1437 break;
1438 emitMOVK(AArch64::X17, static_cast<uint16_t>(MaxTableEntry >> Offset),
1439 Offset);
1440 }
1441 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXrs)
1442 .addReg(AArch64::XZR)
1443 .addReg(AArch64::X16)
1444 .addReg(AArch64::X17)
1445 .addImm(0));
1446 }
1447
1448 // This picks entry #0 on failure.
1449 // We might want to trap instead.
1450 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::CSELXr)
1451 .addReg(AArch64::X16)
1452 .addReg(AArch64::X16)
1453 .addReg(AArch64::XZR)
1454 .addImm(AArch64CC::LS));
1455
1456 // Prepare the @PAGE/@PAGEOFF low/high operands.
1457 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1458 MCOperand JTMCHi, JTMCLo;
1459
1460 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1461 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1462
1463 MCInstLowering.lowerOperand(JTMOHi, JTMCHi);
1464 MCInstLowering.lowerOperand(JTMOLo, JTMCLo);
1465
1466 EmitToStreamer(
1467 *OutStreamer,
1468 MCInstBuilder(AArch64::ADRP).addReg(AArch64::X17).addOperand(JTMCHi));
1469
1470 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXri)
1471 .addReg(AArch64::X17)
1472 .addReg(AArch64::X17)
1473 .addOperand(JTMCLo)
1474 .addImm(0));
1475
1476 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
1477 .addReg(AArch64::X16)
1478 .addReg(AArch64::X17)
1479 .addReg(AArch64::X16)
1480 .addImm(0)
1481 .addImm(1));
1482
1483 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1484 const auto *AdrLabelE = MCSymbolRefExpr::create(AdrLabel, MF->getContext());
1485 AArch64FI->setJumpTableEntryInfo(JTI, 4, AdrLabel);
1486
1487 OutStreamer->emitLabel(AdrLabel);
1488 EmitToStreamer(
1489 *OutStreamer,
1490 MCInstBuilder(AArch64::ADR).addReg(AArch64::X17).addExpr(AdrLabelE));
1491
1492 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1493 .addReg(AArch64::X16)
1494 .addReg(AArch64::X17)
1495 .addReg(AArch64::X16)
1496 .addImm(0));
1497
1498 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BR).addReg(AArch64::X16));
1499}
1500
1501void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1502 const llvm::MachineInstr &MI) {
1503 unsigned Opcode = MI.getOpcode();
1504 assert(STI->hasMOPS());
1505 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1506
1507 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1508 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1509 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1510 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1511 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1512 if (Opcode == AArch64::MOPSMemorySetPseudo)
1513 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1514 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1515 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1516 llvm_unreachable("Unhandled memory operation pseudo");
1517 }();
1518 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1519 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1520
1521 for (auto Op : Ops) {
1522 int i = 0;
1523 auto MCIB = MCInstBuilder(Op);
1524 // Destination registers
1525 MCIB.addReg(MI.getOperand(i++).getReg());
1526 MCIB.addReg(MI.getOperand(i++).getReg());
1527 if (!IsSet)
1528 MCIB.addReg(MI.getOperand(i++).getReg());
1529 // Input registers
1530 MCIB.addReg(MI.getOperand(i++).getReg());
1531 MCIB.addReg(MI.getOperand(i++).getReg());
1532 MCIB.addReg(MI.getOperand(i++).getReg());
1533
1534 EmitToStreamer(OutStreamer, MCIB);
1535 }
1536}
1537
1538void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1539 const MachineInstr &MI) {
1540 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1541
1542 auto &Ctx = OutStreamer.getContext();
1543 MCSymbol *MILabel = Ctx.createTempSymbol();
1544 OutStreamer.emitLabel(MILabel);
1545
1546 SM.recordStackMap(*MILabel, MI);
1547 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1548
1549 // Scan ahead to trim the shadow.
1550 const MachineBasicBlock &MBB = *MI.getParent();
1552 ++MII;
1553 while (NumNOPBytes > 0) {
1554 if (MII == MBB.end() || MII->isCall() ||
1555 MII->getOpcode() == AArch64::DBG_VALUE ||
1556 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1557 MII->getOpcode() == TargetOpcode::STACKMAP)
1558 break;
1559 ++MII;
1560 NumNOPBytes -= 4;
1561 }
1562
1563 // Emit nops.
1564 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1565 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1566}
1567
1568// Lower a patchpoint of the form:
1569// [<def>], <id>, <numBytes>, <target>, <numArgs>
1570void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1571 const MachineInstr &MI) {
1572 auto &Ctx = OutStreamer.getContext();
1573 MCSymbol *MILabel = Ctx.createTempSymbol();
1574 OutStreamer.emitLabel(MILabel);
1575 SM.recordPatchPoint(*MILabel, MI);
1576
1577 PatchPointOpers Opers(&MI);
1578
1579 int64_t CallTarget = Opers.getCallTarget().getImm();
1580 unsigned EncodedBytes = 0;
1581 if (CallTarget) {
1582 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1583 "High 16 bits of call target should be zero.");
1584 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
1585 EncodedBytes = 16;
1586 // Materialize the jump address:
1587 emitMOVZ(ScratchReg, (CallTarget >> 32) & 0xFFFF, 32);
1588 emitMOVK(ScratchReg, (CallTarget >> 16) & 0xFFFF, 16);
1589 emitMOVK(ScratchReg, CallTarget & 0xFFFF, 0);
1590 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1591 }
1592 // Emit padding.
1593 unsigned NumBytes = Opers.getNumPatchBytes();
1594 assert(NumBytes >= EncodedBytes &&
1595 "Patchpoint can't request size less than the length of a call.");
1596 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1597 "Invalid number of NOP bytes requested!");
1598 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1599 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1600}
1601
1602void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1603 const MachineInstr &MI) {
1604 StatepointOpers SOpers(&MI);
1605 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1606 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1607 for (unsigned i = 0; i < PatchBytes; i += 4)
1608 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1609 } else {
1610 // Lower call target and choose correct opcode
1611 const MachineOperand &CallTarget = SOpers.getCallTarget();
1612 MCOperand CallTargetMCOp;
1613 unsigned CallOpcode;
1614 switch (CallTarget.getType()) {
1617 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1618 CallOpcode = AArch64::BL;
1619 break;
1621 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1622 CallOpcode = AArch64::BL;
1623 break;
1625 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1626 CallOpcode = AArch64::BLR;
1627 break;
1628 default:
1629 llvm_unreachable("Unsupported operand type in statepoint call target");
1630 break;
1631 }
1632
1633 EmitToStreamer(OutStreamer,
1634 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1635 }
1636
1637 auto &Ctx = OutStreamer.getContext();
1638 MCSymbol *MILabel = Ctx.createTempSymbol();
1639 OutStreamer.emitLabel(MILabel);
1640 SM.recordStatepoint(*MILabel, MI);
1641}
1642
1643void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1644 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1645 // <opcode>, <operands>
1646
1647 Register DefRegister = FaultingMI.getOperand(0).getReg();
1649 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1650 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1651 unsigned Opcode = FaultingMI.getOperand(3).getImm();
1652 unsigned OperandsBeginIdx = 4;
1653
1654 auto &Ctx = OutStreamer->getContext();
1655 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1656 OutStreamer->emitLabel(FaultingLabel);
1657
1658 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1659 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1660
1661 MCInst MI;
1662 MI.setOpcode(Opcode);
1663
1664 if (DefRegister != (Register)0)
1665 MI.addOperand(MCOperand::createReg(DefRegister));
1666
1667 for (const MachineOperand &MO :
1668 llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx)) {
1669 MCOperand Dest;
1670 lowerOperand(MO, Dest);
1671 MI.addOperand(Dest);
1672 }
1673
1674 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1675 EmitToStreamer(MI);
1676}
1677
1678void AArch64AsmPrinter::emitMovXReg(Register Dest, Register Src) {
1679 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
1680 .addReg(Dest)
1681 .addReg(AArch64::XZR)
1682 .addReg(Src)
1683 .addImm(0));
1684}
1685
1686void AArch64AsmPrinter::emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift) {
1687 bool Is64Bit = AArch64::GPR64RegClass.contains(Dest);
1688 EmitToStreamer(*OutStreamer,
1689 MCInstBuilder(Is64Bit ? AArch64::MOVZXi : AArch64::MOVZWi)
1690 .addReg(Dest)
1691 .addImm(Imm)
1692 .addImm(Shift));
1693}
1694
1695void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) {
1696 bool Is64Bit = AArch64::GPR64RegClass.contains(Dest);
1697 EmitToStreamer(*OutStreamer,
1698 MCInstBuilder(Is64Bit ? AArch64::MOVKXi : AArch64::MOVKWi)
1699 .addReg(Dest)
1700 .addReg(Dest)
1701 .addImm(Imm)
1702 .addImm(Shift));
1703}
1704
1705void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1706 Register DestReg = MI.getOperand(0).getReg();
1707 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
1708 STI->isNeonAvailable()) {
1709 // Convert H/S register to corresponding D register
1710 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1711 DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1712 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1713 DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1714 else
1715 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1716
1717 MCInst MOVI;
1718 MOVI.setOpcode(AArch64::MOVID);
1719 MOVI.addOperand(MCOperand::createReg(DestReg));
1720 MOVI.addOperand(MCOperand::createImm(0));
1721 EmitToStreamer(*OutStreamer, MOVI);
1722 } else {
1723 MCInst FMov;
1724 switch (MI.getOpcode()) {
1725 default: llvm_unreachable("Unexpected opcode");
1726 case AArch64::FMOVH0:
1727 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
1728 if (!STI->hasFullFP16())
1729 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
1730 FMov.addOperand(MCOperand::createReg(DestReg));
1731 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1732 break;
1733 case AArch64::FMOVS0:
1734 FMov.setOpcode(AArch64::FMOVWSr);
1735 FMov.addOperand(MCOperand::createReg(DestReg));
1736 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1737 break;
1738 case AArch64::FMOVD0:
1739 FMov.setOpcode(AArch64::FMOVXDr);
1740 FMov.addOperand(MCOperand::createReg(DestReg));
1741 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1742 break;
1743 }
1744 EmitToStreamer(*OutStreamer, FMov);
1745 }
1746}
1747
1748Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
1749 Register AddrDisc,
1750 Register ScratchReg,
1751 bool MayUseAddrAsScratch) {
1752 assert(ScratchReg == AArch64::X16 || ScratchReg == AArch64::X17);
1753 // So far we've used NoRegister in pseudos. Now we need real encodings.
1754 if (AddrDisc == AArch64::NoRegister)
1755 AddrDisc = AArch64::XZR;
1756
1757 // If there is no constant discriminator, there's no blend involved:
1758 // just use the address discriminator register as-is (XZR or not).
1759 if (!Disc)
1760 return AddrDisc;
1761
1762 // If there's only a constant discriminator, MOV it into the scratch register.
1763 if (AddrDisc == AArch64::XZR) {
1764 emitMOVZ(ScratchReg, Disc, 0);
1765 return ScratchReg;
1766 }
1767
1768 // If there are both, emit a blend into the scratch register.
1769
1770 // Check if we can save one MOV instruction.
1771 assert(MayUseAddrAsScratch || ScratchReg != AddrDisc);
1772 bool AddrDiscIsSafe = AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
1773 if (MayUseAddrAsScratch && AddrDiscIsSafe)
1774 ScratchReg = AddrDisc;
1775 else
1776 emitMovXReg(ScratchReg, AddrDisc);
1777
1778 emitMOVK(ScratchReg, Disc, 48);
1779 return ScratchReg;
1780}
1781
1782/// Emits a code sequence to check an authenticated pointer value.
1783///
1784/// If OnFailure argument is passed, jump there on check failure instead
1785/// of proceeding to the next instruction (only if ShouldTrap is false).
1786void AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue(
1787 Register TestedReg, Register ScratchReg, AArch64PACKey::ID Key,
1788 AArch64PAuth::AuthCheckMethod Method, bool ShouldTrap,
1789 const MCSymbol *OnFailure) {
1790 // Insert a sequence to check if authentication of TestedReg succeeded,
1791 // such as:
1792 //
1793 // - checked and clearing:
1794 // ; x16 is TestedReg, x17 is ScratchReg
1795 // mov x17, x16
1796 // xpaci x17
1797 // cmp x16, x17
1798 // b.eq Lsuccess
1799 // mov x16, x17
1800 // b Lend
1801 // Lsuccess:
1802 // ; skipped if authentication failed
1803 // Lend:
1804 // ...
1805 //
1806 // - checked and trapping:
1807 // mov x17, x16
1808 // xpaci x17
1809 // cmp x16, x17
1810 // b.eq Lsuccess
1811 // brk #<0xc470 + aut key>
1812 // Lsuccess:
1813 // ...
1814 //
1815 // See the documentation on AuthCheckMethod enumeration constants for
1816 // the specific code sequences that can be used to perform the check.
1818
1819 if (Method == AuthCheckMethod::None)
1820 return;
1821 if (Method == AuthCheckMethod::DummyLoad) {
1822 EmitToStreamer(MCInstBuilder(AArch64::LDRWui)
1823 .addReg(getWRegFromXReg(ScratchReg))
1824 .addReg(TestedReg)
1825 .addImm(0));
1826 assert(ShouldTrap && !OnFailure && "DummyLoad always traps on error");
1827 return;
1828 }
1829
1830 MCSymbol *SuccessSym = createTempSymbol("auth_success_");
1831 if (Method == AuthCheckMethod::XPAC || Method == AuthCheckMethod::XPACHint) {
1832 // mov Xscratch, Xtested
1833 emitMovXReg(ScratchReg, TestedReg);
1834
1835 if (Method == AuthCheckMethod::XPAC) {
1836 // xpac(i|d) Xscratch
1837 unsigned XPACOpc = getXPACOpcodeForKey(Key);
1838 EmitToStreamer(
1839 MCInstBuilder(XPACOpc).addReg(ScratchReg).addReg(ScratchReg));
1840 } else {
1841 // xpaclri
1842
1843 // Note that this method applies XPAC to TestedReg instead of ScratchReg.
1844 assert(TestedReg == AArch64::LR &&
1845 "XPACHint mode is only compatible with checking the LR register");
1846 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
1847 "XPACHint mode is only compatible with I-keys");
1848 EmitToStreamer(MCInstBuilder(AArch64::XPACLRI));
1849 }
1850
1851 // cmp Xtested, Xscratch
1852 EmitToStreamer(MCInstBuilder(AArch64::SUBSXrs)
1853 .addReg(AArch64::XZR)
1854 .addReg(TestedReg)
1855 .addReg(ScratchReg)
1856 .addImm(0));
1857
1858 // b.eq Lsuccess
1859 EmitToStreamer(
1860 MCInstBuilder(AArch64::Bcc)
1861 .addImm(AArch64CC::EQ)
1862 .addExpr(MCSymbolRefExpr::create(SuccessSym, OutContext)));
1863 } else if (Method == AuthCheckMethod::HighBitsNoTBI) {
1864 // eor Xscratch, Xtested, Xtested, lsl #1
1865 EmitToStreamer(MCInstBuilder(AArch64::EORXrs)
1866 .addReg(ScratchReg)
1867 .addReg(TestedReg)
1868 .addReg(TestedReg)
1869 .addImm(1));
1870 // tbz Xscratch, #62, Lsuccess
1871 EmitToStreamer(
1872 MCInstBuilder(AArch64::TBZX)
1873 .addReg(ScratchReg)
1874 .addImm(62)
1875 .addExpr(MCSymbolRefExpr::create(SuccessSym, OutContext)));
1876 } else {
1877 llvm_unreachable("Unsupported check method");
1878 }
1879
1880 if (ShouldTrap) {
1881 assert(!OnFailure && "Cannot specify OnFailure with ShouldTrap");
1882 // Trapping sequences do a 'brk'.
1883 // brk #<0xc470 + aut key>
1884 EmitToStreamer(MCInstBuilder(AArch64::BRK).addImm(0xc470 | Key));
1885 } else {
1886 // Non-trapping checked sequences return the stripped result in TestedReg,
1887 // skipping over success-only code (such as re-signing the pointer) if
1888 // there is one.
1889 // Note that this can introduce an authentication oracle (such as based on
1890 // the high bits of the re-signed value).
1891
1892 // FIXME: The XPAC method can be optimized by applying XPAC to TestedReg
1893 // instead of ScratchReg, thus eliminating one `mov` instruction.
1894 // Both XPAC and XPACHint can be further optimized by not using a
1895 // conditional branch jumping over an unconditional one.
1896
1897 switch (Method) {
1898 case AuthCheckMethod::XPACHint:
1899 // LR is already XPAC-ed at this point.
1900 break;
1901 case AuthCheckMethod::XPAC:
1902 // mov Xtested, Xscratch
1903 emitMovXReg(TestedReg, ScratchReg);
1904 break;
1905 default:
1906 // If Xtested was not XPAC-ed so far, emit XPAC here.
1907 // xpac(i|d) Xtested
1908 unsigned XPACOpc = getXPACOpcodeForKey(Key);
1909 EmitToStreamer(
1910 MCInstBuilder(XPACOpc).addReg(TestedReg).addReg(TestedReg));
1911 }
1912
1913 if (OnFailure) {
1914 // b Lend
1915 EmitToStreamer(
1916 MCInstBuilder(AArch64::B)
1917 .addExpr(MCSymbolRefExpr::create(OnFailure, OutContext)));
1918 }
1919 }
1920
1921 // If the auth check succeeds, we can continue.
1922 // Lsuccess:
1923 OutStreamer->emitLabel(SuccessSym);
1924}
1925
1926// With Pointer Authentication, it may be needed to explicitly check the
1927// authenticated value in LR before performing a tail call.
1928// Otherwise, the callee may re-sign the invalid return address,
1929// introducing a signing oracle.
1930void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) {
1931 if (!AArch64FI->shouldSignReturnAddress(*MF))
1932 return;
1933
1934 auto LRCheckMethod = STI->getAuthenticatedLRCheckMethod(*MF);
1935 if (LRCheckMethod == AArch64PAuth::AuthCheckMethod::None)
1936 return;
1937
1938 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
1939 Register ScratchReg =
1940 TC->readsRegister(AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
1941 assert(!TC->readsRegister(ScratchReg, TRI) &&
1942 "Neither x16 nor x17 is available as a scratch register");
1944 AArch64FI->shouldSignWithBKey() ? AArch64PACKey::IB : AArch64PACKey::IA;
1945 emitPtrauthCheckAuthenticatedValue(
1946 AArch64::LR, ScratchReg, Key, LRCheckMethod,
1947 /*ShouldTrap=*/true, /*OnFailure=*/nullptr);
1948}
1949
1950void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
1951 const bool IsAUTPAC = MI->getOpcode() == AArch64::AUTPAC;
1952
1953 // We expand AUT/AUTPAC into a sequence of the form
1954 //
1955 // ; authenticate x16
1956 // ; check pointer in x16
1957 // Lsuccess:
1958 // ; sign x16 (if AUTPAC)
1959 // Lend: ; if not trapping on failure
1960 //
1961 // with the checking sequence chosen depending on whether/how we should check
1962 // the pointer and whether we should trap on failure.
1963
1964 // By default, auth/resign sequences check for auth failures.
1965 bool ShouldCheck = true;
1966 // In the checked sequence, we only trap if explicitly requested.
1967 bool ShouldTrap = MF->getFunction().hasFnAttribute("ptrauth-auth-traps");
1968
1969 // On an FPAC CPU, you get traps whether you want them or not: there's
1970 // no point in emitting checks or traps.
1971 if (STI->hasFPAC())
1972 ShouldCheck = ShouldTrap = false;
1973
1974 // However, command-line flags can override this, for experimentation.
1975 switch (PtrauthAuthChecks) {
1977 break;
1979 ShouldCheck = ShouldTrap = false;
1980 break;
1982 ShouldCheck = true;
1983 ShouldTrap = false;
1984 break;
1986 ShouldCheck = ShouldTrap = true;
1987 break;
1988 }
1989
1990 auto AUTKey = (AArch64PACKey::ID)MI->getOperand(0).getImm();
1991 uint64_t AUTDisc = MI->getOperand(1).getImm();
1992 unsigned AUTAddrDisc = MI->getOperand(2).getReg();
1993
1994 // Compute aut discriminator into x17
1995 assert(isUInt<16>(AUTDisc));
1996 Register AUTDiscReg =
1997 emitPtrauthDiscriminator(AUTDisc, AUTAddrDisc, AArch64::X17);
1998 bool AUTZero = AUTDiscReg == AArch64::XZR;
1999 unsigned AUTOpc = getAUTOpcodeForKey(AUTKey, AUTZero);
2000
2001 // autiza x16 ; if AUTZero
2002 // autia x16, x17 ; if !AUTZero
2003 MCInst AUTInst;
2004 AUTInst.setOpcode(AUTOpc);
2005 AUTInst.addOperand(MCOperand::createReg(AArch64::X16));
2006 AUTInst.addOperand(MCOperand::createReg(AArch64::X16));
2007 if (!AUTZero)
2008 AUTInst.addOperand(MCOperand::createReg(AUTDiscReg));
2009 EmitToStreamer(*OutStreamer, AUTInst);
2010
2011 // Unchecked or checked-but-non-trapping AUT is just an "AUT": we're done.
2012 if (!IsAUTPAC && (!ShouldCheck || !ShouldTrap))
2013 return;
2014
2015 MCSymbol *EndSym = nullptr;
2016
2017 if (ShouldCheck) {
2018 if (IsAUTPAC && !ShouldTrap)
2019 EndSym = createTempSymbol("resign_end_");
2020
2021 emitPtrauthCheckAuthenticatedValue(AArch64::X16, AArch64::X17, AUTKey,
2022 AArch64PAuth::AuthCheckMethod::XPAC,
2023 ShouldTrap, EndSym);
2024 }
2025
2026 // We already emitted unchecked and checked-but-non-trapping AUTs.
2027 // That left us with trapping AUTs, and AUTPACs.
2028 // Trapping AUTs don't need PAC: we're done.
2029 if (!IsAUTPAC)
2030 return;
2031
2032 auto PACKey = (AArch64PACKey::ID)MI->getOperand(3).getImm();
2033 uint64_t PACDisc = MI->getOperand(4).getImm();
2034 unsigned PACAddrDisc = MI->getOperand(5).getReg();
2035
2036 // Compute pac discriminator into x17
2037 assert(isUInt<16>(PACDisc));
2038 Register PACDiscReg =
2039 emitPtrauthDiscriminator(PACDisc, PACAddrDisc, AArch64::X17);
2040 bool PACZero = PACDiscReg == AArch64::XZR;
2041 unsigned PACOpc = getPACOpcodeForKey(PACKey, PACZero);
2042
2043 // pacizb x16 ; if PACZero
2044 // pacib x16, x17 ; if !PACZero
2045 MCInst PACInst;
2046 PACInst.setOpcode(PACOpc);
2047 PACInst.addOperand(MCOperand::createReg(AArch64::X16));
2048 PACInst.addOperand(MCOperand::createReg(AArch64::X16));
2049 if (!PACZero)
2050 PACInst.addOperand(MCOperand::createReg(PACDiscReg));
2051 EmitToStreamer(*OutStreamer, PACInst);
2052
2053 // Lend:
2054 if (EndSym)
2055 OutStreamer->emitLabel(EndSym);
2056}
2057
2058void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2059 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2060 unsigned BrTarget = MI->getOperand(0).getReg();
2061
2062 auto Key = (AArch64PACKey::ID)MI->getOperand(1).getImm();
2063 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2064 "Invalid auth call key");
2065
2066 uint64_t Disc = MI->getOperand(2).getImm();
2067 assert(isUInt<16>(Disc));
2068
2069 unsigned AddrDisc = MI->getOperand(3).getReg();
2070
2071 // Make sure AddrDisc is solely used to compute the discriminator.
2072 // While hardly meaningful, it is still possible to describe an authentication
2073 // of a pointer against its own value (instead of storage address) with
2074 // intrinsics, so use report_fatal_error instead of assert.
2075 if (BrTarget == AddrDisc)
2076 report_fatal_error("Branch target is signed with its own value");
2077
2078 // If we are printing BLRA pseudo instruction, then x16 and x17 are
2079 // implicit-def'ed by the MI and AddrDisc is not used as any other input, so
2080 // try to save one MOV by setting MayUseAddrAsScratch.
2081 // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not
2082 // declared as clobbering x16/x17.
2083 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, AArch64::X17,
2084 /*MayUseAddrAsScratch=*/IsCall);
2085 bool IsZeroDisc = DiscReg == AArch64::XZR;
2086
2087 unsigned Opc;
2088 if (IsCall) {
2089 if (Key == AArch64PACKey::IA)
2090 Opc = IsZeroDisc ? AArch64::BLRAAZ : AArch64::BLRAA;
2091 else
2092 Opc = IsZeroDisc ? AArch64::BLRABZ : AArch64::BLRAB;
2093 } else {
2094 if (Key == AArch64PACKey::IA)
2095 Opc = IsZeroDisc ? AArch64::BRAAZ : AArch64::BRAA;
2096 else
2097 Opc = IsZeroDisc ? AArch64::BRABZ : AArch64::BRAB;
2098 }
2099
2100 MCInst BRInst;
2101 BRInst.setOpcode(Opc);
2102 BRInst.addOperand(MCOperand::createReg(BrTarget));
2103 if (!IsZeroDisc)
2104 BRInst.addOperand(MCOperand::createReg(DiscReg));
2105 EmitToStreamer(*OutStreamer, BRInst);
2106}
2107
2108const MCExpr *
2109AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2110 MCContext &Ctx = OutContext;
2111
2112 // Figure out the base symbol and the addend, if any.
2113 APInt Offset(64, 0);
2114 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2115 getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2116
2117 auto *BaseGVB = dyn_cast<GlobalValue>(BaseGV);
2118
2119 // If we can't understand the referenced ConstantExpr, there's nothing
2120 // else we can do: emit an error.
2121 if (!BaseGVB) {
2122 BaseGV->getContext().emitError(
2123 "cannot resolve target base/addend of ptrauth constant");
2124 return nullptr;
2125 }
2126
2127 // If there is an addend, turn that into the appropriate MCExpr.
2128 const MCExpr *Sym = MCSymbolRefExpr::create(getSymbol(BaseGVB), Ctx);
2129 if (Offset.sgt(0))
2131 Sym, MCConstantExpr::create(Offset.getSExtValue(), Ctx), Ctx);
2132 else if (Offset.slt(0))
2134 Sym, MCConstantExpr::create((-Offset).getSExtValue(), Ctx), Ctx);
2135
2136 uint64_t KeyID = CPA.getKey()->getZExtValue();
2137 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2138 // AArch64AuthMCExpr::printImpl, so fail fast.
2139 if (KeyID > AArch64PACKey::LAST)
2140 report_fatal_error("AArch64 PAC Key ID '" + Twine(KeyID) +
2141 "' out of range [0, " +
2142 Twine((unsigned)AArch64PACKey::LAST) + "]");
2143
2144 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2145 if (!isUInt<16>(Disc))
2146 report_fatal_error("AArch64 PAC Discriminator '" + Twine(Disc) +
2147 "' out of range [0, 0xFFFF]");
2148
2149 // Finally build the complete @AUTH expr.
2150 return AArch64AuthMCExpr::create(Sym, Disc, AArch64PACKey::ID(KeyID),
2151 CPA.hasAddressDiscriminator(), Ctx);
2152}
2153
2154void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2155 unsigned DstReg = MI.getOperand(0).getReg();
2156 const MachineOperand &GAOp = MI.getOperand(1);
2157 const uint64_t KeyC = MI.getOperand(2).getImm();
2158 assert(KeyC <= AArch64PACKey::LAST &&
2159 "key is out of range [0, AArch64PACKey::LAST]");
2160 const auto Key = (AArch64PACKey::ID)KeyC;
2161 const uint64_t Disc = MI.getOperand(3).getImm();
2162 assert(isUInt<16>(Disc) &&
2163 "constant discriminator is out of range [0, 0xffff]");
2164
2165 // Emit instruction sequence like the following:
2166 // ADRP x16, symbol$auth_ptr$key$disc
2167 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2168 //
2169 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2170 // to symbol.
2171 MCSymbol *AuthPtrStubSym;
2172 if (TM.getTargetTriple().isOSBinFormatELF()) {
2173 const auto &TLOF =
2174 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2175
2176 assert(GAOp.getOffset() == 0 &&
2177 "non-zero offset for $auth_ptr$ stub slots is not supported");
2178 const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
2179 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, GASym, Key, Disc);
2180 } else {
2181 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2182 "LOADauthptrstatic is implemented only for MachO/ELF");
2183
2184 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2185 getObjFileLowering());
2186
2187 assert(GAOp.getOffset() == 0 &&
2188 "non-zero offset for $auth_ptr$ stub slots is not supported");
2189 const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
2190 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, GASym, Key, Disc);
2191 }
2192
2193 MachineOperand StubMOHi =
2196 AuthPtrStubSym, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2197 MCOperand StubMCHi, StubMCLo;
2198
2199 MCInstLowering.lowerOperand(StubMOHi, StubMCHi);
2200 MCInstLowering.lowerOperand(StubMOLo, StubMCLo);
2201
2202 EmitToStreamer(
2203 *OutStreamer,
2204 MCInstBuilder(AArch64::ADRP).addReg(DstReg).addOperand(StubMCHi));
2205
2206 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRXui)
2207 .addReg(DstReg)
2208 .addReg(DstReg)
2209 .addOperand(StubMCLo));
2210}
2211
2212void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2213 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2214 const bool IsELFSignedGOT = MI.getParent()
2215 ->getParent()
2216 ->getInfo<AArch64FunctionInfo>()
2217 ->hasELFSignedGOT();
2218 MachineOperand GAOp = MI.getOperand(0);
2219 const uint64_t KeyC = MI.getOperand(1).getImm();
2220 assert(KeyC <= AArch64PACKey::LAST &&
2221 "key is out of range [0, AArch64PACKey::LAST]");
2222 const auto Key = (AArch64PACKey::ID)KeyC;
2223 const unsigned AddrDisc = MI.getOperand(2).getReg();
2224 const uint64_t Disc = MI.getOperand(3).getImm();
2225 assert(isUInt<16>(Disc) &&
2226 "constant discriminator is out of range [0, 0xffff]");
2227
2228 const int64_t Offset = GAOp.getOffset();
2229 GAOp.setOffset(0);
2230
2231 // Emit:
2232 // target materialization:
2233 // - via GOT:
2234 // - unsigned GOT:
2235 // adrp x16, :got:target
2236 // ldr x16, [x16, :got_lo12:target]
2237 // add offset to x16 if offset != 0
2238 // - ELF signed GOT:
2239 // adrp x17, :got:target
2240 // add x17, x17, :got_auth_lo12:target
2241 // ldr x16, [x17]
2242 // aut{i|d}a x16, x17
2243 // check+trap sequence (if no FPAC)
2244 // add offset to x16 if offset != 0
2245 //
2246 // - direct:
2247 // adrp x16, target
2248 // add x16, x16, :lo12:target
2249 // add offset to x16 if offset != 0
2250 //
2251 // add offset to x16:
2252 // - abs(offset) fits 24 bits:
2253 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
2254 // - abs(offset) does not fit 24 bits:
2255 // - offset < 0:
2256 // movn+movk sequence filling x17 register with the offset (up to 4
2257 // instructions)
2258 // add x16, x16, x17
2259 // - offset > 0:
2260 // movz+movk sequence filling x17 register with the offset (up to 4
2261 // instructions)
2262 // add x16, x16, x17
2263 //
2264 // signing:
2265 // - 0 discriminator:
2266 // paciza x16
2267 // - Non-0 discriminator, no address discriminator:
2268 // mov x17, #Disc
2269 // pacia x16, x17
2270 // - address discriminator (with potentially folded immediate discriminator):
2271 // pacia x16, xAddrDisc
2272
2273 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
2274 MCOperand GAMCHi, GAMCLo;
2275
2276 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
2277 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2278 if (IsGOTLoad) {
2279 GAMOHi.addTargetFlag(AArch64II::MO_GOT);
2280 GAMOLo.addTargetFlag(AArch64II::MO_GOT);
2281 }
2282
2283 MCInstLowering.lowerOperand(GAMOHi, GAMCHi);
2284 MCInstLowering.lowerOperand(GAMOLo, GAMCLo);
2285
2286 EmitToStreamer(
2287 MCInstBuilder(AArch64::ADRP)
2288 .addReg(IsGOTLoad && IsELFSignedGOT ? AArch64::X17 : AArch64::X16)
2289 .addOperand(GAMCHi));
2290
2291 if (IsGOTLoad) {
2292 if (IsELFSignedGOT) {
2293 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
2294 .addReg(AArch64::X17)
2295 .addReg(AArch64::X17)
2296 .addOperand(GAMCLo)
2297 .addImm(0));
2298
2299 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
2300 .addReg(AArch64::X16)
2301 .addReg(AArch64::X17)
2302 .addImm(0));
2303
2304 assert(GAOp.isGlobal());
2305 assert(GAOp.getGlobal()->getValueType() != nullptr);
2306 unsigned AuthOpcode = GAOp.getGlobal()->getValueType()->isFunctionTy()
2307 ? AArch64::AUTIA
2308 : AArch64::AUTDA;
2309
2310 EmitToStreamer(MCInstBuilder(AuthOpcode)
2311 .addReg(AArch64::X16)
2312 .addReg(AArch64::X16)
2313 .addReg(AArch64::X17));
2314
2315 if (!STI->hasFPAC()) {
2316 auto AuthKey = (AuthOpcode == AArch64::AUTIA ? AArch64PACKey::IA
2318
2319 emitPtrauthCheckAuthenticatedValue(AArch64::X16, AArch64::X17, AuthKey,
2320 AArch64PAuth::AuthCheckMethod::XPAC,
2321 /*ShouldTrap=*/true,
2322 /*OnFailure=*/nullptr);
2323 }
2324 } else {
2325 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
2326 .addReg(AArch64::X16)
2327 .addReg(AArch64::X16)
2328 .addOperand(GAMCLo));
2329 }
2330 } else {
2331 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
2332 .addReg(AArch64::X16)
2333 .addReg(AArch64::X16)
2334 .addOperand(GAMCLo)
2335 .addImm(0));
2336 }
2337
2338 if (Offset != 0) {
2339 const uint64_t AbsOffset = (Offset > 0 ? Offset : -((uint64_t)Offset));
2340 const bool IsNeg = Offset < 0;
2341 if (isUInt<24>(AbsOffset)) {
2342 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2343 BitPos += 12) {
2344 EmitToStreamer(
2345 MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2346 .addReg(AArch64::X16)
2347 .addReg(AArch64::X16)
2348 .addImm((AbsOffset >> BitPos) & 0xfff)
2349 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, BitPos)));
2350 }
2351 } else {
2352 const uint64_t UOffset = Offset;
2353 EmitToStreamer(MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2354 .addReg(AArch64::X17)
2355 .addImm((IsNeg ? ~UOffset : UOffset) & 0xffff)
2356 .addImm(/*shift=*/0));
2357 auto NeedMovk = [IsNeg, UOffset](int BitPos) -> bool {
2358 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2359 uint64_t Shifted = UOffset >> BitPos;
2360 if (!IsNeg)
2361 return Shifted != 0;
2362 for (int I = 0; I != 64 - BitPos; I += 16)
2363 if (((Shifted >> I) & 0xffff) != 0xffff)
2364 return true;
2365 return false;
2366 };
2367 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16)
2368 emitMOVK(AArch64::X17, (UOffset >> BitPos) & 0xffff, BitPos);
2369
2370 EmitToStreamer(MCInstBuilder(AArch64::ADDXrs)
2371 .addReg(AArch64::X16)
2372 .addReg(AArch64::X16)
2373 .addReg(AArch64::X17)
2374 .addImm(/*shift=*/0));
2375 }
2376 }
2377
2378 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, AArch64::X17);
2379
2380 auto MIB = MCInstBuilder(getPACOpcodeForKey(Key, DiscReg == AArch64::XZR))
2381 .addReg(AArch64::X16)
2382 .addReg(AArch64::X16);
2383 if (DiscReg != AArch64::XZR)
2384 MIB.addReg(DiscReg);
2385 EmitToStreamer(MIB);
2386}
2387
2388void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
2389 Register DstReg = MI.getOperand(0).getReg();
2390 Register AuthResultReg = STI->hasFPAC() ? DstReg : AArch64::X16;
2391 const MachineOperand &GAMO = MI.getOperand(1);
2392 assert(GAMO.getOffset() == 0);
2393
2394 if (MI.getMF()->getTarget().getCodeModel() == CodeModel::Tiny) {
2395 MCOperand GAMC;
2396 MCInstLowering.lowerOperand(GAMO, GAMC);
2397 EmitToStreamer(
2398 MCInstBuilder(AArch64::ADR).addReg(AArch64::X17).addOperand(GAMC));
2399 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
2400 .addReg(AuthResultReg)
2401 .addReg(AArch64::X17)
2402 .addImm(0));
2403 } else {
2404 MachineOperand GAHiOp(GAMO);
2405 MachineOperand GALoOp(GAMO);
2406 GAHiOp.addTargetFlag(AArch64II::MO_PAGE);
2407 GALoOp.addTargetFlag(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2408
2409 MCOperand GAMCHi, GAMCLo;
2410 MCInstLowering.lowerOperand(GAHiOp, GAMCHi);
2411 MCInstLowering.lowerOperand(GALoOp, GAMCLo);
2412
2413 EmitToStreamer(
2414 MCInstBuilder(AArch64::ADRP).addReg(AArch64::X17).addOperand(GAMCHi));
2415
2416 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
2417 .addReg(AArch64::X17)
2418 .addReg(AArch64::X17)
2419 .addOperand(GAMCLo)
2420 .addImm(0));
2421
2422 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
2423 .addReg(AuthResultReg)
2424 .addReg(AArch64::X17)
2425 .addImm(0));
2426 }
2427
2428 assert(GAMO.isGlobal());
2429 MCSymbol *UndefWeakSym;
2430 if (GAMO.getGlobal()->hasExternalWeakLinkage()) {
2431 UndefWeakSym = createTempSymbol("undef_weak");
2432 EmitToStreamer(
2433 MCInstBuilder(AArch64::CBZX)
2434 .addReg(AuthResultReg)
2435 .addExpr(MCSymbolRefExpr::create(UndefWeakSym, OutContext)));
2436 }
2437
2438 assert(GAMO.getGlobal()->getValueType() != nullptr);
2439 unsigned AuthOpcode = GAMO.getGlobal()->getValueType()->isFunctionTy()
2440 ? AArch64::AUTIA
2441 : AArch64::AUTDA;
2442 EmitToStreamer(MCInstBuilder(AuthOpcode)
2443 .addReg(AuthResultReg)
2444 .addReg(AuthResultReg)
2445 .addReg(AArch64::X17));
2446
2447 if (GAMO.getGlobal()->hasExternalWeakLinkage())
2448 OutStreamer->emitLabel(UndefWeakSym);
2449
2450 if (!STI->hasFPAC()) {
2451 auto AuthKey =
2452 (AuthOpcode == AArch64::AUTIA ? AArch64PACKey::IA : AArch64PACKey::DA);
2453
2454 emitPtrauthCheckAuthenticatedValue(AuthResultReg, AArch64::X17, AuthKey,
2455 AArch64PAuth::AuthCheckMethod::XPAC,
2456 /*ShouldTrap=*/true,
2457 /*OnFailure=*/nullptr);
2458
2459 emitMovXReg(DstReg, AuthResultReg);
2460 }
2461}
2462
2463const MCExpr *
2464AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
2466 const Function &Fn = *BA.getFunction();
2467
2468 if (std::optional<uint16_t> BADisc =
2469 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(Fn))
2470 return AArch64AuthMCExpr::create(BAE, *BADisc, AArch64PACKey::IA,
2471 /*HasAddressDiversity=*/false, OutContext);
2472
2473 return BAE;
2474}
2475
2476// Simple pseudo-instructions have their lowering (with expansion to real
2477// instructions) auto-generated.
2478#include "AArch64GenMCPseudoLowering.inc"
2479
2480void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
2481 S.emitInstruction(Inst, *STI);
2482#ifndef NDEBUG
2483 ++InstsEmitted;
2484#endif
2485}
2486
2487void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
2488 AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
2489
2490#ifndef NDEBUG
2491 InstsEmitted = 0;
2492 auto CheckMISize = make_scope_exit([&]() {
2493 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
2494 });
2495#endif
2496
2497 // Do any auto-generated pseudo lowerings.
2498 if (MCInst OutInst; lowerPseudoInstExpansion(MI, OutInst)) {
2499 EmitToStreamer(*OutStreamer, OutInst);
2500 return;
2501 }
2502
2503 if (MI->getOpcode() == AArch64::ADRP) {
2504 for (auto &Opd : MI->operands()) {
2505 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
2506 "swift_async_extendedFramePointerFlags") {
2507 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
2508 }
2509 }
2510 }
2511
2512 if (AArch64FI->getLOHRelated().count(MI)) {
2513 // Generate a label for LOH related instruction
2514 MCSymbol *LOHLabel = createTempSymbol("loh");
2515 // Associate the instruction with the label
2516 LOHInstToLabel[MI] = LOHLabel;
2517 OutStreamer->emitLabel(LOHLabel);
2518 }
2519
2521 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
2522 // Do any manual lowerings.
2523 switch (MI->getOpcode()) {
2524 default:
2526 "Unhandled tail call instruction");
2527 break;
2528 case AArch64::HINT: {
2529 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2530 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2531 // non-empty. If MI is the initial BTI, place the
2532 // __patchable_function_entries label after BTI.
2533 if (CurrentPatchableFunctionEntrySym &&
2534 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2535 MI == &MF->front().front()) {
2536 int64_t Imm = MI->getOperand(0).getImm();
2537 if ((Imm & 32) && (Imm & 6)) {
2538 MCInst Inst;
2539 MCInstLowering.Lower(MI, Inst);
2540 EmitToStreamer(*OutStreamer, Inst);
2541 CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
2542 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
2543 return;
2544 }
2545 }
2546 break;
2547 }
2548 case AArch64::MOVMCSym: {
2549 Register DestReg = MI->getOperand(0).getReg();
2550 const MachineOperand &MO_Sym = MI->getOperand(1);
2551 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
2552 MCOperand Hi_MCSym, Lo_MCSym;
2553
2554 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
2555 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
2556
2557 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
2558 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
2559
2560 MCInst MovZ;
2561 MovZ.setOpcode(AArch64::MOVZXi);
2562 MovZ.addOperand(MCOperand::createReg(DestReg));
2563 MovZ.addOperand(Hi_MCSym);
2565 EmitToStreamer(*OutStreamer, MovZ);
2566
2567 MCInst MovK;
2568 MovK.setOpcode(AArch64::MOVKXi);
2569 MovK.addOperand(MCOperand::createReg(DestReg));
2570 MovK.addOperand(MCOperand::createReg(DestReg));
2571 MovK.addOperand(Lo_MCSym);
2573 EmitToStreamer(*OutStreamer, MovK);
2574 return;
2575 }
2576 case AArch64::MOVIv2d_ns:
2577 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
2578 // as movi is more efficient across all cores. Newer cores can eliminate
2579 // fmovs early and there is no difference with movi, but this not true for
2580 // all implementations.
2581 //
2582 // The floating-point version doesn't quite work in rare cases on older
2583 // CPUs, so on those targets we lower this instruction to movi.16b instead.
2584 if (STI->hasZeroCycleZeroingFPWorkaround() &&
2585 MI->getOperand(1).getImm() == 0) {
2586 MCInst TmpInst;
2587 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
2588 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
2589 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
2590 EmitToStreamer(*OutStreamer, TmpInst);
2591 return;
2592 }
2593 break;
2594
2595 case AArch64::DBG_VALUE:
2596 case AArch64::DBG_VALUE_LIST:
2597 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
2598 SmallString<128> TmpStr;
2599 raw_svector_ostream OS(TmpStr);
2600 PrintDebugValueComment(MI, OS);
2601 OutStreamer->emitRawText(StringRef(OS.str()));
2602 }
2603 return;
2604
2605 case AArch64::EMITBKEY: {
2606 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
2607 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
2608 ExceptionHandlingType != ExceptionHandling::ARM)
2609 return;
2610
2611 if (getFunctionCFISectionType(*MF) == CFISection::None)
2612 return;
2613
2614 OutStreamer->emitCFIBKeyFrame();
2615 return;
2616 }
2617
2618 case AArch64::EMITMTETAGGED: {
2619 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
2620 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
2621 ExceptionHandlingType != ExceptionHandling::ARM)
2622 return;
2623
2624 if (getFunctionCFISectionType(*MF) != CFISection::None)
2625 OutStreamer->emitCFIMTETaggedFrame();
2626 return;
2627 }
2628
2629 case AArch64::AUT:
2630 case AArch64::AUTPAC:
2631 emitPtrauthAuthResign(MI);
2632 return;
2633
2634 case AArch64::LOADauthptrstatic:
2635 LowerLOADauthptrstatic(*MI);
2636 return;
2637
2638 case AArch64::LOADgotPAC:
2639 case AArch64::MOVaddrPAC:
2640 LowerMOVaddrPAC(*MI);
2641 return;
2642
2643 case AArch64::LOADgotAUTH:
2644 LowerLOADgotAUTH(*MI);
2645 return;
2646
2647 case AArch64::BRA:
2648 case AArch64::BLRA:
2649 emitPtrauthBranch(MI);
2650 return;
2651
2652 // Tail calls use pseudo instructions so they have the proper code-gen
2653 // attributes (isCall, isReturn, etc.). We lower them to the real
2654 // instruction here.
2655 case AArch64::AUTH_TCRETURN:
2656 case AArch64::AUTH_TCRETURN_BTI: {
2657 Register Callee = MI->getOperand(0).getReg();
2658 const uint64_t Key = MI->getOperand(2).getImm();
2659 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2660 "Invalid auth key for tail-call return");
2661
2662 const uint64_t Disc = MI->getOperand(3).getImm();
2663 assert(isUInt<16>(Disc) && "Integer discriminator is too wide");
2664
2665 Register AddrDisc = MI->getOperand(4).getReg();
2666
2667 Register ScratchReg = Callee == AArch64::X16 ? AArch64::X17 : AArch64::X16;
2668
2669 emitPtrauthTailCallHardening(MI);
2670
2671 // See the comments in emitPtrauthBranch.
2672 if (Callee == AddrDisc)
2673 report_fatal_error("Call target is signed with its own value");
2674 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg,
2675 /*MayUseAddrAsScratch=*/true);
2676
2677 const bool IsZero = DiscReg == AArch64::XZR;
2678 const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ},
2679 {AArch64::BRAB, AArch64::BRABZ}};
2680
2681 MCInst TmpInst;
2682 TmpInst.setOpcode(Opcodes[Key][IsZero]);
2683 TmpInst.addOperand(MCOperand::createReg(Callee));
2684 if (!IsZero)
2685 TmpInst.addOperand(MCOperand::createReg(DiscReg));
2686 EmitToStreamer(*OutStreamer, TmpInst);
2687 return;
2688 }
2689
2690 case AArch64::TCRETURNri:
2691 case AArch64::TCRETURNrix16x17:
2692 case AArch64::TCRETURNrix17:
2693 case AArch64::TCRETURNrinotx16:
2694 case AArch64::TCRETURNriALL: {
2695 emitPtrauthTailCallHardening(MI);
2696
2697 MCInst TmpInst;
2698 TmpInst.setOpcode(AArch64::BR);
2699 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
2700 EmitToStreamer(*OutStreamer, TmpInst);
2701 return;
2702 }
2703 case AArch64::TCRETURNdi: {
2704 emitPtrauthTailCallHardening(MI);
2705
2706 MCOperand Dest;
2707 MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
2708 MCInst TmpInst;
2709 TmpInst.setOpcode(AArch64::B);
2710 TmpInst.addOperand(Dest);
2711 EmitToStreamer(*OutStreamer, TmpInst);
2712 return;
2713 }
2714 case AArch64::SpeculationBarrierISBDSBEndBB: {
2715 // Print DSB SYS + ISB
2716 MCInst TmpInstDSB;
2717 TmpInstDSB.setOpcode(AArch64::DSB);
2718 TmpInstDSB.addOperand(MCOperand::createImm(0xf));
2719 EmitToStreamer(*OutStreamer, TmpInstDSB);
2720 MCInst TmpInstISB;
2721 TmpInstISB.setOpcode(AArch64::ISB);
2722 TmpInstISB.addOperand(MCOperand::createImm(0xf));
2723 EmitToStreamer(*OutStreamer, TmpInstISB);
2724 return;
2725 }
2726 case AArch64::SpeculationBarrierSBEndBB: {
2727 // Print SB
2728 MCInst TmpInstSB;
2729 TmpInstSB.setOpcode(AArch64::SB);
2730 EmitToStreamer(*OutStreamer, TmpInstSB);
2731 return;
2732 }
2733 case AArch64::TLSDESC_CALLSEQ: {
2734 /// lower this to:
2735 /// adrp x0, :tlsdesc:var
2736 /// ldr x1, [x0, #:tlsdesc_lo12:var]
2737 /// add x0, x0, #:tlsdesc_lo12:var
2738 /// .tlsdesccall var
2739 /// blr x1
2740 /// (TPIDR_EL0 offset now in x0)
2741 const MachineOperand &MO_Sym = MI->getOperand(0);
2742 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
2743 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
2744 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
2745 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
2746 MCInstLowering.lowerOperand(MO_Sym, Sym);
2747 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
2748 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
2749
2750 MCInst Adrp;
2751 Adrp.setOpcode(AArch64::ADRP);
2752 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
2753 Adrp.addOperand(SymTLSDesc);
2754 EmitToStreamer(*OutStreamer, Adrp);
2755
2756 MCInst Ldr;
2757 if (STI->isTargetILP32()) {
2758 Ldr.setOpcode(AArch64::LDRWui);
2759 Ldr.addOperand(MCOperand::createReg(AArch64::W1));
2760 } else {
2761 Ldr.setOpcode(AArch64::LDRXui);
2762 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
2763 }
2764 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
2765 Ldr.addOperand(SymTLSDescLo12);
2767 EmitToStreamer(*OutStreamer, Ldr);
2768
2769 MCInst Add;
2770 if (STI->isTargetILP32()) {
2771 Add.setOpcode(AArch64::ADDWri);
2772 Add.addOperand(MCOperand::createReg(AArch64::W0));
2773 Add.addOperand(MCOperand::createReg(AArch64::W0));
2774 } else {
2775 Add.setOpcode(AArch64::ADDXri);
2776 Add.addOperand(MCOperand::createReg(AArch64::X0));
2777 Add.addOperand(MCOperand::createReg(AArch64::X0));
2778 }
2779 Add.addOperand(SymTLSDescLo12);
2781 EmitToStreamer(*OutStreamer, Add);
2782
2783 // Emit a relocation-annotation. This expands to no code, but requests
2784 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
2785 MCInst TLSDescCall;
2786 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
2787 TLSDescCall.addOperand(Sym);
2788 EmitToStreamer(*OutStreamer, TLSDescCall);
2789#ifndef NDEBUG
2790 --InstsEmitted; // no code emitted
2791#endif
2792
2793 MCInst Blr;
2794 Blr.setOpcode(AArch64::BLR);
2795 Blr.addOperand(MCOperand::createReg(AArch64::X1));
2796 EmitToStreamer(*OutStreamer, Blr);
2797
2798 return;
2799 }
2800
2801 case AArch64::JumpTableDest32:
2802 case AArch64::JumpTableDest16:
2803 case AArch64::JumpTableDest8:
2804 LowerJumpTableDest(*OutStreamer, *MI);
2805 return;
2806
2807 case AArch64::BR_JumpTable:
2808 LowerHardenedBRJumpTable(*MI);
2809 return;
2810
2811 case AArch64::FMOVH0:
2812 case AArch64::FMOVS0:
2813 case AArch64::FMOVD0:
2814 emitFMov0(*MI);
2815 return;
2816
2817 case AArch64::MOPSMemoryCopyPseudo:
2818 case AArch64::MOPSMemoryMovePseudo:
2819 case AArch64::MOPSMemorySetPseudo:
2820 case AArch64::MOPSMemorySetTaggingPseudo:
2821 LowerMOPS(*OutStreamer, *MI);
2822 return;
2823
2824 case TargetOpcode::STACKMAP:
2825 return LowerSTACKMAP(*OutStreamer, SM, *MI);
2826
2827 case TargetOpcode::PATCHPOINT:
2828 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
2829
2830 case TargetOpcode::STATEPOINT:
2831 return LowerSTATEPOINT(*OutStreamer, SM, *MI);
2832
2833 case TargetOpcode::FAULTING_OP:
2834 return LowerFAULTING_OP(*MI);
2835
2836 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2837 LowerPATCHABLE_FUNCTION_ENTER(*MI);
2838 return;
2839
2840 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
2841 LowerPATCHABLE_FUNCTION_EXIT(*MI);
2842 return;
2843
2844 case TargetOpcode::PATCHABLE_TAIL_CALL:
2845 LowerPATCHABLE_TAIL_CALL(*MI);
2846 return;
2847 case TargetOpcode::PATCHABLE_EVENT_CALL:
2848 return LowerPATCHABLE_EVENT_CALL(*MI, false);
2849 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2850 return LowerPATCHABLE_EVENT_CALL(*MI, true);
2851
2852 case AArch64::KCFI_CHECK:
2853 LowerKCFI_CHECK(*MI);
2854 return;
2855
2856 case AArch64::HWASAN_CHECK_MEMACCESS:
2857 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
2858 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
2859 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
2860 LowerHWASAN_CHECK_MEMACCESS(*MI);
2861 return;
2862
2863 case AArch64::SEH_StackAlloc:
2864 TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
2865 return;
2866
2867 case AArch64::SEH_SaveFPLR:
2868 TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
2869 return;
2870
2871 case AArch64::SEH_SaveFPLR_X:
2872 assert(MI->getOperand(0).getImm() < 0 &&
2873 "Pre increment SEH opcode must have a negative offset");
2874 TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
2875 return;
2876
2877 case AArch64::SEH_SaveReg:
2878 TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
2879 MI->getOperand(1).getImm());
2880 return;
2881
2882 case AArch64::SEH_SaveReg_X:
2883 assert(MI->getOperand(1).getImm() < 0 &&
2884 "Pre increment SEH opcode must have a negative offset");
2885 TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
2886 -MI->getOperand(1).getImm());
2887 return;
2888
2889 case AArch64::SEH_SaveRegP:
2890 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
2891 MI->getOperand(0).getImm() <= 28) {
2892 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
2893 "Register paired with LR must be odd");
2894 TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
2895 MI->getOperand(2).getImm());
2896 return;
2897 }
2898 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2899 "Non-consecutive registers not allowed for save_regp");
2900 TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
2901 MI->getOperand(2).getImm());
2902 return;
2903
2904 case AArch64::SEH_SaveRegP_X:
2905 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2906 "Non-consecutive registers not allowed for save_regp_x");
2907 assert(MI->getOperand(2).getImm() < 0 &&
2908 "Pre increment SEH opcode must have a negative offset");
2909 TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
2910 -MI->getOperand(2).getImm());
2911 return;
2912
2913 case AArch64::SEH_SaveFReg:
2914 TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
2915 MI->getOperand(1).getImm());
2916 return;
2917
2918 case AArch64::SEH_SaveFReg_X:
2919 assert(MI->getOperand(1).getImm() < 0 &&
2920 "Pre increment SEH opcode must have a negative offset");
2921 TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
2922 -MI->getOperand(1).getImm());
2923 return;
2924
2925 case AArch64::SEH_SaveFRegP:
2926 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2927 "Non-consecutive registers not allowed for save_regp");
2928 TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
2929 MI->getOperand(2).getImm());
2930 return;
2931
2932 case AArch64::SEH_SaveFRegP_X:
2933 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2934 "Non-consecutive registers not allowed for save_regp_x");
2935 assert(MI->getOperand(2).getImm() < 0 &&
2936 "Pre increment SEH opcode must have a negative offset");
2937 TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
2938 -MI->getOperand(2).getImm());
2939 return;
2940
2941 case AArch64::SEH_SetFP:
2943 return;
2944
2945 case AArch64::SEH_AddFP:
2946 TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm());
2947 return;
2948
2949 case AArch64::SEH_Nop:
2950 TS->emitARM64WinCFINop();
2951 return;
2952
2953 case AArch64::SEH_PrologEnd:
2955 return;
2956
2957 case AArch64::SEH_EpilogStart:
2959 return;
2960
2961 case AArch64::SEH_EpilogEnd:
2963 return;
2964
2965 case AArch64::SEH_PACSignLR:
2967 return;
2968
2969 case AArch64::SEH_SaveAnyRegQP:
2970 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
2971 "Non-consecutive registers not allowed for save_any_reg");
2972 assert(MI->getOperand(2).getImm() >= 0 &&
2973 "SaveAnyRegQP SEH opcode offset must be non-negative");
2974 assert(MI->getOperand(2).getImm() <= 1008 &&
2975 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
2976 TS->emitARM64WinCFISaveAnyRegQP(MI->getOperand(0).getImm(),
2977 MI->getOperand(2).getImm());
2978 return;
2979
2980 case AArch64::SEH_SaveAnyRegQPX:
2981 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
2982 "Non-consecutive registers not allowed for save_any_reg");
2983 assert(MI->getOperand(2).getImm() < 0 &&
2984 "SaveAnyRegQPX SEH opcode offset must be negative");
2985 assert(MI->getOperand(2).getImm() >= -1008 &&
2986 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
2987 TS->emitARM64WinCFISaveAnyRegQPX(MI->getOperand(0).getImm(),
2988 -MI->getOperand(2).getImm());
2989 return;
2990 }
2991
2992 // Finally, do the automated lowerings for everything else.
2993 MCInst TmpInst;
2994 MCInstLowering.Lower(MI, TmpInst);
2995 EmitToStreamer(*OutStreamer, TmpInst);
2996}
2997
2998void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
2999 MCSymbol *LazyPointer) {
3000 // _ifunc:
3001 // adrp x16, lazy_pointer@GOTPAGE
3002 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
3003 // ldr x16, [x16]
3004 // br x16
3005
3006 {
3007 MCInst Adrp;
3008 Adrp.setOpcode(AArch64::ADRP);
3009 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
3010 MCOperand SymPage;
3011 MCInstLowering.lowerOperand(
3014 SymPage);
3015 Adrp.addOperand(SymPage);
3016 EmitToStreamer(Adrp);
3017 }
3018
3019 {
3020 MCInst Ldr;
3021 Ldr.setOpcode(AArch64::LDRXui);
3022 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
3023 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
3024 MCOperand SymPageOff;
3025 MCInstLowering.lowerOperand(
3028 SymPageOff);
3029 Ldr.addOperand(SymPageOff);
3031 EmitToStreamer(Ldr);
3032 }
3033
3034 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
3035 .addReg(AArch64::X16)
3036 .addReg(AArch64::X16)
3037 .addImm(0));
3038
3039 EmitToStreamer(MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
3040 : AArch64::BR)
3041 .addReg(AArch64::X16));
3042}
3043
3044void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
3045 const GlobalIFunc &GI,
3046 MCSymbol *LazyPointer) {
3047 // These stub helpers are only ever called once, so here we're optimizing for
3048 // minimum size by using the pre-indexed store variants, which saves a few
3049 // bytes of instructions to bump & restore sp.
3050
3051 // _ifunc.stub_helper:
3052 // stp fp, lr, [sp, #-16]!
3053 // mov fp, sp
3054 // stp x1, x0, [sp, #-16]!
3055 // stp x3, x2, [sp, #-16]!
3056 // stp x5, x4, [sp, #-16]!
3057 // stp x7, x6, [sp, #-16]!
3058 // stp d1, d0, [sp, #-16]!
3059 // stp d3, d2, [sp, #-16]!
3060 // stp d5, d4, [sp, #-16]!
3061 // stp d7, d6, [sp, #-16]!
3062 // bl _resolver
3063 // adrp x16, lazy_pointer@GOTPAGE
3064 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
3065 // str x0, [x16]
3066 // mov x16, x0
3067 // ldp d7, d6, [sp], #16
3068 // ldp d5, d4, [sp], #16
3069 // ldp d3, d2, [sp], #16
3070 // ldp d1, d0, [sp], #16
3071 // ldp x7, x6, [sp], #16
3072 // ldp x5, x4, [sp], #16
3073 // ldp x3, x2, [sp], #16
3074 // ldp x1, x0, [sp], #16
3075 // ldp fp, lr, [sp], #16
3076 // br x16
3077
3078 EmitToStreamer(MCInstBuilder(AArch64::STPXpre)
3079 .addReg(AArch64::SP)
3080 .addReg(AArch64::FP)
3081 .addReg(AArch64::LR)
3082 .addReg(AArch64::SP)
3083 .addImm(-2));
3084
3085 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
3086 .addReg(AArch64::FP)
3087 .addReg(AArch64::SP)
3088 .addImm(0)
3089 .addImm(0));
3090
3091 for (int I = 0; I != 4; ++I)
3092 EmitToStreamer(MCInstBuilder(AArch64::STPXpre)
3093 .addReg(AArch64::SP)
3094 .addReg(AArch64::X1 + 2 * I)
3095 .addReg(AArch64::X0 + 2 * I)
3096 .addReg(AArch64::SP)
3097 .addImm(-2));
3098
3099 for (int I = 0; I != 4; ++I)
3100 EmitToStreamer(MCInstBuilder(AArch64::STPDpre)
3101 .addReg(AArch64::SP)
3102 .addReg(AArch64::D1 + 2 * I)
3103 .addReg(AArch64::D0 + 2 * I)
3104 .addReg(AArch64::SP)
3105 .addImm(-2));
3106
3107 EmitToStreamer(
3108 MCInstBuilder(AArch64::BL)
3110
3111 {
3112 MCInst Adrp;
3113 Adrp.setOpcode(AArch64::ADRP);
3114 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
3115 MCOperand SymPage;
3116 MCInstLowering.lowerOperand(
3117 MachineOperand::CreateES(LazyPointer->getName().data() + 1,
3119 SymPage);
3120 Adrp.addOperand(SymPage);
3121 EmitToStreamer(Adrp);
3122 }
3123
3124 {
3125 MCInst Ldr;
3126 Ldr.setOpcode(AArch64::LDRXui);
3127 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
3128 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
3129 MCOperand SymPageOff;
3130 MCInstLowering.lowerOperand(
3131 MachineOperand::CreateES(LazyPointer->getName().data() + 1,
3133 SymPageOff);
3134 Ldr.addOperand(SymPageOff);
3136 EmitToStreamer(Ldr);
3137 }
3138
3139 EmitToStreamer(MCInstBuilder(AArch64::STRXui)
3140 .addReg(AArch64::X0)
3141 .addReg(AArch64::X16)
3142 .addImm(0));
3143
3144 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
3145 .addReg(AArch64::X16)
3146 .addReg(AArch64::X0)
3147 .addImm(0)
3148 .addImm(0));
3149
3150 for (int I = 3; I != -1; --I)
3151 EmitToStreamer(MCInstBuilder(AArch64::LDPDpost)
3152 .addReg(AArch64::SP)
3153 .addReg(AArch64::D1 + 2 * I)
3154 .addReg(AArch64::D0 + 2 * I)
3155 .addReg(AArch64::SP)
3156 .addImm(2));
3157
3158 for (int I = 3; I != -1; --I)
3159 EmitToStreamer(MCInstBuilder(AArch64::LDPXpost)
3160 .addReg(AArch64::SP)
3161 .addReg(AArch64::X1 + 2 * I)
3162 .addReg(AArch64::X0 + 2 * I)
3163 .addReg(AArch64::SP)
3164 .addImm(2));
3165
3166 EmitToStreamer(MCInstBuilder(AArch64::LDPXpost)
3167 .addReg(AArch64::SP)
3168 .addReg(AArch64::FP)
3169 .addReg(AArch64::LR)
3170 .addReg(AArch64::SP)
3171 .addImm(2));
3172
3173 EmitToStreamer(MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
3174 : AArch64::BR)
3175 .addReg(AArch64::X16));
3176}
3177
3178const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV) {
3179 if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
3180 return MCSymbolRefExpr::create(MCInstLowering.GetGlobalValueSymbol(GV, 0),
3181 OutContext);
3182 }
3183
3184 return AsmPrinter::lowerConstant(CV);
3185}
3186
3187// Force static initialization.
3194}
static cl::opt< PtrauthCheckMode > PtrauthAuthChecks("aarch64-ptrauth-auth-checks", cl::Hidden, cl::values(clEnumValN(Unchecked, "none", "don't test for failure"), clEnumValN(Poison, "poison", "poison on failure"), clEnumValN(Trap, "trap", "trap on failure")), cl::desc("Check pointer authentication auth/resign failures"), cl::init(Default))
PtrauthCheckMode
@ Poison
@ Default
@ Unchecked
static void emitAuthenticatedPointer(MCStreamer &OutStreamer, MCSymbol *StubLabel, const MCExpr *StubAuthPtrRef)
LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter()
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:686
#define LLVM_EXTERNAL_VISIBILITY
Definition: Compiler.h:128
std::string Name
uint64_t Size
Symbol * Sym
Definition: ELF_riscv.cpp:479
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition: Value.cpp:469
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
static bool printOperand(raw_ostream &OS, const SelectionDAG *G, const SDValue Value)
This file defines the SmallString class.
This file defines the SmallVector class.
static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, char Mode, raw_ostream &O)
static const AArch64AuthMCExpr * create(const MCExpr *Expr, uint16_t Discriminator, AArch64PACKey::ID Key, bool HasAddressDiversity, MCContext &Ctx)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
std::optional< std::string > getOutliningStyle() const
static const char * getRegisterName(MCRegister Reg, unsigned AltIdx=AArch64::NoRegAltName)
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
static const AArch64MCExpr * create(const MCExpr *Expr, VariantKind Kind, MCContext &Ctx)
AArch64MCInstLower - This class is used to lower an MachineInstr into an MCInst.
bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const
virtual void emitARM64WinCFISaveRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveRegPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFIAllocStack(unsigned Size)
virtual void emitARM64WinCFISaveFPLRX(int Offset)
virtual void emitDirectiveVariantPCS(MCSymbol *Symbol)
Callback used to implement the .variant_pcs directive.
virtual void emitARM64WinCFIAddFP(unsigned Size)
virtual void emitARM64WinCFISaveFPLR(int Offset)
virtual void emitARM64WinCFISaveFRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegX(unsigned Reg, int Offset)
void emitNoteSection(unsigned Flags, uint64_t PAuthABIPlatform=-1, uint64_t PAuthABIVersion=-1)
Callback used to implement the .note.gnu.property section.
virtual void emitARM64WinCFISaveReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveLRPair(unsigned Reg, int Offset)
This implementation is used for AArch64 ELF targets (Linux in particular).
AArch64_MachoTargetObjectFile - This TLOF implementation is used for Darwin.
Class for arbitrary precision integers.
Definition: APInt.h:78
Represent the analysis usage information of a pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
This class is intended to be used as a driving class for all asm writers.
Definition: AsmPrinter.h:86
virtual void emitInstruction(const MachineInstr *)
Targets should implement this to emit instructions.
Definition: AsmPrinter.h:561
void EmitToStreamer(MCStreamer &S, const MCInst &Inst)
Definition: AsmPrinter.cpp:428
virtual const MCExpr * lowerConstantPtrAuth(const ConstantPtrAuth &CPA)
Definition: AsmPrinter.h:582
void emitXRayTable()
Emit a table with all XRay instrumentation points.
virtual void emitGlobalAlias(const Module &M, const GlobalAlias &GA)
virtual MCSymbol * GetCPISymbol(unsigned CPID) const
Return the symbol for the specified constant pool entry.
virtual void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI, MCSymbol *LazyPointer)
Definition: AsmPrinter.h:613
virtual void emitJumpTableInfo()
Print assembly representations of the jump tables used by the current function to the current output ...
virtual void SetupMachineFunction(MachineFunction &MF)
This should be called when a new MachineFunction is being processed from runOnMachineFunction.
void emitFunctionBody()
This method emits the body and trailer for a function.
virtual void emitStartOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the start of their fi...
Definition: AsmPrinter.h:537
virtual void emitEndOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the end of their file...
Definition: AsmPrinter.h:541
virtual void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, MCSymbol *LazyPointer)
Definition: AsmPrinter.h:607
void getAnalysisUsage(AnalysisUsage &AU) const override
Record analysis usage.
Definition: AsmPrinter.cpp:450
virtual bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const
Definition: AsmPrinter.h:916
virtual const MCSubtargetInfo * getIFuncMCSubtargetInfo() const
getSubtargetInfo() cannot be used where this is needed because we don't have a MachineFunction when w...
Definition: AsmPrinter.h:603
bool runOnMachineFunction(MachineFunction &MF) override
Emit the specified function out to the OutStreamer.
Definition: AsmPrinter.h:387
virtual const MCExpr * lowerConstant(const Constant *CV)
Lower the specified LLVM Constant to an MCExpr.
virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant as...
virtual void emitXXStructor(const DataLayout &DL, const Constant *CV)
Targets can override this to change how global constants that are part of a C++ static/global constru...
Definition: AsmPrinter.h:578
virtual void emitFunctionBodyEnd()
Targets can override this to emit stuff after the last basic block in the function.
Definition: AsmPrinter.h:549
virtual void emitFunctionEntryLabel()
EmitFunctionEntryLabel - Emit the label that is the entrypoint for the function.
virtual std::tuple< const MCSymbol *, uint64_t, const MCSymbol *, codeview::JumpTableEntrySize > getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr, const MCSymbol *BranchLabel) const
Gets information required to create a CodeView debug symbol for a jump table.
virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant.
virtual const MCExpr * lowerBlockAddressConstant(const BlockAddress &BA)
Lower the specified BlockAddress to an MCExpr.
The address of a basic block.
Definition: Constants.h:893
Function * getFunction() const
Definition: Constants.h:923
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:157
A signed pointer, in the ptrauth sense.
Definition: Constants.h:1021
Constant * getPointer() const
The pointer that is signed in this ptrauth signed pointer.
Definition: Constants.h:1048
ConstantInt * getKey() const
The Key ID, an i32 constant.
Definition: Constants.h:1051
bool hasAddressDiscriminator() const
Whether there is any non-null address discriminator.
Definition: Constants.h:1066
ConstantInt * getDiscriminator() const
The integer discriminator, an i64 constant, or 0.
Definition: Constants.h:1054
This is an important base class in LLVM.
Definition: Constant.h:42
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
const Constant * getAliasee() const
Definition: GlobalAlias.h:86
const Constant * getResolver() const
Definition: GlobalIFunc.h:72
bool hasLocalLinkage() const
Definition: GlobalValue.h:528
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:529
Type * getValueType() const
Definition: GlobalValue.h:296
void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
static const MCBinaryExpr * createLShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:617
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:537
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:622
static const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Definition: MCExpr.cpp:222
Context object for machine code objects.
Definition: MCContext.h:83
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:34
MCInstBuilder & addReg(MCRegister Reg)
Add a new register operand.
Definition: MCInstBuilder.h:37
Instances of this class represent a single low-level machine instruction.
Definition: MCInst.h:185
void addOperand(const MCOperand Op)
Definition: MCInst.h:211
void setOpcode(unsigned Op)
Definition: MCInst.h:198
MCSection * getDataSection() const
Instances of this class represent operands of the MCInst class.
Definition: MCInst.h:37
static MCOperand createExpr(const MCExpr *Val)
Definition: MCInst.h:163
static MCOperand createReg(MCRegister Reg)
Definition: MCInst.h:135
static MCOperand createImm(int64_t Val)
Definition: MCInst.h:142
uint16_t getEncodingValue(MCRegister Reg) const
Returns the encoding for Reg.
Instances of this class represent a uniqued identifier for a section in the current translation unit.
Definition: MCSection.h:36
Streaming machine code generation interface.
Definition: MCStreamer.h:213
virtual void emitCFIBKeyFrame()
Definition: MCStreamer.cpp:248
virtual void beginCOFFSymbolDef(const MCSymbol *Symbol)
Start emitting COFF symbol definition.
virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI)
Emit the given Instruction into the current section.
virtual void emitCOFFSymbolType(int Type)
Emit the type of the symbol.
virtual bool hasRawTextSupport() const
Return true if this asm streamer supports emitting unformatted text to the .s file with EmitRawText.
Definition: MCStreamer.h:347
virtual void endCOFFSymbolDef()
Marks the end of the symbol definition.
MCContext & getContext() const
Definition: MCStreamer.h:300
virtual void AddComment(const Twine &T, bool EOL=true)
Add a textual comment.
Definition: MCStreamer.h:366
virtual void emitCFIMTETaggedFrame()
Definition: MCStreamer.cpp:255
void emitValue(const MCExpr *Value, unsigned Size, SMLoc Loc=SMLoc())
Definition: MCStreamer.cpp:179
virtual void emitLabel(MCSymbol *Symbol, SMLoc Loc=SMLoc())
Emit a label for Symbol into the current section.
Definition: MCStreamer.cpp:420
MCTargetStreamer * getTargetStreamer()
Definition: MCStreamer.h:309
void emitRawText(const Twine &String)
If this file is backed by a assembly streamer, this dumps the specified string in the output ....
virtual void emitCOFFSymbolStorageClass(int StorageClass)
Emit the storage class of the symbol.
Generic base class for all target subtargets.
Represent a reference to a symbol from inside an expression.
Definition: MCExpr.h:192
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:398
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
StringRef getName() const
getName - Get the symbol name.
Definition: MCSymbol.h:205
Metadata node.
Definition: Metadata.h:1069
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
Definition: MachineInstr.h:69
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:691
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:585
const std::vector< MachineJumpTableEntry > & getJumpTables() const
MachineModuleInfoELF - This is a MachineModuleInfoImpl implementation for ELF targets.
MachineModuleInfoMachO - This is a MachineModuleInfoImpl implementation for MachO targets.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
static MachineOperand CreateMCSymbol(MCSymbol *Sym, unsigned TargetFlags=0)
const GlobalValue * getGlobal() const
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
const BlockAddress * getBlockAddress() const
void setOffset(int64_t Offset)
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
@ MO_Immediate
Immediate operand.
@ MO_GlobalAddress
Address of a global value.
@ MO_BlockAddress
Address of a basic block.
@ MO_Register
Register operand.
@ MO_ExternalSymbol
Name of external global symbol.
int64_t getOffset() const
Return the offset from the symbol in this operand.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
MI-level patchpoint operands.
Definition: StackMaps.h:76
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
static SectionKind getMetadata()
Definition: SectionKind.h:188
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
MI-level stackmap operands.
Definition: StackMaps.h:35
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
Definition: StackMaps.h:50
void recordStatepoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a statepoint instruction.
Definition: StackMaps.cpp:562
void recordPatchPoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a patchpoint instruction.
Definition: StackMaps.cpp:541
void recordStackMap(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a stackmap instruction.
Definition: StackMaps.cpp:531
MI-level Statepoint operands.
Definition: StackMaps.h:158
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:144
virtual MCSection * getSectionForJumpTable(const Function &F, const TargetMachine &TM) const
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
MCRegister getRegister(unsigned i) const
Return the specified register in the class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isFunctionTy() const
True if this is an instance of FunctionType.
Definition: Type.h:255
LLVM Value Representation.
Definition: Value.h:74
const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr) const
Accumulate the constant offset this value has compared to a base pointer.
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an SmallVector or SmallString.
Definition: raw_ostream.h:691
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_S
MO_S - Indicates that the bits of the symbol operand represented by MO_G0 etc are signed.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
AuthCheckMethod
Variants of check performed on an authenticated pointer.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
Key
PAL metadata keys.
SymbolStorageClass
Storage class tells where and what the symbol represents.
Definition: COFF.h:217
@ IMAGE_SYM_CLASS_EXTERNAL
External symbol.
Definition: COFF.h:223
@ IMAGE_SYM_CLASS_STATIC
Static.
Definition: COFF.h:224
@ IMAGE_SYM_DTYPE_NULL
No complex type; simple scalar variable.
Definition: COFF.h:273
@ IMAGE_SYM_DTYPE_FUNCTION
A function that returns a base type.
Definition: COFF.h:275
@ SCT_COMPLEX_TYPE_SHIFT
Type is formed as (base + (derived << SCT_COMPLEX_TYPE_SHIFT))
Definition: COFF.h:279
@ AArch64_VectorCall
Used between AArch64 Advanced SIMD functions.
Definition: CallingConv.h:221
@ AArch64_SVE_VectorCall
Used between AArch64 SVE functions.
Definition: CallingConv.h:224
@ SHT_PROGBITS
Definition: ELF.h:1090
@ GNU_PROPERTY_AARCH64_FEATURE_1_BTI
Definition: ELF.h:1793
@ GNU_PROPERTY_AARCH64_FEATURE_1_PAC
Definition: ELF.h:1794
@ GNU_PROPERTY_AARCH64_FEATURE_1_GCS
Definition: ELF.h:1795
@ SHF_ALLOC
Definition: ELF.h:1188
@ SHF_GROUP
Definition: ELF.h:1210
@ SHF_EXECINSTR
Definition: ELF.h:1191
@ S_REGULAR
S_REGULAR - Regular section.
Definition: MachO.h:127
Reg
All possible values of the reg field in the ModR/M byte.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:711
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
constexpr double e
Definition: MathExtras.h:47
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Offset
Definition: DWP.cpp:480
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
static unsigned getXPACOpcodeForKey(AArch64PACKey::ID K)
Return XPAC opcode to be used for a ptrauth strip using the given key.
ExceptionHandling
Target & getTheAArch64beTarget()
Target & getTheAArch64leTarget()
Target & getTheAArch64_32Target()
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
Target & getTheARM64_32Target()
@ MCAF_SubsectionsViaSymbols
.subsections_via_symbols (MachO)
Definition: MCDirectives.h:55
static MCRegister getXRegFromWReg(MCRegister Reg)
@ Add
Sum of integers.
Target & getTheARM64Target()
static MCRegister getXRegFromXRegTuple(MCRegister RegTuple)
static unsigned getPACOpcodeForKey(AArch64PACKey::ID K, bool Zero)
Return PAC opcode to be used for a ptrauth sign using the given key, or its PAC*Z variant that doesn'...
static MCRegister getWRegFromXReg(MCRegister Reg)
static unsigned getAUTOpcodeForKey(AArch64PACKey::ID K, bool Zero)
Return AUT opcode to be used for a ptrauth auth using the given key, or its AUT*Z variant that doesn'...
@ MCSA_Weak
.weak
Definition: MCDirectives.h:45
@ MCSA_Global
.type _foo, @gnu_unique_object
Definition: MCDirectives.h:30
@ MCSA_WeakAntiDep
.weak_anti_dep (COFF)
Definition: MCDirectives.h:49
@ MCSA_ELF_TypeFunction
.type _foo, STT_FUNC # aka @function
Definition: MCDirectives.h:23
@ MCSA_Hidden
.hidden (ELF)
Definition: MCDirectives.h:33
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
RegisterAsmPrinter - Helper template for registering a target specific assembly printer,...