LLVM 24.0.0git
AArch64AsmPrinter.cpp
Go to the documentation of this file.
1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64AsmPrinter.h"
15#include "AArch64.h"
16#include "AArch64MCInstLower.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
28#include "llvm/ADT/ScopeExit.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/ADT/Twine.h"
48#include "llvm/IR/Analysis.h"
49#include "llvm/IR/DataLayout.h"
51#include "llvm/IR/Mangler.h"
52#include "llvm/IR/Module.h"
53#include "llvm/IR/PassManager.h"
54#include "llvm/MC/MCAsmInfo.h"
55#include "llvm/MC/MCContext.h"
56#include "llvm/MC/MCExpr.h"
57#include "llvm/MC/MCInst.h"
61#include "llvm/MC/MCStreamer.h"
62#include "llvm/MC/MCSymbol.h"
63#include "llvm/MC/MCValue.h"
73#include <cassert>
74#include <cstdint>
75#include <map>
76#include <memory>
77
78using namespace llvm;
80
81#define DEBUG_TYPE "AArch64AsmPrinter"
82
83// Doesn't count FPR128 ZCZ instructions which are handled
84// by TableGen pattern matching
85STATISTIC(NumZCZeroingInstrsFPR,
86 "Number of zero-cycle FPR zeroing instructions expanded from "
87 "canonical pseudo instructions");
88
89namespace {
90
91class AArch64AsmPrinter : public AsmPrinter {
92 AArch64MCInstLower MCInstLowering;
93 FaultMaps FM;
94 const AArch64Subtarget *STI;
95 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
96 bool PtrauthInitFini = false;
97 bool PtrauthInitFiniAddressDisc = false;
98#ifndef NDEBUG
99 unsigned InstsEmitted;
100 bool ExplicitWinCFIRangeOpen = false;
101 bool EmittedExplicitWinCFIRange = false;
102#endif
103 bool EnableImportCallOptimization = false;
105 SectionToImportedFunctionCalls;
106 unsigned PAuthIFuncNextUniqueID = 1;
107
108public:
109 static char ID;
110
111 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
112 : AsmPrinter(TM, std::move(Streamer), ID),
113 MCInstLowering(OutContext, *this), FM(*this) {}
114
115 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
116
117 bool hasExplicitWinCFIRanges() const override {
118 return AArch64FI->hasExplicitWinCFIRanges();
119 }
120
121 /// Wrapper for MCInstLowering.lowerOperand() for the
122 /// tblgen'erated pseudo lowering.
123 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
124 return MCInstLowering.lowerOperand(MO, MCOp);
125 }
126
127 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
128
129 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
130
131 void emitStartOfAsmFile(Module &M) override;
132 void emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
133 ArrayRef<unsigned> JumpTableIndices) override;
134 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
136 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
137 const MCSymbol *BranchLabel) const override;
138
139 void emitFunctionEntryLabel() override;
140
141 void emitXXStructor(const DataLayout &DL, const Constant *CV) override;
142
143 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
144
145 void LowerHardenedBRJumpTable(const MachineInstr &MI);
146
147 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
148
149 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
150 const MachineInstr &MI);
151 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
152 const MachineInstr &MI);
153 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
154 const MachineInstr &MI);
155 void LowerFAULTING_OP(const MachineInstr &MI);
156
157 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
158 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
159 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
160 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
161
162 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
163 HwasanMemaccessTuple;
164 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
165 void LowerKCFI_CHECK(const MachineInstr &MI);
166 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
167 void emitHwasanMemaccessSymbols(Module &M);
168
169 void emitSled(const MachineInstr &MI, SledKind Kind);
170
171 // Returns whether Reg may be used to store sensitive temporary values when
172 // expanding PtrAuth pseudos. Some OSes may take extra care to protect a
173 // small subset of GPRs on context switches - use these registers then.
174 //
175 // If there are no preferred registers, returns true for any Reg.
176 bool isPtrauthRegSafe(Register Reg) const {
177 if (STI->isX16X17Safer())
178 return Reg == AArch64::X16 || Reg == AArch64::X17;
179
180 return true;
181 }
182
183 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
184 void emitPtrauthBranch(const MachineInstr *MI);
185
186 void emitPtrauthCheckAuthenticatedValue(Register TestedReg,
187 Register ScratchReg,
190 const MCSymbol *OnFailure = nullptr);
191
192 // Check authenticated LR before tail calling.
193 void emitPtrauthTailCallHardening(const MachineInstr *TC);
194
195 struct PtrAuthSchema {
196 static PtrAuthSchema CreateImmReg(AArch64PACKey::ID Key, uint64_t IntDisc,
197 const MachineOperand &AddrDiscOp);
198 static PtrAuthSchema CreateRegReg(AArch64PACKey::ID Key, Register AddrDisc,
199 Register PCDisc);
200
202 uint64_t IntDisc;
203 Register AddrDisc;
204 bool AddrDiscIsKilled;
205 Register PCDisc;
206
207 bool addrDiscIsKilledAndNoneOf(std::initializer_list<Register> Regs) {
208 return AddrDiscIsKilled && !llvm::is_contained(Regs, AddrDisc);
209 }
210 };
211
212 // Helper for emitting AUTRELLOADPAC: increment Pointer by Addend and then by
213 // a 32-bit signed value loaded from memory. The instructions emitted are
214 //
215 // ldrsw Scratch, [Pointer, #Addend]!
216 // add Pointer, Pointer, Scratch
217 //
218 // for small Addend value, with longer sequences required for wider Addend.
219 void emitPtrauthApplyIndirectAddend(Register Pointer, Register Scratch,
220 int64_t Addend);
221
222 // Emit the sequence for AUT or AUTPAC (or their PC-blending variants).
223 // Addend is only used for AUTRELLOADPAC.
224 void emitPtrauthAuthResign(Register Pointer, Register Scratch,
225 PtrAuthSchema AuthSchema,
226 std::optional<PtrAuthSchema> SignSchema,
227 std::optional<int64_t> Addend, Value *DS);
228
229 // Emit R_AARCH64_PATCHINST, the deactivation symbol relocation. Returns true
230 // if no instruction should be emitted because the deactivation symbol is
231 // defined in the current module so this function emitted a NOP instead.
232 bool emitDeactivationSymbolRelocation(Value *DS);
233
234 // Emit the sequence for PAC.
235 void emitPtrauthSign(const MachineInstr *MI);
236
237 // Emit the sequence to compute the discriminator.
238 //
239 // The Scratch register passed to this function must be safe, as returned by
240 // isPtrauthRegSafe(ScratchReg).
241 //
242 // The returned register is either ScratchReg, AddrDisc, or XZR. Furthermore,
243 // it is guaranteed to be safe (or XZR), with the only exception of
244 // passing-through an *unmodified* unsafe AddrDisc register.
245 //
246 // If the expanded pseudo is allowed to clobber AddrDisc register, setting
247 // MayClobberAddrDisc may save one MOV instruction, provided
248 // isPtrauthRegSafe(AddrDisc) is true:
249 //
250 // mov x17, x16
251 // movk x17, #1234, lsl #48
252 // ; x16 is not used anymore
253 //
254 // can be replaced by
255 //
256 // movk x16, #1234, lsl #48
257 Register emitPtrauthDiscriminator(uint64_t Disc, Register AddrDisc,
258 Register ScratchReg,
259 bool MayClobberAddrDisc = false);
260
261 // Emit the sequence for LOADauthptrstatic
262 void LowerLOADauthptrstatic(const MachineInstr &MI);
263
264 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
265 // adrp-add followed by PAC sign)
266 void LowerMOVaddrPAC(const MachineInstr &MI);
267
268 // Emit the sequence for LOADgotAUTH (load signed pointer from signed ELF GOT
269 // and authenticate it with, if FPAC bit is not set, check+trap sequence after
270 // authenticating)
271 void LowerLOADgotAUTH(const MachineInstr &MI);
272
273 void emitAddImm(MCRegister Val, int64_t Addend, MCRegister Tmp);
274 void emitAddress(MCRegister Reg, const MCExpr *Expr, MCRegister Tmp,
275 bool DSOLocal, const MCSubtargetInfo &STI);
276
277 const MCExpr *emitPAuthRelocationAsIRelative(
278 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
279 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr);
280
281 /// tblgen'erated driver function for lowering simple MI->MC
282 /// pseudo instructions.
283 bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
284
285 // Emit Build Attributes
286 void emitAttributes(unsigned Flags, uint64_t PAuthABIPlatform,
287 uint64_t PAuthABIVersion, AArch64TargetStreamer *TS);
288
289 // Emit expansion of Compare-and-branch pseudo instructions
290 void emitCBPseudoExpansion(const MachineInstr *MI);
291
292 // Emit expansion of atomic store with hint pseudo instructions
293 void emitAtomicHintPseudoExpansion(const MachineInstr *MI);
294 void emitAtomicHintPseudoExpansionRO(const MachineInstr *MI);
295 void emitAtomicHintPseudoExpansionImm(const MachineInstr *MI);
296
297 // Emit expansion of atomic fetch with hint pseudo instructions
298 void emitAtomicFetchHintPseudoExpansion(const MachineInstr *MI);
299
300 void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
301 void EmitToStreamer(const MCInst &Inst) {
302 EmitToStreamer(*OutStreamer, Inst);
303 }
304
305 void emitInstruction(const MachineInstr *MI) override;
306
307 void emitFunctionHeaderComment() override;
308
309 void getAnalysisUsage(AnalysisUsage &AU) const override {
311 AU.setPreservesAll();
312 }
313
314 bool runOnMachineFunction(MachineFunction &MF) override {
315 if (auto *PSIW = getAnalysisIfAvailable<ProfileSummaryInfoWrapperPass>())
316 PSI = &PSIW->getPSI();
317 if (auto *SDPIW =
318 getAnalysisIfAvailable<StaticDataProfileInfoWrapperPass>())
319 SDPI = &SDPIW->getStaticDataProfileInfo();
320
321 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
322 STI = &MF.getSubtarget<AArch64Subtarget>();
323
324#ifndef NDEBUG
325 ExplicitWinCFIRangeOpen = false;
326 EmittedExplicitWinCFIRange = false;
327#endif
328 SetupMachineFunction(MF);
329
330 if (STI->isTargetCOFF()) {
331 bool Local = MF.getFunction().hasLocalLinkage();
334 int Type =
336
337 OutStreamer->beginCOFFSymbolDef(CurrentFnSym);
338 OutStreamer->emitCOFFSymbolStorageClass(Scl);
339 OutStreamer->emitCOFFSymbolType(Type);
340 OutStreamer->endCOFFSymbolDef();
341 }
342
343 // Emit the rest of the function body.
344 emitFunctionBody();
345
346 // Emit the XRay table for this function.
347 emitXRayTable();
348
349 // We didn't modify anything.
350 return false;
351 }
352
353 const MCExpr *lowerConstant(const Constant *CV,
354 const Constant *BaseCV = nullptr,
355 uint64_t Offset = 0) override;
356
357private:
358 void emitInstructionImpl(const MachineInstr *MI);
359 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
360 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
361 bool printAsmRegInClass(const MachineOperand &MO,
362 const TargetRegisterClass *RC, unsigned AltName,
363 raw_ostream &O);
364
365 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
366 const char *ExtraCode, raw_ostream &O) override;
367 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
368 const char *ExtraCode, raw_ostream &O) override;
369
370 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
371
372 void emitFunctionBodyEnd() override;
373 void emitBasicBlockEnd(const MachineBasicBlock &MBB) override;
374 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
375
376 MCSymbol *GetCPISymbol(unsigned CPID) const override;
377 void emitEndOfAsmFile(Module &M) override;
378
379 AArch64FunctionInfo *AArch64FI = nullptr;
380
381 /// Emit the LOHs contained in AArch64FI.
382 void emitLOHs();
383
384 void emitMovXReg(Register Dest, Register Src);
385 void emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift);
386 void emitMOVK(Register Dest, uint64_t Imm, unsigned Shift);
387
388 void emitAUT(AArch64PACKey::ID Key, Register Pointer, Register Disc);
389 void emitPAC(AArch64PACKey::ID Key, Register Pointer, Register Disc);
390 void emitBLRA(bool IsCall, AArch64PACKey::ID Key, Register Target,
391 Register Disc);
392
393 /// Emit instruction to set float register to zero.
394 void emitFMov0(const MachineInstr &MI);
395 void emitFMov0AsFMov(const MachineInstr &MI, Register DestReg);
396
397 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
398
399 MInstToMCSymbol LOHInstToLabel;
400
401 bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
402 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
403 }
404
405 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
406 assert(STI);
407 return STI;
408 }
409 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
410 MCSymbol *LazyPointer) override;
411 void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI,
412 MCSymbol *LazyPointer) override;
413
414 /// Checks if this instruction is part of a sequence that is eligle for import
415 /// call optimization and, if so, records it to be emitted in the import call
416 /// section.
417 void recordIfImportCall(const MachineInstr *BranchInst);
418};
419
420} // end anonymous namespace
421
422// Get boolean module flag (0 or 1), treating absent flag as having value 0.
424 Metadata *Flag = M.getModuleFlag(Name);
425 if (!Flag)
426 return false;
427
428 uint64_t Value = mdconst::extract<ConstantInt>(Flag)->getZExtValue();
429 assert((Value == 0 || Value == 1) && "Boolean flag is expected, if present");
430 return Value;
431}
432
433void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
434 const Triple &TT = M.getTargetTriple();
435
436 if (TT.isOSBinFormatCOFF()) {
437 emitCOFFFeatureSymbol(M);
438 emitCOFFReplaceableFunctionData(M);
439
440 if (M.getModuleFlag("import-call-optimization"))
441 EnableImportCallOptimization = true;
442 }
443
444 PtrauthInitFini = getOptionalBooleanModuleFlag(M, "ptrauth-init-fini");
445 PtrauthInitFiniAddressDisc = getOptionalBooleanModuleFlag(
446 M, "ptrauth-init-fini-address-discrimination");
447
448 if (!TT.isOSBinFormatELF())
449 return;
450
451 // For emitting build attributes and .note.gnu.property section
452 auto *TS =
453 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
454 // Assemble feature flags that may require creation of build attributes and a
455 // note section.
456 unsigned BAFlags = 0;
457 unsigned GNUFlags = 0;
458 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
459 M.getModuleFlag("branch-target-enforcement"))) {
460 if (!BTE->isZero()) {
461 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_BTI_Flag;
463 }
464 }
465
466 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
467 M.getModuleFlag("guarded-control-stack"))) {
468 if (!GCS->isZero()) {
469 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_GCS_Flag;
471 }
472 }
473
475 M.getModuleFlag("sign-return-address"))) {
476 if (!Sign->isZero()) {
477 BAFlags |= AArch64BuildAttributes::FeatureAndBitsFlag::Feature_PAC_Flag;
479 }
480 }
481
482 uint64_t PAuthABIPlatform = -1;
483 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
484 M.getModuleFlag("aarch64-elf-pauthabi-platform"))) {
485 PAuthABIPlatform = PAP->getZExtValue();
486 }
487
488 uint64_t PAuthABIVersion = -1;
489 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
490 M.getModuleFlag("aarch64-elf-pauthabi-version"))) {
491 PAuthABIVersion = PAV->getZExtValue();
492 }
493
494 // For LLVM_LINUX experimental platform, version value of 0 means no PAuth
495 // support. Do not emit corresponding PAuthABI GNU property note and AArch64
496 // build attributes for this case to keep Linux binaries not using PAuth
497 // unaffected.
498 if (PAuthABIPlatform == ELF::AARCH64_PAUTH_PLATFORM_LLVM_LINUX &&
499 PAuthABIVersion == 0) {
500 PAuthABIPlatform = uint64_t(-1);
501 PAuthABIVersion = uint64_t(-1);
502 }
503
504 // Emit AArch64 Build Attributes
505 emitAttributes(BAFlags, PAuthABIPlatform, PAuthABIVersion, TS);
506 // Emit a .note.gnu.property section with the flags.
507 TS->emitNoteSection(GNUFlags, PAuthABIPlatform, PAuthABIVersion);
508}
509
510void AArch64AsmPrinter::emitFunctionHeaderComment() {
511 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
512 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
513 if (OutlinerString != std::nullopt)
514 OutStreamer->getCommentOS() << ' ' << OutlinerString;
515}
516
517void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
518{
519 const Function &F = MF->getFunction();
520 if (F.hasFnAttribute("patchable-function-entry")) {
521 unsigned Num;
522 if (F.getFnAttribute("patchable-function-entry")
523 .getValueAsString()
524 .getAsInteger(10, Num))
525 return;
526 emitNops(Num);
527 return;
528 }
529
530 emitSled(MI, SledKind::FUNCTION_ENTER);
531}
532
533void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
534 emitSled(MI, SledKind::FUNCTION_EXIT);
535}
536
537void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
538 emitSled(MI, SledKind::TAIL_CALL);
539}
540
541void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
542 static const int8_t NoopsInSledCount = 7;
543 // We want to emit the following pattern:
544 //
545 // .Lxray_sled_N:
546 // ALIGN
547 // B #32
548 // ; 7 NOP instructions (28 bytes)
549 // .tmpN
550 //
551 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
552 // over the full 32 bytes (8 instructions) with the following pattern:
553 //
554 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
555 // LDR W17, #12 ; W17 := function ID
556 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
557 // BLR X16 ; call the tracing trampoline
558 // ;DATA: 32 bits of function ID
559 // ;DATA: lower 32 bits of the address of the trampoline
560 // ;DATA: higher 32 bits of the address of the trampoline
561 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
562 //
563 OutStreamer->emitCodeAlignment(Align(4), getSubtargetInfo());
564 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
565 OutStreamer->emitLabel(CurSled);
566 auto Target = OutContext.createTempSymbol();
567
568 // Emit "B #32" instruction, which jumps over the next 28 bytes.
569 // The operand has to be the number of 4-byte instructions to jump over,
570 // including the current instruction.
571 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
572
573 for (int8_t I = 0; I < NoopsInSledCount; I++)
574 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::NOP));
575
576 OutStreamer->emitLabel(Target);
577 recordSled(CurSled, MI, Kind, 2);
578}
579
580void AArch64AsmPrinter::emitAttributes(unsigned Flags,
581 uint64_t PAuthABIPlatform,
582 uint64_t PAuthABIVersion,
583 AArch64TargetStreamer *TS) {
584
585 PAuthABIPlatform = (uint64_t(-1) == PAuthABIPlatform) ? 0 : PAuthABIPlatform;
586 PAuthABIVersion = (uint64_t(-1) == PAuthABIVersion) ? 0 : PAuthABIVersion;
587
588 if (PAuthABIPlatform || PAuthABIVersion) {
592 AArch64BuildAttributes::SubsectionOptional::REQUIRED,
593 AArch64BuildAttributes::SubsectionType::ULEB128);
597 PAuthABIPlatform, "");
601 "");
602 }
603
604 unsigned BTIValue =
606 unsigned PACValue =
608 unsigned GCSValue =
610
611 if (BTIValue || PACValue || GCSValue) {
615 AArch64BuildAttributes::SubsectionOptional::OPTIONAL,
616 AArch64BuildAttributes::SubsectionType::ULEB128);
626 }
627}
628
629// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
630// (built-in functions __xray_customevent/__xray_typedevent).
631//
632// .Lxray_event_sled_N:
633// b 1f
634// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
635// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
636// bl __xray_CustomEvent or __xray_TypedEvent
637// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
638// 1:
639//
640// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
641//
642// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
643// After patching, b .+N will become a nop.
644void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
645 bool Typed) {
646 auto &O = *OutStreamer;
647 MCSymbol *CurSled = OutContext.createTempSymbol("xray_sled_", true);
648 O.emitLabel(CurSled);
649 bool MachO = MMI->getModule()->getTargetTriple().isOSBinFormatMachO();
650 auto *Sym = MCSymbolRefExpr::create(
651 OutContext.getOrCreateSymbol(
652 Twine(MachO ? "_" : "") +
653 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
654 OutContext);
655 if (Typed) {
656 O.AddComment("Begin XRay typed event");
657 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(9));
658 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
659 .addReg(AArch64::SP)
660 .addReg(AArch64::X0)
661 .addReg(AArch64::X1)
662 .addReg(AArch64::SP)
663 .addImm(-4));
664 EmitToStreamer(O, MCInstBuilder(AArch64::STRXui)
665 .addReg(AArch64::X2)
666 .addReg(AArch64::SP)
667 .addImm(2));
668 emitMovXReg(AArch64::X0, MI.getOperand(0).getReg());
669 emitMovXReg(AArch64::X1, MI.getOperand(1).getReg());
670 emitMovXReg(AArch64::X2, MI.getOperand(2).getReg());
671 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
672 EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui)
673 .addReg(AArch64::X2)
674 .addReg(AArch64::SP)
675 .addImm(2));
676 O.AddComment("End XRay typed event");
677 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
678 .addReg(AArch64::SP)
679 .addReg(AArch64::X0)
680 .addReg(AArch64::X1)
681 .addReg(AArch64::SP)
682 .addImm(4));
683
684 recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
685 } else {
686 O.AddComment("Begin XRay custom event");
687 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(6));
688 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
689 .addReg(AArch64::SP)
690 .addReg(AArch64::X0)
691 .addReg(AArch64::X1)
692 .addReg(AArch64::SP)
693 .addImm(-2));
694 emitMovXReg(AArch64::X0, MI.getOperand(0).getReg());
695 emitMovXReg(AArch64::X1, MI.getOperand(1).getReg());
696 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
697 O.AddComment("End XRay custom event");
698 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
699 .addReg(AArch64::SP)
700 .addReg(AArch64::X0)
701 .addReg(AArch64::X1)
702 .addReg(AArch64::SP)
703 .addImm(2));
704
705 recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
706 }
707}
708
709void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
710 Register AddrReg = MI.getOperand(0).getReg();
711 assert(std::next(MI.getIterator())->isCall() &&
712 "KCFI_CHECK not followed by a call instruction");
713 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
714 "KCFI_CHECK call target doesn't match call operand");
715
716 // Default to using the intra-procedure-call temporary registers for
717 // comparing the hashes.
718 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
719 if (AddrReg == AArch64::XZR) {
720 // Checking XZR makes no sense. Instead of emitting a load, zero
721 // ScratchRegs[0] and use it for the ESR AddrIndex below.
722 AddrReg = getXRegFromWReg(ScratchRegs[0]);
723 emitMovXReg(AddrReg, AArch64::XZR);
724 } else {
725 // If one of the scratch registers is used for the call target (e.g.
726 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
727 // temporary register instead (in this case, AArch64::W9) as the check
728 // is immediately followed by the call instruction.
729 for (auto &Reg : ScratchRegs) {
730 if (Reg == getWRegFromXReg(AddrReg)) {
731 Reg = AArch64::W9;
732 break;
733 }
734 }
735 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
736 "Invalid scratch registers for KCFI_CHECK");
737
738 // Adjust the offset for patchable-function-prefix. This assumes that
739 // patchable-function-prefix is the same for all functions.
740 int64_t PrefixNops =
741 MI.getMF()->getFunction().getFnAttributeAsParsedInteger(
742 "patchable-function-prefix");
743
744 // Load the target function type hash.
745 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi)
746 .addReg(ScratchRegs[0])
747 .addReg(AddrReg)
748 .addImm(-(PrefixNops * 4 + 4)));
749 }
750
751 // Load the expected type hash.
752 const int64_t Type = MI.getOperand(1).getImm();
753 emitMOVK(ScratchRegs[1], Type & 0xFFFF, 0);
754 emitMOVK(ScratchRegs[1], (Type >> 16) & 0xFFFF, 16);
755
756 // Compare the hashes and trap if there's a mismatch.
757 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs)
758 .addReg(AArch64::WZR)
759 .addReg(ScratchRegs[0])
760 .addReg(ScratchRegs[1])
761 .addImm(0));
762
763 MCSymbol *Pass = OutContext.createTempSymbol();
764 EmitToStreamer(*OutStreamer,
765 MCInstBuilder(AArch64::Bcc)
766 .addImm(AArch64CC::EQ)
767 .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
768
769 // The base ESR is 0x8000 and the register information is encoded in bits
770 // 0-9 as follows:
771 // - 0-4: n, where the register Xn contains the target address
772 // - 5-9: m, where the register Wm contains the expected type hash
773 // Where n, m are in [0, 30].
774 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
775 unsigned AddrIndex;
776 switch (AddrReg) {
777 default:
778 AddrIndex = AddrReg - AArch64::X0;
779 break;
780 case AArch64::FP:
781 AddrIndex = 29;
782 break;
783 case AArch64::LR:
784 AddrIndex = 30;
785 break;
786 }
787
788 assert(AddrIndex < 31 && TypeIndex < 31);
789
790 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
791 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR));
792 OutStreamer->emitLabel(Pass);
793}
794
795void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
796 Register Reg = MI.getOperand(0).getReg();
797
798 // The HWASan pass won't emit a CHECK_MEMACCESS intrinsic with a pointer
799 // statically known to be zero. However, conceivably, the HWASan pass may
800 // encounter a "cannot currently statically prove to be null" pointer (and is
801 // therefore unable to omit the intrinsic) that later optimization passes
802 // convert into a statically known-null pointer.
803 if (Reg == AArch64::XZR)
804 return;
805
806 bool IsShort =
807 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
808 (MI.getOpcode() ==
809 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
810 uint32_t AccessInfo = MI.getOperand(1).getImm();
811 bool IsFixedShadow =
812 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
813 (MI.getOpcode() ==
814 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
815 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(2).getImm() : 0;
816
817 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
818 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
819 if (!Sym) {
820 // FIXME: Make this work on non-ELF.
821 if (!TM.getTargetTriple().isOSBinFormatELF())
822 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
823
824 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
825 utostr(AccessInfo);
826 if (IsFixedShadow)
827 SymName += "_fixed_" + utostr(FixedShadowOffset);
828 if (IsShort)
829 SymName += "_short_v2";
830 Sym = OutContext.getOrCreateSymbol(SymName);
831 }
832
833 EmitToStreamer(*OutStreamer,
834 MCInstBuilder(AArch64::BL)
835 .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
836}
837
838void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
839 if (HwasanMemaccessSymbols.empty())
840 return;
841
842 const Triple &TT = M.getTargetTriple();
843 assert(TT.isOSBinFormatELF());
844 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
845 // space.
846 auto STI = std::make_unique<AArch64Subtarget>(
847 TT, TM.getTargetCPU(), TM.getTargetCPU(), TM.getTargetFeatureString(), TM,
848 true);
849 this->STI = STI.get();
850
851 MCSymbol *HwasanTagMismatchV1Sym =
852 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
853 MCSymbol *HwasanTagMismatchV2Sym =
854 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
855
856 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
857 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
858 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
859 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
860
861 for (auto &P : HwasanMemaccessSymbols) {
862 unsigned Reg = std::get<0>(P.first);
863 bool IsShort = std::get<1>(P.first);
864 uint32_t AccessInfo = std::get<2>(P.first);
865 bool IsFixedShadow = std::get<3>(P.first);
866 uint64_t FixedShadowOffset = std::get<4>(P.first);
867 const MCSymbolRefExpr *HwasanTagMismatchRef =
868 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
869 MCSymbol *Sym = P.second;
870
871 bool HasMatchAllTag =
872 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
873 uint8_t MatchAllTag =
874 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
875 unsigned Size =
876 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
877 bool CompileKernel =
878 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
879
880 OutStreamer->switchSection(OutContext.getELFSection(
881 ".text.hot", ELF::SHT_PROGBITS,
883 /*IsComdat=*/true));
884
885 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
886 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
887 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
888 OutStreamer->emitLabel(Sym);
889
890 EmitToStreamer(MCInstBuilder(AArch64::SBFMXri)
891 .addReg(AArch64::X16)
892 .addReg(Reg)
893 .addImm(4)
894 .addImm(55));
895
896 if (IsFixedShadow) {
897 // Aarch64 makes it difficult to embed large constants in the code.
898 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
899 // left-shift option in the MOV instruction. Combined with the 16-bit
900 // immediate, this is enough to represent any offset up to 2**48.
901 emitMOVZ(AArch64::X17, FixedShadowOffset >> 32, 32);
902 EmitToStreamer(MCInstBuilder(AArch64::LDRBBroX)
903 .addReg(AArch64::W16)
904 .addReg(AArch64::X17)
905 .addReg(AArch64::X16)
906 .addImm(0)
907 .addImm(0));
908 } else {
909 EmitToStreamer(MCInstBuilder(AArch64::LDRBBroX)
910 .addReg(AArch64::W16)
911 .addReg(IsShort ? AArch64::X20 : AArch64::X9)
912 .addReg(AArch64::X16)
913 .addImm(0)
914 .addImm(0));
915 }
916
917 EmitToStreamer(MCInstBuilder(AArch64::SUBSXrs)
918 .addReg(AArch64::XZR)
919 .addReg(AArch64::X16)
920 .addReg(Reg)
922 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
923 EmitToStreamer(MCInstBuilder(AArch64::Bcc)
924 .addImm(AArch64CC::NE)
926 HandleMismatchOrPartialSym, OutContext)));
927 MCSymbol *ReturnSym = OutContext.createTempSymbol();
928 OutStreamer->emitLabel(ReturnSym);
929 EmitToStreamer(MCInstBuilder(AArch64::RET).addReg(AArch64::LR));
930 OutStreamer->emitLabel(HandleMismatchOrPartialSym);
931
932 if (HasMatchAllTag) {
933 EmitToStreamer(MCInstBuilder(AArch64::UBFMXri)
934 .addReg(AArch64::X17)
935 .addReg(Reg)
936 .addImm(56)
937 .addImm(63));
938 EmitToStreamer(MCInstBuilder(AArch64::SUBSXri)
939 .addReg(AArch64::XZR)
940 .addReg(AArch64::X17)
941 .addImm(MatchAllTag)
942 .addImm(0));
943 EmitToStreamer(
944 MCInstBuilder(AArch64::Bcc)
945 .addImm(AArch64CC::EQ)
946 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)));
947 }
948
949 if (IsShort) {
950 EmitToStreamer(MCInstBuilder(AArch64::SUBSWri)
951 .addReg(AArch64::WZR)
952 .addReg(AArch64::W16)
953 .addImm(15)
954 .addImm(0));
955 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
956 EmitToStreamer(
957 MCInstBuilder(AArch64::Bcc)
958 .addImm(AArch64CC::HI)
959 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)));
960
961 EmitToStreamer(MCInstBuilder(AArch64::ANDXri)
962 .addReg(AArch64::X17)
963 .addReg(Reg)
964 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)));
965 if (Size != 1)
966 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
967 .addReg(AArch64::X17)
968 .addReg(AArch64::X17)
969 .addImm(Size - 1)
970 .addImm(0));
971 EmitToStreamer(MCInstBuilder(AArch64::SUBSWrs)
972 .addReg(AArch64::WZR)
973 .addReg(AArch64::W16)
974 .addReg(AArch64::W17)
975 .addImm(0));
976 EmitToStreamer(
977 MCInstBuilder(AArch64::Bcc)
978 .addImm(AArch64CC::LS)
979 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)));
980
981 EmitToStreamer(MCInstBuilder(AArch64::ORRXri)
982 .addReg(AArch64::X16)
983 .addReg(Reg)
984 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)));
985 EmitToStreamer(MCInstBuilder(AArch64::LDRBBui)
986 .addReg(AArch64::W16)
987 .addReg(AArch64::X16)
988 .addImm(0));
989 EmitToStreamer(
990 MCInstBuilder(AArch64::SUBSXrs)
991 .addReg(AArch64::XZR)
992 .addReg(AArch64::X16)
993 .addReg(Reg)
995 EmitToStreamer(
996 MCInstBuilder(AArch64::Bcc)
997 .addImm(AArch64CC::EQ)
998 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)));
999
1000 OutStreamer->emitLabel(HandleMismatchSym);
1001 }
1002
1003 EmitToStreamer(MCInstBuilder(AArch64::STPXpre)
1004 .addReg(AArch64::SP)
1005 .addReg(AArch64::X0)
1006 .addReg(AArch64::X1)
1007 .addReg(AArch64::SP)
1008 .addImm(-32));
1009 EmitToStreamer(MCInstBuilder(AArch64::STPXi)
1010 .addReg(AArch64::FP)
1011 .addReg(AArch64::LR)
1012 .addReg(AArch64::SP)
1013 .addImm(29));
1014
1015 if (Reg != AArch64::X0)
1016 emitMovXReg(AArch64::X0, Reg);
1017 emitMOVZ(AArch64::X1, AccessInfo & HWASanAccessInfo::RuntimeMask, 0);
1018
1019 if (CompileKernel) {
1020 // The Linux kernel's dynamic loader doesn't support GOT relative
1021 // relocations, but it doesn't support late binding either, so just call
1022 // the function directly.
1023 EmitToStreamer(MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef));
1024 } else {
1025 // Intentionally load the GOT entry and branch to it, rather than possibly
1026 // late binding the function, which may clobber the registers before we
1027 // have a chance to save them.
1028 EmitToStreamer(MCInstBuilder(AArch64::ADRP)
1029 .addReg(AArch64::X16)
1030 .addExpr(MCSpecifierExpr::create(HwasanTagMismatchRef,
1032 OutContext)));
1033 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
1034 .addReg(AArch64::X16)
1035 .addReg(AArch64::X16)
1036 .addExpr(MCSpecifierExpr::create(HwasanTagMismatchRef,
1038 OutContext)));
1039 EmitToStreamer(MCInstBuilder(AArch64::BR).addReg(AArch64::X16));
1040 }
1041 }
1042 this->STI = nullptr;
1043}
1044
1045static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
1046 MCSymbol *StubLabel,
1047 const MCExpr *StubAuthPtrRef) {
1048 // sym$auth_ptr$key$disc:
1049 OutStreamer.emitLabel(StubLabel);
1050 OutStreamer.emitValue(StubAuthPtrRef, /*size=*/8);
1051}
1052
1053void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
1054 emitHwasanMemaccessSymbols(M);
1055
1056 const Triple &TT = M.getTargetTriple();
1057 if (TT.isOSBinFormatMachO()) {
1058 // Output authenticated pointers as indirect symbols, if we have any.
1059 MachineModuleInfoMachO &MMIMacho =
1060 MMI->getObjFileInfo<MachineModuleInfoMachO>();
1061
1062 auto Stubs = MMIMacho.getAuthGVStubList();
1063
1064 if (!Stubs.empty()) {
1065 // Switch to the "__auth_ptr" section.
1066 OutStreamer->switchSection(
1067 OutContext.getMachOSection("__DATA", "__auth_ptr", MachO::S_REGULAR,
1069 emitAlignment(Align(8));
1070
1071 for (const auto &Stub : Stubs)
1072 emitAuthenticatedPointer(*OutStreamer, Stub.first, Stub.second);
1073
1074 OutStreamer->addBlankLine();
1075 }
1076
1077 // Funny Darwin hack: This flag tells the linker that no global symbols
1078 // contain code that falls through to other global symbols (e.g. the obvious
1079 // implementation of multiple entry points). If this doesn't occur, the
1080 // linker can safely perform dead code stripping. Since LLVM never
1081 // generates code that does this, it is always safe to set.
1082 OutStreamer->emitSubsectionsViaSymbols();
1083 }
1084
1085 if (TT.isOSBinFormatELF()) {
1086 // Output authenticated pointers as indirect symbols, if we have any.
1087 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
1088
1089 auto Stubs = MMIELF.getAuthGVStubList();
1090
1091 if (!Stubs.empty()) {
1092 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1093 OutStreamer->switchSection(TLOF.getDataSection());
1094 emitAlignment(Align(8));
1095
1096 for (const auto &Stub : Stubs)
1097 emitAuthenticatedPointer(*OutStreamer, Stub.first, Stub.second);
1098
1099 OutStreamer->addBlankLine();
1100 }
1101
1102 // With signed ELF GOT enabled, the linker looks at the symbol type to
1103 // choose between keys IA (for STT_FUNC) and DA (for other types). Symbols
1104 // for functions not defined in the module have STT_NOTYPE type by default.
1105 // This makes linker to emit signing schema with DA key (instead of IA) for
1106 // corresponding R_AARCH64_AUTH_GLOB_DAT dynamic reloc. To avoid that, force
1107 // all function symbols used in the module to have STT_FUNC type. See
1108 // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#default-signing-schema
1109 const auto *PtrAuthELFGOTFlag = mdconst::extract_or_null<ConstantInt>(
1110 M.getModuleFlag("ptrauth-elf-got"));
1111 if (PtrAuthELFGOTFlag && PtrAuthELFGOTFlag->getZExtValue() == 1)
1112 for (const GlobalValue &GV : M.global_values())
1113 if (!GV.use_empty() && isa<Function>(GV) &&
1114 !GV.getName().starts_with("llvm."))
1115 OutStreamer->emitSymbolAttribute(getSymbol(&GV),
1117 }
1118
1119 // Emit stack and fault map information.
1121
1122 // If import call optimization is enabled, emit the appropriate section.
1123 // We do this whether or not we recorded any import calls.
1124 if (EnableImportCallOptimization && TT.isOSBinFormatCOFF()) {
1125 OutStreamer->switchSection(getObjFileLowering().getImportCallSection());
1126
1127 // Section always starts with some magic.
1128 constexpr char ImpCallMagic[12] = "Imp_Call_V1";
1129 OutStreamer->emitBytes(StringRef{ImpCallMagic, sizeof(ImpCallMagic)});
1130
1131 // Layout of this section is:
1132 // Per section that contains calls to imported functions:
1133 // uint32_t SectionSize: Size in bytes for information in this section.
1134 // uint32_t Section Number
1135 // Per call to imported function in section:
1136 // uint32_t Kind: the kind of imported function.
1137 // uint32_t BranchOffset: the offset of the branch instruction in its
1138 // parent section.
1139 // uint32_t TargetSymbolId: the symbol id of the called function.
1140 for (auto &[Section, CallsToImportedFuncs] :
1141 SectionToImportedFunctionCalls) {
1142 unsigned SectionSize =
1143 sizeof(uint32_t) * (2 + 3 * CallsToImportedFuncs.size());
1144 OutStreamer->emitInt32(SectionSize);
1145 OutStreamer->emitCOFFSecNumber(Section->getBeginSymbol());
1146 for (auto &[CallsiteSymbol, CalledSymbol] : CallsToImportedFuncs) {
1147 // Kind is always IMAGE_REL_ARM64_DYNAMIC_IMPORT_CALL (0x13).
1148 OutStreamer->emitInt32(0x13);
1149 OutStreamer->emitCOFFSecOffset(CallsiteSymbol);
1150 OutStreamer->emitCOFFSymbolIndex(CalledSymbol);
1151 }
1152 }
1153 }
1154}
1155
1156void AArch64AsmPrinter::emitLOHs() {
1158
1159 for (const auto &D : AArch64FI->getLOHContainer()) {
1160 for (const MachineInstr *MI : D.getArgs()) {
1161 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
1162 assert(LabelIt != LOHInstToLabel.end() &&
1163 "Label hasn't been inserted for LOH related instruction");
1164 MCArgs.push_back(LabelIt->second);
1165 }
1166 OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
1167 MCArgs.clear();
1168 }
1169}
1170
1171// Unwinding can get confused if the last instruction in an exception-handling
1172// region (function, funclet, try block, etc.) is a call, because the return
1173// address then points to the *next* region (see
1174// X86AsmPrinter::maybeEmitNopAfterCallForWindowsEH for the X86 equivalent).
1175// Trap after a noreturn call that would end a function or funclet, and elide
1176// the trap when the next instruction is in the same function.
1177void AArch64AsmPrinter::emitBasicBlockEnd(const MachineBasicBlock &MBB) {
1178 if (MBB.succ_empty() && MF->usesWindowsUnwindInfo()) {
1179 auto Next = std::next(MBB.getIterator());
1180 if (Next == MF->end() || Next->isEHFuncletEntry()) {
1181 for (const MachineInstr &MI : reverse(MBB)) {
1182 if (MI.isMetaInstruction() || AArch64InstrInfo::isSEHInstruction(MI))
1183 continue;
1184 if (MI.isCall() && !MI.isReturn())
1185 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(1));
1186 break;
1187 }
1188 }
1189 }
1190
1192}
1193
1194void AArch64AsmPrinter::emitFunctionBodyEnd() {
1195#ifndef NDEBUG
1196 assert(!ExplicitWinCFIRangeOpen &&
1197 "Explicit Windows CFI range is still open at function end");
1198 assert(EmittedExplicitWinCFIRange == AArch64FI->hasExplicitWinCFIRanges() &&
1199 "Explicit Windows CFI ranges should only be used for flagged "
1200 "functions. If a function is flagged then it must have explicit "
1201 "Windows CFI ranges.");
1202#endif
1203 if (!AArch64FI->getLOHRelated().empty())
1204 emitLOHs();
1205}
1206
1207/// GetCPISymbol - Return the symbol for the specified constant pool entry.
1208MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
1209 // Darwin uses a linker-private symbol name for constant-pools (to
1210 // avoid addends on the relocation?), ELF has no such concept and
1211 // uses a normal private symbol.
1212 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
1213 return OutContext.getOrCreateSymbol(
1214 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
1215 Twine(getFunctionNumber()) + "_" + Twine(CPID));
1216
1217 return AsmPrinter::GetCPISymbol(CPID);
1218}
1219
1220void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
1221 raw_ostream &O) {
1222 const MachineOperand &MO = MI->getOperand(OpNum);
1223 switch (MO.getType()) {
1224 default:
1225 llvm_unreachable("<unknown operand type>");
1227 Register Reg = MO.getReg();
1229 assert(!MO.getSubReg() && "Subregs should be eliminated!");
1231 break;
1232 }
1234 O << MO.getImm();
1235 break;
1236 }
1238 PrintSymbolOperand(MO, O);
1239 break;
1240 }
1242 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
1243 Sym->print(O, MAI);
1244 break;
1245 }
1246 }
1247}
1248
1249bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
1250 raw_ostream &O) {
1251 Register Reg = MO.getReg();
1252 switch (Mode) {
1253 default:
1254 return true; // Unknown mode.
1255 case 'w':
1257 break;
1258 case 'x':
1260 break;
1261 case 't':
1263 break;
1264 }
1265
1267 return false;
1268}
1269
1270// Prints the register in MO using class RC using the offset in the
1271// new register class. This should not be used for cross class
1272// printing.
1273bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1274 const TargetRegisterClass *RC,
1275 unsigned AltName, raw_ostream &O) {
1276 assert(MO.isReg() && "Should only get here with a register!");
1277 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1278 Register Reg = MO.getReg();
1279 MCRegister RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
1280 if (!RI->regsOverlap(RegToPrint, Reg))
1281 return true;
1282 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
1283 return false;
1284}
1285
1286bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1287 const char *ExtraCode, raw_ostream &O) {
1288 const MachineOperand &MO = MI->getOperand(OpNum);
1289
1290 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1291 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
1292 return false;
1293
1294 // Does this asm operand have a single letter operand modifier?
1295 if (ExtraCode && ExtraCode[0]) {
1296 if (ExtraCode[1] != 0)
1297 return true; // Unknown modifier.
1298
1299 switch (ExtraCode[0]) {
1300 default:
1301 return true; // Unknown modifier.
1302 case 'w': // Print W register
1303 case 'x': // Print X register
1304 if (MO.isReg())
1305 return printAsmMRegister(MO, ExtraCode[0], O);
1306 if (MO.isImm() && MO.getImm() == 0) {
1307 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1309 return false;
1310 }
1311 printOperand(MI, OpNum, O);
1312 return false;
1313 case 'b': // Print B register.
1314 case 'h': // Print H register.
1315 case 's': // Print S register.
1316 case 'd': // Print D register.
1317 case 'q': // Print Q register.
1318 case 'z': // Print Z register.
1319 if (MO.isReg()) {
1320 const TargetRegisterClass *RC;
1321 switch (ExtraCode[0]) {
1322 case 'b':
1323 RC = &AArch64::FPR8RegClass;
1324 break;
1325 case 'h':
1326 RC = &AArch64::FPR16RegClass;
1327 break;
1328 case 's':
1329 RC = &AArch64::FPR32RegClass;
1330 break;
1331 case 'd':
1332 RC = &AArch64::FPR64RegClass;
1333 break;
1334 case 'q':
1335 RC = &AArch64::FPR128RegClass;
1336 break;
1337 case 'z':
1338 RC = &AArch64::ZPRRegClass;
1339 break;
1340 default:
1341 return true;
1342 }
1343 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
1344 }
1345 printOperand(MI, OpNum, O);
1346 return false;
1347 }
1348 }
1349
1350 // According to ARM, we should emit x and v registers unless we have a
1351 // modifier.
1352 if (MO.isReg()) {
1353 Register Reg = MO.getReg();
1354
1355 // If this is a w or x register, print an x register.
1356 if (AArch64::GPR32allRegClass.contains(Reg) ||
1357 AArch64::GPR64allRegClass.contains(Reg))
1358 return printAsmMRegister(MO, 'x', O);
1359
1360 // If this is an x register tuple, print an x register.
1361 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1362 return printAsmMRegister(MO, 't', O);
1363
1364 unsigned AltName = AArch64::NoRegAltName;
1365 const TargetRegisterClass *RegClass;
1366 if (AArch64::ZPRRegClass.contains(Reg)) {
1367 RegClass = &AArch64::ZPRRegClass;
1368 } else if (AArch64::PPRRegClass.contains(Reg)) {
1369 RegClass = &AArch64::PPRRegClass;
1370 } else if (AArch64::PNRRegClass.contains(Reg)) {
1371 RegClass = &AArch64::PNRRegClass;
1372 } else {
1373 RegClass = &AArch64::FPR128RegClass;
1374 AltName = AArch64::vreg;
1375 }
1376
1377 // If this is a b, h, s, d, or q register, print it as a v register.
1378 return printAsmRegInClass(MO, RegClass, AltName, O);
1379 }
1380
1381 printOperand(MI, OpNum, O);
1382 return false;
1383}
1384
1385bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1386 unsigned OpNum,
1387 const char *ExtraCode,
1388 raw_ostream &O) {
1389 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1390 return true; // Unknown modifier.
1391
1392 const MachineOperand &MO = MI->getOperand(OpNum);
1393 assert(MO.isReg() && "unexpected inline asm memory operand");
1394 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
1395 return false;
1396}
1397
1398void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1399 raw_ostream &OS) {
1400 unsigned NOps = MI->getNumOperands();
1401 assert(NOps == 4);
1402 OS << '\t' << MAI.getCommentString() << "DEBUG_VALUE: ";
1403 // cast away const; DIetc do not take const operands for some reason.
1404 OS << MI->getDebugVariable()->getName();
1405 OS << " <- ";
1406 // Frame address. Currently handles register +- offset only.
1407 assert(MI->isIndirectDebugValue());
1408 OS << '[';
1409 for (unsigned I = 0, E = llvm::size(MI->debug_operands()); I < E; ++I) {
1410 if (I != 0)
1411 OS << ", ";
1412 printOperand(MI, I, OS);
1413 }
1414 OS << ']';
1415 OS << "+";
1416 printOperand(MI, NOps - 2, OS);
1417}
1418
1419void AArch64AsmPrinter::emitJumpTableImpl(const MachineJumpTableInfo &MJTI,
1420 ArrayRef<unsigned> JumpTableIndices) {
1421 // Fast return if there is nothing to emit to avoid creating empty sections.
1422 if (JumpTableIndices.empty())
1423 return;
1424 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1425 const auto &F = MF->getFunction();
1427
1428 MCSection *ReadOnlySec = nullptr;
1429 if (TM.Options.EnableStaticDataPartitioning) {
1430 ReadOnlySec =
1431 TLOF.getSectionForJumpTable(F, TM, &JT[JumpTableIndices.front()]);
1432 } else {
1433 ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
1434 }
1435 OutStreamer->switchSection(ReadOnlySec);
1436
1437 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1438 for (unsigned JTI : JumpTableIndices) {
1439 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1440
1441 // If this jump table was deleted, ignore it.
1442 if (JTBBs.empty()) continue;
1443
1444 unsigned Size = AFI->getJumpTableEntrySize(JTI);
1445 emitAlignment(Align(Size));
1446 OutStreamer->emitLabel(GetJTISymbol(JTI));
1447
1448 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1449 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
1450
1451 for (auto *JTBB : JTBBs) {
1452 const MCExpr *Value =
1453 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
1454
1455 // Each entry is:
1456 // .byte/.hword (LBB - Lbase)>>2
1457 // or plain:
1458 // .word LBB - Lbase
1459 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
1460 if (Size != 4)
1462 Value, MCConstantExpr::create(2, OutContext), OutContext);
1463
1464 OutStreamer->emitValue(Value, Size);
1465 }
1466 }
1467}
1468
1469std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1471AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1472 const MachineInstr *BranchInstr,
1473 const MCSymbol *BranchLabel) const {
1474 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1475 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1477 switch (AFI->getJumpTableEntrySize(JTI)) {
1478 case 1:
1479 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1480 break;
1481 case 2:
1482 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1483 break;
1484 case 4:
1485 EntrySize = codeview::JumpTableEntrySize::Int32;
1486 break;
1487 default:
1488 llvm_unreachable("Unexpected jump table entry size");
1489 }
1490 return std::make_tuple(Base, 0, BranchLabel, EntrySize);
1491}
1492
1493void AArch64AsmPrinter::emitFunctionEntryLabel() {
1494 const Triple &TT = TM.getTargetTriple();
1495 if (TT.isOSBinFormatELF() &&
1496 (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1497 MF->getFunction().getCallingConv() ==
1498 CallingConv::AArch64_SVE_VectorCall ||
1499 MF->getInfo<AArch64FunctionInfo>()->isSVECC())) {
1500 auto *TS =
1501 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1502 TS->emitDirectiveVariantPCS(CurrentFnSym);
1503 }
1504
1506
1507 if (TT.isWindowsArm64EC() && !MF->getFunction().hasLocalLinkage()) {
1508 // For ARM64EC targets, a function definition's name is mangled differently
1509 // from the normal symbol, emit required aliases here.
1510 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1511 OutStreamer->emitSymbolAttribute(Src, MCSA_WeakAntiDep);
1512 OutStreamer->emitAssignment(
1513 Src, MCSymbolRefExpr::create(Dst, MMI->getContext()));
1514 };
1515
1516 auto getSymbolFromMetadata = [&](StringRef Name) {
1517 MCSymbol *Sym = nullptr;
1518 if (MDNode *Node = MF->getFunction().getMetadata(Name)) {
1519 StringRef NameStr = cast<MDString>(Node->getOperand(0))->getString();
1520 Sym = MMI->getContext().getOrCreateSymbol(NameStr);
1521 }
1522 return Sym;
1523 };
1524
1525 SmallVector<MDNode *> UnmangledNames;
1526 MF->getFunction().getMetadata("arm64ec_unmangled_name", UnmangledNames);
1527 for (MDNode *Node : UnmangledNames) {
1528 StringRef NameStr = cast<MDString>(Node->getOperand(0))->getString();
1529 MCSymbol *UnmangledSym = MMI->getContext().getOrCreateSymbol(NameStr);
1530 if (std::optional<std::string> MangledName =
1531 getArm64ECMangledFunctionName(UnmangledSym->getName())) {
1532 MCSymbol *ECMangledSym =
1533 MMI->getContext().getOrCreateSymbol(*MangledName);
1534 emitFunctionAlias(UnmangledSym, ECMangledSym);
1535 }
1536 }
1537 if (MCSymbol *ECMangledSym =
1538 getSymbolFromMetadata("arm64ec_ecmangled_name"))
1539 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1540 }
1541}
1542
1543void AArch64AsmPrinter::emitXXStructor(const DataLayout &DL,
1544 const Constant *CV) {
1545 LLVMContext &C = CV->getContext();
1547 "ctors/dtors are to be signed by asm printer");
1548
1549 if (PtrauthInitFini) {
1550 IntegerType *Int32Ty = IntegerType::get(C, 32);
1551 IntegerType *Int64Ty = IntegerType::get(C, 64);
1552 PointerType *PtrTy = PointerType::get(C, 0);
1553
1554 ConstantInt *Key = ConstantInt::get(Int32Ty, AArch64PAuth::InitFiniKey);
1555 ConstantInt *IntDisc = ConstantInt::get(
1558 Constant *AddressDisc = Null;
1559 if (PtrauthInitFiniAddressDisc) {
1561 AddressDisc =
1562 ConstantExpr::getIntToPtr(ConstantInt::get(Int64Ty, Marker), PtrTy);
1563 }
1564
1565 CV = ConstantPtrAuth::get(const_cast<Constant *>(CV), Key, IntDisc,
1566 AddressDisc, /*DeactivationSymbol=*/Null);
1567 }
1568
1569 // Signed pointers will be lowered by AArch64AsmPrinter::lowerConstantPtrAuth.
1571}
1572
1573void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1574 const GlobalAlias &GA) {
1575 if (auto F = dyn_cast_or_null<Function>(GA.getAliasee())) {
1576 // Global aliases must point to a definition, but unmangled patchable
1577 // symbols are special and need to point to an undefined symbol with "EXP+"
1578 // prefix. Such undefined symbol is resolved by the linker by creating
1579 // x86 thunk that jumps back to the actual EC target.
1580 if (MDNode *Node = F->getMetadata("arm64ec_exp_name")) {
1581 StringRef ExpStr = cast<MDString>(Node->getOperand(0))->getString();
1582 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(ExpStr);
1583 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(GA.getName());
1584
1585 OutStreamer->beginCOFFSymbolDef(ExpSym);
1586 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_EXTERNAL);
1587 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION
1589 OutStreamer->endCOFFSymbolDef();
1590
1591 OutStreamer->beginCOFFSymbolDef(Sym);
1592 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_EXTERNAL);
1593 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION
1595 OutStreamer->endCOFFSymbolDef();
1596 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
1597 OutStreamer->emitAssignment(
1598 Sym, MCSymbolRefExpr::create(ExpSym, MMI->getContext()));
1599 return;
1600 }
1601 }
1603}
1604
1605/// Small jump tables contain an unsigned byte or half, representing the offset
1606/// from the lowest-addressed possible destination to the desired basic
1607/// block. Since all instructions are 4-byte aligned, this is further compressed
1608/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1609/// materialize the correct destination we need:
1610///
1611/// adr xDest, .LBB0_0
1612/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1613/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1614void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1615 const llvm::MachineInstr &MI) {
1616 Register DestReg = MI.getOperand(0).getReg();
1617 Register ScratchReg = MI.getOperand(1).getReg();
1618 Register ScratchRegW =
1619 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
1620 Register TableReg = MI.getOperand(2).getReg();
1621 Register EntryReg = MI.getOperand(3).getReg();
1622 int JTIdx = MI.getOperand(4).getIndex();
1623 int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
1624
1625 // This has to be first because the compression pass based its reachability
1626 // calculations on the start of the JumpTableDest instruction.
1627 auto Label =
1628 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
1629
1630 // If we don't already have a symbol to use as the base, use the ADR
1631 // instruction itself.
1632 if (!Label) {
1634 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
1635 OutStreamer.emitLabel(Label);
1636 }
1637
1638 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
1639 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
1640 .addReg(DestReg)
1641 .addExpr(LabelExpr));
1642
1643 // Load the number of instruction-steps to offset from the label.
1644 unsigned LdrOpcode;
1645 switch (Size) {
1646 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1647 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1648 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1649 default:
1650 llvm_unreachable("Unknown jump table size");
1651 }
1652
1653 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
1654 .addReg(Size == 4 ? ScratchReg : ScratchRegW)
1655 .addReg(TableReg)
1656 .addReg(EntryReg)
1657 .addImm(0)
1658 .addImm(Size == 1 ? 0 : 1));
1659
1660 // Add to the already materialized base label address, multiplying by 4 if
1661 // compressed.
1662 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1663 .addReg(DestReg)
1664 .addReg(DestReg)
1665 .addReg(ScratchReg)
1666 .addImm(Size == 4 ? 0 : 2));
1667}
1668
1669void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1670 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1671 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1672
1673 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1674 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1675
1676 // Emit:
1677 // mov x17, #<size of table> ; depending on table size, with MOVKs
1678 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1679 // csel x16, x16, xzr, ls ; check for index overflow
1680 //
1681 // adrp x17, Ltable@PAGE ; materialize table address
1682 // add x17, Ltable@PAGEOFF
1683 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1684 //
1685 // Lanchor:
1686 // adr x17, Lanchor ; compute target address
1687 // add x16, x17, x16
1688 // br x16 ; branch to target
1689
1690 MachineOperand JTOp = MI.getOperand(0);
1691
1692 unsigned JTI = JTOp.getIndex();
1693 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1694 "unsupported compressed jump table");
1695
1696 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1697
1698 // cmp only supports a 12-bit immediate. If we need more, materialize the
1699 // immediate, using x17 as a scratch register.
1700 uint64_t MaxTableEntry = NumTableEntries - 1;
1701 if (isUInt<12>(MaxTableEntry)) {
1702 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXri)
1703 .addReg(AArch64::XZR)
1704 .addReg(AArch64::X16)
1705 .addImm(MaxTableEntry)
1706 .addImm(0));
1707 } else {
1708 emitMOVZ(AArch64::X17, static_cast<uint16_t>(MaxTableEntry), 0);
1709 // It's sad that we have to manually materialize instructions, but we can't
1710 // trivially reuse the main pseudo expansion logic.
1711 // A MOVK sequence is easy enough to generate and handles the general case.
1712 for (int Offset = 16; Offset < 64; Offset += 16) {
1713 if ((MaxTableEntry >> Offset) == 0)
1714 break;
1715 emitMOVK(AArch64::X17, static_cast<uint16_t>(MaxTableEntry >> Offset),
1716 Offset);
1717 }
1718 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXrs)
1719 .addReg(AArch64::XZR)
1720 .addReg(AArch64::X16)
1721 .addReg(AArch64::X17)
1722 .addImm(0));
1723 }
1724
1725 // This picks entry #0 on failure.
1726 // We might want to trap instead.
1727 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::CSELXr)
1728 .addReg(AArch64::X16)
1729 .addReg(AArch64::X16)
1730 .addReg(AArch64::XZR)
1731 .addImm(AArch64CC::LS));
1732
1733 // Prepare the @PAGE/@PAGEOFF low/high operands.
1734 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1735 MCOperand JTMCHi, JTMCLo;
1736
1737 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1738 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1739
1740 MCInstLowering.lowerOperand(JTMOHi, JTMCHi);
1741 MCInstLowering.lowerOperand(JTMOLo, JTMCLo);
1742
1743 EmitToStreamer(
1744 *OutStreamer,
1745 MCInstBuilder(AArch64::ADRP).addReg(AArch64::X17).addOperand(JTMCHi));
1746
1747 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXri)
1748 .addReg(AArch64::X17)
1749 .addReg(AArch64::X17)
1750 .addOperand(JTMCLo)
1751 .addImm(0));
1752
1753 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
1754 .addReg(AArch64::X16)
1755 .addReg(AArch64::X17)
1756 .addReg(AArch64::X16)
1757 .addImm(0)
1758 .addImm(1));
1759
1760 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1761 const auto *AdrLabelE = MCSymbolRefExpr::create(AdrLabel, MF->getContext());
1762 AArch64FI->setJumpTableEntryInfo(JTI, 4, AdrLabel);
1763
1764 OutStreamer->emitLabel(AdrLabel);
1765 EmitToStreamer(
1766 *OutStreamer,
1767 MCInstBuilder(AArch64::ADR).addReg(AArch64::X17).addExpr(AdrLabelE));
1768
1769 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1770 .addReg(AArch64::X16)
1771 .addReg(AArch64::X17)
1772 .addReg(AArch64::X16)
1773 .addImm(0));
1774
1775 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BR).addReg(AArch64::X16));
1776}
1777
1778void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1779 const llvm::MachineInstr &MI) {
1780 unsigned Opcode = MI.getOpcode();
1781 assert(STI->hasMOPS());
1782 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1783
1784 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1785 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1786 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1787 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1788 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1789 if (Opcode == AArch64::MOPSMemorySetPseudo)
1790 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1791 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1792 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1793 llvm_unreachable("Unhandled memory operation pseudo");
1794 }();
1795 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1796 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1797
1798 for (auto Op : Ops) {
1799 int i = 0;
1800 auto MCIB = MCInstBuilder(Op);
1801 // Destination registers
1802 MCIB.addReg(MI.getOperand(i++).getReg());
1803 MCIB.addReg(MI.getOperand(i++).getReg());
1804 if (!IsSet)
1805 MCIB.addReg(MI.getOperand(i++).getReg());
1806 // Input registers
1807 MCIB.addReg(MI.getOperand(i++).getReg());
1808 MCIB.addReg(MI.getOperand(i++).getReg());
1809 MCIB.addReg(MI.getOperand(i++).getReg());
1810
1811 EmitToStreamer(OutStreamer, MCIB);
1812 }
1813}
1814
1815void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1816 const MachineInstr &MI) {
1817 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1818
1819 auto &Ctx = OutStreamer.getContext();
1820 MCSymbol *MILabel = Ctx.createTempSymbol();
1821 OutStreamer.emitLabel(MILabel);
1822
1823 SM.recordStackMap(*MILabel, MI);
1824 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1825
1826 // Scan ahead to trim the shadow.
1827 const MachineBasicBlock &MBB = *MI.getParent();
1829 ++MII;
1830 while (NumNOPBytes > 0) {
1831 if (MII != MBB.end() && MII->isDebugInstr()) {
1832 ++MII;
1833 continue;
1834 }
1835 if (MII == MBB.end() || MII->isCall() ||
1836 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1837 MII->getOpcode() == TargetOpcode::STACKMAP)
1838 break;
1839 ++MII;
1840 NumNOPBytes -= 4;
1841 }
1842
1843 // Emit nops.
1844 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1845 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::NOP));
1846}
1847
1848// Lower a patchpoint of the form:
1849// [<def>], <id>, <numBytes>, <target>, <numArgs>
1850void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1851 const MachineInstr &MI) {
1852 auto &Ctx = OutStreamer.getContext();
1853 MCSymbol *MILabel = Ctx.createTempSymbol();
1854 OutStreamer.emitLabel(MILabel);
1855 SM.recordPatchPoint(*MILabel, MI);
1856
1857 PatchPointOpers Opers(&MI);
1858
1859 int64_t CallTarget = Opers.getCallTarget().getImm();
1860 unsigned EncodedBytes = 0;
1861 if (CallTarget) {
1862 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1863 "High 16 bits of call target should be zero.");
1864 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
1865 EncodedBytes = 16;
1866 // Materialize the jump address:
1867 emitMOVZ(ScratchReg, (CallTarget >> 32) & 0xFFFF, 32);
1868 emitMOVK(ScratchReg, (CallTarget >> 16) & 0xFFFF, 16);
1869 emitMOVK(ScratchReg, CallTarget & 0xFFFF, 0);
1870 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1871 }
1872 // Emit padding.
1873 unsigned NumBytes = Opers.getNumPatchBytes();
1874 assert(NumBytes >= EncodedBytes &&
1875 "Patchpoint can't request size less than the length of a call.");
1876 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1877 "Invalid number of NOP bytes requested!");
1878 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1879 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::NOP));
1880}
1881
1882void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1883 const MachineInstr &MI) {
1884 StatepointOpers SOpers(&MI);
1885 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1886 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1887 for (unsigned i = 0; i < PatchBytes; i += 4)
1888 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::NOP));
1889 } else {
1890 // Lower call target and choose correct opcode
1891 const MachineOperand &CallTarget = SOpers.getCallTarget();
1892 MCOperand CallTargetMCOp;
1893 unsigned CallOpcode;
1894 switch (CallTarget.getType()) {
1897 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1898 CallOpcode = AArch64::BL;
1899 break;
1901 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1902 CallOpcode = AArch64::BL;
1903 break;
1905 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1906 CallOpcode = AArch64::BLR;
1907 break;
1908 default:
1909 llvm_unreachable("Unsupported operand type in statepoint call target");
1910 break;
1911 }
1912
1913 EmitToStreamer(OutStreamer,
1914 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1915 }
1916
1917 auto &Ctx = OutStreamer.getContext();
1918 MCSymbol *MILabel = Ctx.createTempSymbol();
1919 OutStreamer.emitLabel(MILabel);
1920 SM.recordStatepoint(*MILabel, MI);
1921}
1922
1923void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1924 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1925 // <opcode>, <operands>
1926
1927 Register DefRegister = FaultingMI.getOperand(0).getReg();
1929 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1930 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1931 unsigned Opcode = FaultingMI.getOperand(3).getImm();
1932 unsigned OperandsBeginIdx = 4;
1933
1934 auto &Ctx = OutStreamer->getContext();
1935 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1936 OutStreamer->emitLabel(FaultingLabel);
1937
1938 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1939 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1940
1941 MCInst MI;
1942 MI.setOpcode(Opcode);
1943
1944 if (DefRegister != (Register)0)
1945 MI.addOperand(MCOperand::createReg(DefRegister));
1946
1947 for (const MachineOperand &MO :
1948 llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx)) {
1949 MCOperand Dest;
1950 lowerOperand(MO, Dest);
1951 MI.addOperand(Dest);
1952 }
1953
1954 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1955 EmitToStreamer(MI);
1956}
1957
1958void AArch64AsmPrinter::emitMovXReg(Register Dest, Register Src) {
1959 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
1960 .addReg(Dest)
1961 .addReg(AArch64::XZR)
1962 .addReg(Src)
1963 .addImm(0));
1964}
1965
1966void AArch64AsmPrinter::emitMOVZ(Register Dest, uint64_t Imm, unsigned Shift) {
1967 bool Is64Bit = AArch64::GPR64RegClass.contains(Dest);
1968 EmitToStreamer(*OutStreamer,
1969 MCInstBuilder(Is64Bit ? AArch64::MOVZXi : AArch64::MOVZWi)
1970 .addReg(Dest)
1971 .addImm(Imm)
1972 .addImm(Shift));
1973}
1974
1975void AArch64AsmPrinter::emitMOVK(Register Dest, uint64_t Imm, unsigned Shift) {
1976 bool Is64Bit = AArch64::GPR64RegClass.contains(Dest);
1977 EmitToStreamer(*OutStreamer,
1978 MCInstBuilder(Is64Bit ? AArch64::MOVKXi : AArch64::MOVKWi)
1979 .addReg(Dest)
1980 .addReg(Dest)
1981 .addImm(Imm)
1982 .addImm(Shift));
1983}
1984
1985void AArch64AsmPrinter::emitAUT(AArch64PACKey::ID Key, Register Pointer,
1986 Register Disc) {
1987 bool IsZeroDisc = Disc == AArch64::XZR;
1988 unsigned Opcode = getAUTOpcodeForKey(Key, IsZeroDisc);
1989
1990 // autiza x16 ; if IsZeroDisc
1991 // autia x16, x17 ; if !IsZeroDisc
1992 MCInst AUTInst;
1993 AUTInst.setOpcode(Opcode);
1994 AUTInst.addOperand(MCOperand::createReg(Pointer));
1995 AUTInst.addOperand(MCOperand::createReg(Pointer));
1996 if (!IsZeroDisc)
1997 AUTInst.addOperand(MCOperand::createReg(Disc));
1998
1999 EmitToStreamer(AUTInst);
2000}
2001
2002void AArch64AsmPrinter::emitPAC(AArch64PACKey::ID Key, Register Pointer,
2003 Register Disc) {
2004 bool IsZeroDisc = Disc == AArch64::XZR;
2005 unsigned Opcode = getPACOpcodeForKey(Key, IsZeroDisc);
2006
2007 // paciza x16 ; if IsZeroDisc
2008 // pacia x16, x17 ; if !IsZeroDisc
2009 MCInst PACInst;
2010 PACInst.setOpcode(Opcode);
2011 PACInst.addOperand(MCOperand::createReg(Pointer));
2012 PACInst.addOperand(MCOperand::createReg(Pointer));
2013 if (!IsZeroDisc)
2014 PACInst.addOperand(MCOperand::createReg(Disc));
2015
2016 EmitToStreamer(PACInst);
2017}
2018
2019void AArch64AsmPrinter::emitBLRA(bool IsCall, AArch64PACKey::ID Key,
2020 Register Target, Register Disc) {
2021 bool IsZeroDisc = Disc == AArch64::XZR;
2022 unsigned Opcode = getBranchOpcodeForKey(IsCall, Key, IsZeroDisc);
2023
2024 // blraaz x16 ; if IsZeroDisc
2025 // blraa x16, x17 ; if !IsZeroDisc
2026 MCInst Inst;
2027 Inst.setOpcode(Opcode);
2028 Inst.addOperand(MCOperand::createReg(Target));
2029 if (!IsZeroDisc)
2030 Inst.addOperand(MCOperand::createReg(Disc));
2031 EmitToStreamer(Inst);
2032}
2033
2034void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
2035 Register DestReg = MI.getOperand(0).getReg();
2036 if (!STI->hasZeroCycleZeroingFPWorkaround() && STI->isNeonAvailable()) {
2037 if (STI->hasZeroCycleZeroingFPR64()) {
2038 // Convert H/S register to corresponding D register
2039 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2040 if (AArch64::FPR16RegClass.contains(DestReg))
2041 DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::hsub,
2042 &AArch64::FPR64RegClass);
2043 else if (AArch64::FPR32RegClass.contains(DestReg))
2044 DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::ssub,
2045 &AArch64::FPR64RegClass);
2046 else
2047 assert(AArch64::FPR64RegClass.contains(DestReg));
2048
2049 MCInst MOVI;
2050 MOVI.setOpcode(AArch64::MOVID);
2051 MOVI.addOperand(MCOperand::createReg(DestReg));
2053 EmitToStreamer(*OutStreamer, MOVI);
2054 ++NumZCZeroingInstrsFPR;
2055 } else if (STI->hasZeroCycleZeroingFPR128()) {
2056 // Convert H/S/D register to corresponding Q register
2057 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2058 if (AArch64::FPR16RegClass.contains(DestReg)) {
2059 DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::hsub,
2060 &AArch64::FPR128RegClass);
2061 } else if (AArch64::FPR32RegClass.contains(DestReg)) {
2062 DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::ssub,
2063 &AArch64::FPR128RegClass);
2064 } else {
2065 assert(AArch64::FPR64RegClass.contains(DestReg));
2066 DestReg = TRI->getMatchingSuperReg(DestReg, AArch64::dsub,
2067 &AArch64::FPR128RegClass);
2068 }
2069
2070 MCInst MOVI;
2071 MOVI.setOpcode(AArch64::MOVIv2d_ns);
2072 MOVI.addOperand(MCOperand::createReg(DestReg));
2074 EmitToStreamer(*OutStreamer, MOVI);
2075 ++NumZCZeroingInstrsFPR;
2076 } else {
2077 emitFMov0AsFMov(MI, DestReg);
2078 }
2079 } else {
2080 emitFMov0AsFMov(MI, DestReg);
2081 }
2082}
2083
2084void AArch64AsmPrinter::emitFMov0AsFMov(const MachineInstr &MI,
2085 Register DestReg) {
2086 MCInst FMov;
2087 switch (MI.getOpcode()) {
2088 default:
2089 llvm_unreachable("Unexpected opcode");
2090 case AArch64::FMOVH0:
2091 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
2092 if (!STI->hasFullFP16())
2093 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
2094 FMov.addOperand(MCOperand::createReg(DestReg));
2095 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
2096 break;
2097 case AArch64::FMOVS0:
2098 FMov.setOpcode(AArch64::FMOVWSr);
2099 FMov.addOperand(MCOperand::createReg(DestReg));
2100 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
2101 break;
2102 case AArch64::FMOVD0:
2103 FMov.setOpcode(AArch64::FMOVXDr);
2104 FMov.addOperand(MCOperand::createReg(DestReg));
2105 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
2106 break;
2107 }
2108 EmitToStreamer(*OutStreamer, FMov);
2109}
2110
2111Register AArch64AsmPrinter::emitPtrauthDiscriminator(uint64_t Disc,
2112 Register AddrDisc,
2113 Register ScratchReg,
2114 bool MayClobberAddrDisc) {
2115 assert(isPtrauthRegSafe(ScratchReg) &&
2116 "Safe scratch register must be provided by the caller");
2117 assert(isUInt<16>(Disc) && "Constant discriminator is too wide");
2118
2119 // So far we've used NoRegister in pseudos. Now we need real encodings.
2120 if (!AddrDisc.isValid())
2121 AddrDisc = AArch64::XZR;
2122
2123 // If there is no constant discriminator, there's no blend involved:
2124 // just use the address discriminator register as-is (XZR or not).
2125 if (!Disc)
2126 return AddrDisc;
2127
2128 // If there's only a constant discriminator, MOV it into the scratch register.
2129 if (AddrDisc == AArch64::XZR) {
2130 emitMOVZ(ScratchReg, Disc, 0);
2131 return ScratchReg;
2132 }
2133
2134 // If there are both, emit a blend into the scratch register.
2135
2136 // Check if we can save one MOV instruction.
2137 if (MayClobberAddrDisc && isPtrauthRegSafe(AddrDisc)) {
2138 ScratchReg = AddrDisc;
2139 } else {
2140 emitMovXReg(ScratchReg, AddrDisc);
2141 assert(ScratchReg != AddrDisc &&
2142 "Forbidden to clobber AddrDisc, but have to");
2143 }
2144
2145 emitMOVK(ScratchReg, Disc, 48);
2146 return ScratchReg;
2147}
2148
2149/// Emit a code sequence to check an authenticated pointer value.
2150///
2151/// This function emits a sequence of instructions that checks if TestedReg was
2152/// authenticated successfully. On success, execution continues at the next
2153/// instruction after the sequence.
2154///
2155/// The action performed on failure depends on the OnFailure argument:
2156/// * if OnFailure is not nullptr, control is transferred to that label after
2157/// clearing the PAC field
2158/// * otherwise, BRK instruction is emitted to generate an error
2159void AArch64AsmPrinter::emitPtrauthCheckAuthenticatedValue(
2160 Register TestedReg, Register ScratchReg, AArch64PACKey::ID Key,
2161 AArch64PAuth::AuthCheckMethod Method, const MCSymbol *OnFailure) {
2162 // Insert a sequence to check if authentication of TestedReg succeeded,
2163 // such as:
2164 //
2165 // - checked and clearing:
2166 // ; x16 is TestedReg, x17 is ScratchReg
2167 // mov x17, x16
2168 // xpaci x17
2169 // cmp x16, x17
2170 // b.eq Lsuccess
2171 // mov x16, x17
2172 // b Lend
2173 // Lsuccess:
2174 // ; skipped if authentication failed
2175 // Lend:
2176 // ...
2177 //
2178 // - checked and trapping:
2179 // mov x17, x16
2180 // xpaci x17
2181 // cmp x16, x17
2182 // b.eq Lsuccess
2183 // brk #<0xc470 + aut key>
2184 // Lsuccess:
2185 // ...
2186 //
2187 // See the documentation on AuthCheckMethod enumeration constants for
2188 // the specific code sequences that can be used to perform the check.
2190
2191 if (Method == AuthCheckMethod::None)
2192 return;
2193 if (Method == AuthCheckMethod::DummyLoad) {
2194 EmitToStreamer(MCInstBuilder(AArch64::LDRWui)
2195 .addReg(getWRegFromXReg(ScratchReg))
2196 .addReg(TestedReg)
2197 .addImm(0));
2198 assert(!OnFailure && "DummyLoad always traps on error");
2199 return;
2200 }
2201
2202 MCSymbol *SuccessSym = createTempSymbol("auth_success_");
2203 if (Method == AuthCheckMethod::XPAC || Method == AuthCheckMethod::XPACHint) {
2204 // mov Xscratch, Xtested
2205 emitMovXReg(ScratchReg, TestedReg);
2206
2207 if (Method == AuthCheckMethod::XPAC) {
2208 // xpac(i|d) Xscratch
2209 unsigned XPACOpc = getXPACOpcodeForKey(Key);
2210 EmitToStreamer(
2211 MCInstBuilder(XPACOpc).addReg(ScratchReg).addReg(ScratchReg));
2212 } else {
2213 // xpaclri
2214
2215 // Note that this method applies XPAC to TestedReg instead of ScratchReg.
2216 assert(TestedReg == AArch64::LR &&
2217 "XPACHint mode is only compatible with checking the LR register");
2219 "XPACHint mode is only compatible with I-keys");
2220 EmitToStreamer(MCInstBuilder(AArch64::XPACLRI));
2221 }
2222
2223 // cmp Xtested, Xscratch
2224 EmitToStreamer(MCInstBuilder(AArch64::SUBSXrs)
2225 .addReg(AArch64::XZR)
2226 .addReg(TestedReg)
2227 .addReg(ScratchReg)
2228 .addImm(0));
2229
2230 // b.eq Lsuccess
2231 EmitToStreamer(
2232 MCInstBuilder(AArch64::Bcc)
2233 .addImm(AArch64CC::EQ)
2234 .addExpr(MCSymbolRefExpr::create(SuccessSym, OutContext)));
2235 } else if (Method == AuthCheckMethod::HighBitsNoTBI) {
2236 // eor Xscratch, Xtested, Xtested, lsl #1
2237 EmitToStreamer(MCInstBuilder(AArch64::EORXrs)
2238 .addReg(ScratchReg)
2239 .addReg(TestedReg)
2240 .addReg(TestedReg)
2241 .addImm(1));
2242 // tbz Xscratch, #62, Lsuccess
2243 EmitToStreamer(
2244 MCInstBuilder(AArch64::TBZX)
2245 .addReg(ScratchReg)
2246 .addImm(62)
2247 .addExpr(MCSymbolRefExpr::create(SuccessSym, OutContext)));
2248 } else {
2249 llvm_unreachable("Unsupported check method");
2250 }
2251
2252 if (!OnFailure) {
2253 // Trapping sequences do a 'brk'.
2254 // brk #<0xc470 + aut key>
2255 EmitToStreamer(MCInstBuilder(AArch64::BRK).addImm(0xc470 | Key));
2256 } else {
2257 // Non-trapping checked sequences return the stripped result in TestedReg,
2258 // skipping over success-only code (such as re-signing the pointer) by
2259 // jumping to OnFailure label.
2260 // Note that this can introduce an authentication oracle (such as based on
2261 // the high bits of the re-signed value).
2262
2263 // FIXME: The XPAC method can be optimized by applying XPAC to TestedReg
2264 // instead of ScratchReg, thus eliminating one `mov` instruction.
2265 // Both XPAC and XPACHint can be further optimized by not using a
2266 // conditional branch jumping over an unconditional one.
2267
2268 switch (Method) {
2269 case AuthCheckMethod::XPACHint:
2270 // LR is already XPAC-ed at this point.
2271 break;
2272 case AuthCheckMethod::XPAC:
2273 // mov Xtested, Xscratch
2274 emitMovXReg(TestedReg, ScratchReg);
2275 break;
2276 default:
2277 // If Xtested was not XPAC-ed so far, emit XPAC here.
2278 // xpac(i|d) Xtested
2279 unsigned XPACOpc = getXPACOpcodeForKey(Key);
2280 EmitToStreamer(
2281 MCInstBuilder(XPACOpc).addReg(TestedReg).addReg(TestedReg));
2282 }
2283
2284 // b Lend
2285 const auto *OnFailureExpr = MCSymbolRefExpr::create(OnFailure, OutContext);
2286 EmitToStreamer(MCInstBuilder(AArch64::B).addExpr(OnFailureExpr));
2287 }
2288
2289 // If the auth check succeeds, we can continue.
2290 // Lsuccess:
2291 OutStreamer->emitLabel(SuccessSym);
2292}
2293
2294// With Pointer Authentication, it may be needed to explicitly check the
2295// authenticated value in LR before performing a tail call.
2296// Otherwise, the callee may re-sign the invalid return address,
2297// introducing a signing oracle.
2298void AArch64AsmPrinter::emitPtrauthTailCallHardening(const MachineInstr *TC) {
2299 if (!AArch64FI->shouldSignReturnAddress(*MF))
2300 return;
2301
2302 auto LRCheckMethod = STI->getAuthenticatedLRCheckMethod(*MF);
2303 if (LRCheckMethod == AArch64PAuth::AuthCheckMethod::None)
2304 return;
2305
2306 const AArch64RegisterInfo *TRI = STI->getRegisterInfo();
2307 Register ScratchReg =
2308 TC->readsRegister(AArch64::X16, TRI) ? AArch64::X17 : AArch64::X16;
2309 assert(!TC->readsRegister(ScratchReg, TRI) &&
2310 "Neither x16 nor x17 is available as a scratch register");
2313 emitPtrauthCheckAuthenticatedValue(AArch64::LR, ScratchReg, Key,
2314 LRCheckMethod);
2315}
2316
2317bool AArch64AsmPrinter::emitDeactivationSymbolRelocation(Value *DS) {
2318 if (!DS)
2319 return false;
2320
2321 if (isa<GlobalAlias>(DS)) {
2322 // Just emit the nop directly.
2323 EmitToStreamer(MCInstBuilder(AArch64::NOP));
2324 return true;
2325 }
2326 MCSymbol *Dot = OutContext.createTempSymbol();
2327 OutStreamer->emitLabel(Dot);
2328 const MCExpr *DeactDotExpr = MCSymbolRefExpr::create(Dot, OutContext);
2329
2330 const MCExpr *DSExpr = MCSymbolRefExpr::create(
2331 OutContext.getOrCreateSymbol(DS->getName()), OutContext);
2332 OutStreamer->emitRelocDirective(*DeactDotExpr, "R_AARCH64_PATCHINST", DSExpr,
2333 SMLoc());
2334 return false;
2335}
2336
2337AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateImmReg(
2338 AArch64PACKey::ID Key, uint64_t IntDisc, const MachineOperand &AddrDiscOp) {
2339 PtrAuthSchema Schema;
2340 Schema.Key = Key;
2341 Schema.IntDisc = IntDisc;
2342 Schema.AddrDisc = AddrDiscOp.getReg();
2343 Schema.AddrDiscIsKilled = AddrDiscOp.isKill();
2344 Schema.PCDisc = Register();
2345 return Schema;
2346}
2347
2348AArch64AsmPrinter::PtrAuthSchema AArch64AsmPrinter::PtrAuthSchema::CreateRegReg(
2349 AArch64PACKey::ID Key, Register AddrDisc, Register PCDisc) {
2350 assert(PCDisc.isValid() && "Use CreateImmReg for non-PC schemas");
2351 PtrAuthSchema Schema;
2352 Schema.Key = Key;
2353 Schema.IntDisc = 0;
2354 Schema.AddrDisc = AddrDisc;
2355 Schema.AddrDiscIsKilled = false;
2356 Schema.PCDisc = PCDisc;
2357 return Schema;
2358}
2359
2360void AArch64AsmPrinter::emitPtrauthApplyIndirectAddend(Register Pointer,
2361 Register Scratch,
2362 int64_t Addend) {
2363 if (isInt<9>(Addend)) {
2364 // ldrsw Scratch, [Pointer, #Addend]! ; note: Pointer+Addend is used later.
2365 EmitToStreamer(MCInstBuilder(AArch64::LDRSWpre)
2366 .addReg(Pointer)
2367 .addReg(Scratch)
2368 .addReg(Pointer)
2369 .addImm(/*simm9:*/ Addend));
2370 } else {
2371 // Pointer += Addend computation has 2 variants
2372 if (isUInt<24>(Addend)) {
2373 // Variant 1: add Pointer, Pointer, (Addend >> shift12) lsl shift12
2374 // This can take up to 2 instructions.
2375 for (int BitPos = 0; BitPos != 24 && (Addend >> BitPos); BitPos += 12) {
2376 EmitToStreamer(
2377 MCInstBuilder(AArch64::ADDXri)
2378 .addReg(Pointer)
2379 .addReg(Pointer)
2380 .addImm((Addend >> BitPos) & 0xfff)
2381 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, BitPos)));
2382 }
2383 } else {
2384 // Variant 2: accumulate constant in Scratch 16 bits at a time,
2385 // and add it to Pointer. This can take 2-5 instructions.
2386 emitMOVZ(Scratch, Addend & 0xffff, 0);
2387 for (int Offset = 16; Offset < 64; Offset += 16) {
2388 if (unsigned Fragment = (Addend >> Offset) & 0xffff)
2389 emitMOVK(Scratch, Fragment, Offset);
2390 }
2391
2392 // add Pointer, Pointer, Scratch
2393 EmitToStreamer(MCInstBuilder(AArch64::ADDXrs)
2394 .addReg(Pointer)
2395 .addReg(Pointer)
2396 .addReg(Scratch)
2397 .addImm(0));
2398 }
2399 // ldrsw Scratch, [Pointer]
2400 EmitToStreamer(MCInstBuilder(AArch64::LDRSWui)
2401 .addReg(Scratch)
2402 .addReg(Pointer)
2403 .addImm(0));
2404 }
2405 // add Pointer, Pointer, Scratch
2406 EmitToStreamer(MCInstBuilder(AArch64::ADDXrs)
2407 .addReg(Pointer)
2408 .addReg(Pointer)
2409 .addReg(Scratch)
2410 .addImm(0));
2411}
2412
2413static PtrauthCheckMode getCheckMode(const MachineFunction *MF) {
2415
2416 // If an override is passed via command line argument, just use that value.
2417 if (std::optional<PtrauthCheckMode> Mode =
2418 STI.getCLOpts().ptrauth_auth_checks)
2419 return *Mode;
2420
2421 // Otherwise, on an FPAC CPU, you get traps whether you want them or not:
2422 // there's no point in emitting checks or traps.
2423 if (STI.hasFPAC())
2424 return PtrauthCheckMode::Unchecked;
2425
2426 bool ShouldTrap = MF->getFunction().hasFnAttribute("ptrauth-auth-traps");
2427 return ShouldTrap ? PtrauthCheckMode::Trap : PtrauthCheckMode::Poison;
2428}
2429
2430// We expand non-signing AUT* pseudo instructions into a sequence of the form
2431//
2432// ; 1. Authenticate Pointer
2433//
2434// or
2435//
2436// ; 1. Authenticate Pointer
2437// ; 2. Check that Pointer is valid, trap otherwise
2438//
2439// We expand AUT*PAC pseudo instructions into a sequence of the form
2440// (with addend only applied if Addend argument is given):
2441//
2442// ; 1. Authenticate Pointer
2443// ; 3. Apply addend and sign Pointer
2444//
2445// or
2446//
2447// ; 1. Authenticate Pointer
2448// ; 2. Check that Pointer is valid, trap otherwise
2449// ; 3. Apply addend and sign Pointer
2450//
2451// or
2452//
2453// ; 1. Authenticate Pointer
2454// ; 2. Check that Pointer is valid, jump to .Lon_failure otherwise
2455// ; 3. Apply addend and sign Pointer
2456// .Lon_failure:
2457//
2458void AArch64AsmPrinter::emitPtrauthAuthResign(
2459 Register Pointer, Register Scratch, PtrAuthSchema AuthSchema,
2460 std::optional<PtrAuthSchema> SignSchema, std::optional<int64_t> Addend,
2461 Value *DS) {
2462 const PtrauthCheckMode CheckMode = getCheckMode(MF);
2463 const bool IsAuthWithPC = AuthSchema.PCDisc.isValid();
2464 assert(!SignSchema || !SignSchema->PCDisc.isValid());
2465
2466 Register SignAddrDiscOrNone = SignSchema ? SignSchema->AddrDisc : Register();
2467
2468 // 1. Authenticate Pointer - this is the only common step.
2469 // It is more complex than signing because AUTI[AB]171615 may be used.
2470
2471 if (IsAuthWithPC) {
2472 assert(Pointer == AArch64::X17 && Scratch == AArch64::X16 &&
2473 "AUTPCPAC must use x17/x16 as Pointer/Scratch");
2474
2475 assert(AuthSchema.AddrDisc == AArch64::X16 &&
2476 "AUTPCPAC requires address discriminator in X16");
2477
2478 assert(AuthSchema.PCDisc == AArch64::X15 &&
2479 "AUTPCPAC requires PC discriminator in X15");
2480
2481 assert(AuthSchema.IntDisc == 0 && "AUTPCPAC does not support IntDisc");
2482
2483 assert((AuthSchema.Key == AArch64PACKey::IB ||
2484 AuthSchema.Key == AArch64PACKey::IA) &&
2485 "AUTPCPAC only supports AUT-ing with IA/IB");
2486
2487 if (!emitDeactivationSymbolRelocation(DS)) {
2488 unsigned AutOpc = (AuthSchema.Key == AArch64PACKey::IB)
2489 ? AArch64::AUTIB171615
2490 : AArch64::AUTIA171615;
2491 EmitToStreamer(MCInstBuilder(AutOpc));
2492 }
2493 } else {
2494 // emitPtrauthDiscriminator is allowed to clobber AuthSchema.AddrDisc as
2495 // long as it is not used past this point neither externally (the register
2496 // operand is "killed"), nor internally (it does not alias anything being
2497 // used later by this pseudo instruction).
2498 //
2499 // Note that, while rather unlikely, it is technically possible to use the
2500 // Pointer to compute its own discriminator.
2501 Register AUTDiscReg = emitPtrauthDiscriminator(
2502 AuthSchema.IntDisc, AuthSchema.AddrDisc, Scratch,
2503 AuthSchema.addrDiscIsKilledAndNoneOf({Pointer, SignAddrDiscOrNone}));
2504 if (!emitDeactivationSymbolRelocation(DS))
2505 emitAUT(AuthSchema.Key, Pointer, AUTDiscReg);
2506 }
2507
2508 // The other two steps are optional, define lambdas for them:
2509 // 2. Check that Pointer is valid, on failure jump to label or trap.
2510 auto EmitCheck = [&](MCSymbol *OnFailure = nullptr) {
2511 emitPtrauthCheckAuthenticatedValue(Pointer, Scratch, AuthSchema.Key,
2512 AArch64PAuth::AuthCheckMethod::XPAC,
2513 OnFailure);
2514 };
2515 // 3. Apply addend and sign Pointer.
2516 auto EmitResignOnSuccess = [&]() {
2517 if (Addend.has_value())
2518 emitPtrauthApplyIndirectAddend(Pointer, Scratch, *Addend);
2519
2520 assert(Pointer != SignSchema->AddrDisc && "Pointer is early-clobbered");
2521 Register PACDiscReg =
2522 emitPtrauthDiscriminator(SignSchema->IntDisc, SignSchema->AddrDisc,
2523 Scratch, SignSchema->AddrDiscIsKilled);
2524 emitPAC(SignSchema->Key, Pointer, PACDiscReg);
2525 };
2526
2527 // Emit checking and resigning as needed.
2528
2529 if (!SignSchema) {
2530 if (CheckMode == PtrauthCheckMode::Trap)
2531 EmitCheck();
2532 // For authentication-only pseudos, Poison is demoted to Unchecked.
2533 return;
2534 }
2535
2536 switch (CheckMode) {
2537 case PtrauthCheckMode::Unchecked:
2538 EmitResignOnSuccess();
2539 break;
2540 case PtrauthCheckMode::Trap:
2541 EmitCheck();
2542 EmitResignOnSuccess();
2543 break;
2544 case PtrauthCheckMode::Poison:
2545 MCSymbol *OnFailure = createTempSymbol("resign_end_");
2546 EmitCheck(OnFailure);
2547 EmitResignOnSuccess();
2548 OutStreamer->emitLabel(OnFailure);
2549 break;
2550 }
2551}
2552
2553void AArch64AsmPrinter::emitPtrauthSign(const MachineInstr *MI) {
2554 Register Val = MI->getOperand(1).getReg();
2555 auto Key = (AArch64PACKey::ID)MI->getOperand(2).getImm();
2556 uint64_t Disc = MI->getOperand(3).getImm();
2557 Register AddrDisc = MI->getOperand(4).getReg();
2558 bool AddrDiscKilled = MI->getOperand(4).isKill();
2559
2560 // As long as at least one of Val and AddrDisc is in GPR64noip, a scratch
2561 // register is available.
2562 Register ScratchReg = Val == AArch64::X16 ? AArch64::X17 : AArch64::X16;
2563 assert(ScratchReg != AddrDisc &&
2564 "Neither X16 nor X17 is available as a scratch register");
2565
2566 // Compute pac discriminator
2567 Register DiscReg = emitPtrauthDiscriminator(
2568 Disc, AddrDisc, ScratchReg, /*MayClobberAddrDisc=*/AddrDiscKilled);
2569
2570 if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol()))
2571 return;
2572
2573 emitPAC(Key, Val, DiscReg);
2574}
2575
2576void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2577 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2578 unsigned BrTarget = MI->getOperand(0).getReg();
2579
2580 auto Key = (AArch64PACKey::ID)MI->getOperand(1).getImm();
2581 uint64_t Disc = MI->getOperand(2).getImm();
2582
2583 unsigned AddrDisc = MI->getOperand(3).getReg();
2584
2585 // Make sure AddrDisc is solely used to compute the discriminator.
2586 // While hardly meaningful, it is still possible to describe an authentication
2587 // of a pointer against its own value (instead of storage address) with
2588 // intrinsics, so use report_fatal_error instead of assert.
2589 if (BrTarget == AddrDisc)
2590 report_fatal_error("Branch target is signed with its own value");
2591
2592 // If we are printing BLRA pseudo, try to save one MOV by making use of the
2593 // fact that x16 and x17 are described as clobbered by the MI instruction and
2594 // AddrDisc is not used as any other input.
2595 //
2596 // Back in the day, emitPtrauthDiscriminator was restricted to only returning
2597 // either x16 or x17, meaning the returned register is always among the
2598 // implicit-def'ed registers of BLRA pseudo. Now this property can be violated
2599 // if isX16X17Safer predicate is false, thus manually check if AddrDisc is
2600 // among x16 and x17 to prevent clobbering unexpected registers.
2601 //
2602 // Unlike BLRA, BRA pseudo is used to perform computed goto, and thus not
2603 // declared as clobbering x16/x17.
2604 //
2605 // FIXME: Make use of `killed` flags and register masks instead.
2606 bool AddrDiscIsImplicitDef =
2607 IsCall && (AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17);
2608 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, AArch64::X17,
2609 AddrDiscIsImplicitDef);
2610 emitBLRA(IsCall, Key, BrTarget, DiscReg);
2611}
2612
2613void AArch64AsmPrinter::emitAddImm(MCRegister Reg, int64_t Addend,
2614 MCRegister Tmp) {
2615 if (Addend != 0) {
2616 const uint64_t AbsOffset = (Addend > 0 ? Addend : -((uint64_t)Addend));
2617 const bool IsNeg = Addend < 0;
2618 if (isUInt<24>(AbsOffset)) {
2619 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2620 BitPos += 12) {
2621 EmitToStreamer(
2622 MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2623 .addReg(Reg)
2624 .addReg(Reg)
2625 .addImm((AbsOffset >> BitPos) & 0xfff)
2626 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, BitPos)));
2627 }
2628 } else {
2629 const uint64_t UAddend = Addend;
2630 EmitToStreamer(MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2631 .addReg(Tmp)
2632 .addImm((IsNeg ? ~UAddend : UAddend) & 0xffff)
2633 .addImm(/*shift=*/0));
2634 auto NeedMovk = [IsNeg, UAddend](int BitPos) -> bool {
2635 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2636 uint64_t Shifted = UAddend >> BitPos;
2637 if (!IsNeg)
2638 return Shifted != 0;
2639 for (int I = 0; I != 64 - BitPos; I += 16)
2640 if (((Shifted >> I) & 0xffff) != 0xffff)
2641 return true;
2642 return false;
2643 };
2644 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16)
2645 emitMOVK(Tmp, (UAddend >> BitPos) & 0xffff, BitPos);
2646
2647 EmitToStreamer(MCInstBuilder(AArch64::ADDXrs)
2648 .addReg(Reg)
2649 .addReg(Reg)
2650 .addReg(Tmp)
2651 .addImm(/*shift=*/0));
2652 }
2653 }
2654}
2655
2656void AArch64AsmPrinter::emitAddress(MCRegister Reg, const MCExpr *Expr,
2657 MCRegister Tmp, bool DSOLocal,
2658 const MCSubtargetInfo &STI) {
2659 MCValue Val;
2660 if (!Expr->evaluateAsRelocatable(Val, nullptr))
2661 report_fatal_error("emitAddress could not evaluate");
2662 if (DSOLocal) {
2663 EmitToStreamer(
2664 MCInstBuilder(AArch64::ADRP)
2665 .addReg(Reg)
2667 OutStreamer->getContext())));
2668 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
2669 .addReg(Reg)
2670 .addReg(Reg)
2671 .addExpr(MCSpecifierExpr::create(
2672 Expr, AArch64::S_LO12, OutStreamer->getContext()))
2673 .addImm(0));
2674 } else {
2675 auto *SymRef =
2676 MCSymbolRefExpr::create(Val.getAddSym(), OutStreamer->getContext());
2677 EmitToStreamer(
2678 MCInstBuilder(AArch64::ADRP)
2679 .addReg(Reg)
2681 OutStreamer->getContext())));
2682 EmitToStreamer(
2683 MCInstBuilder(AArch64::LDRXui)
2684 .addReg(Reg)
2685 .addReg(Reg)
2687 OutStreamer->getContext())));
2688 emitAddImm(Reg, Val.getConstant(), Tmp);
2689 }
2690}
2691
2693 // IFUNCs are ELF-only.
2694 if (!TT.isOSBinFormatELF())
2695 return false;
2696
2697 // IFUNCs are supported on glibc, bionic, and some but not all of the BSDs.
2698 return TT.isOSGlibc() || TT.isAndroid() || TT.isOSFreeBSD() ||
2699 TT.isOSDragonFly() || TT.isOSNetBSD();
2700}
2701
2702// Emit an ifunc resolver that returns a signed pointer to the specified target,
2703// and return a FUNCINIT reference to the resolver. In the linked binary, this
2704// function becomes the target of an IRELATIVE relocation. This resolver is used
2705// to relocate signed pointers in global variable initializers in special cases
2706// where the standard R_AARCH64_AUTH_ABS64 relocation would not work.
2707//
2708// Example (signed null pointer, not address discriminated):
2709//
2710// .8byte .Lpauth_ifunc0
2711// .pushsection .text.startup,"ax",@progbits
2712// .Lpauth_ifunc0:
2713// mov x0, #0
2714// mov x1, #12345
2715// b __emupac_pacda
2716//
2717// Example (signed null pointer, address discriminated):
2718//
2719// .Ltmp:
2720// .8byte .Lpauth_ifunc0
2721// .pushsection .text.startup,"ax",@progbits
2722// .Lpauth_ifunc0:
2723// mov x0, #0
2724// adrp x1, .Ltmp
2725// add x1, x1, :lo12:.Ltmp
2726// b __emupac_pacda
2727// .popsection
2728//
2729// Example (signed pointer to symbol, not address discriminated):
2730//
2731// .Ltmp:
2732// .8byte .Lpauth_ifunc0
2733// .pushsection .text.startup,"ax",@progbits
2734// .Lpauth_ifunc0:
2735// adrp x0, symbol
2736// add x0, x0, :lo12:symbol
2737// mov x1, #12345
2738// b __emupac_pacda
2739// .popsection
2740//
2741// Example (signed null pointer, not address discriminated, with deactivation
2742// symbol ds):
2743//
2744// .8byte .Lpauth_ifunc0
2745// .pushsection .text.startup,"ax",@progbits
2746// .Lpauth_ifunc0:
2747// mov x0, #0
2748// mov x1, #12345
2749// .reloc ., R_AARCH64_PATCHINST, ds
2750// b __emupac_pacda
2751// ret
2752// .popsection
2753const MCExpr *AArch64AsmPrinter::emitPAuthRelocationAsIRelative(
2754 const MCExpr *Target, uint64_t Disc, AArch64PACKey::ID KeyID,
2755 bool HasAddressDiversity, bool IsDSOLocal, const MCExpr *DSExpr) {
2756 const Triple &TT = TM.getTargetTriple();
2757
2758 // We only emit an IRELATIVE relocation if the target supports IRELATIVE.
2760 return nullptr;
2761
2762 // For now, only the DA key is supported.
2763 if (KeyID != AArch64PACKey::DA)
2764 return nullptr;
2765
2766 // AArch64Subtarget is huge, so heap allocate it so we don't run out of stack
2767 // space.
2768 auto STI = std::make_unique<AArch64Subtarget>(
2769 TT, TM.getTargetCPU(), TM.getTargetCPU(), TM.getTargetFeatureString(), TM,
2770 true);
2771 // This may be called while emitting a function, whose subtarget is restored.
2772 SaveAndRestore<const AArch64Subtarget *> SavedSTI(this->STI, STI.get());
2773
2774 MCSymbol *Place = OutStreamer->getContext().createTempSymbol();
2775 OutStreamer->emitLabel(Place);
2776 OutStreamer->pushSection();
2777
2778 const MCSymbolELF *Group =
2779 static_cast<MCSectionELF *>(OutStreamer->getCurrentSectionOnly())
2780 ->getGroup();
2782 if (Group)
2784 OutStreamer->switchSection(OutStreamer->getContext().getELFSection(
2785 ".text.startup", ELF::SHT_PROGBITS, Flags, 0, Group, true,
2786 Group ? MCSection::NonUniqueID : PAuthIFuncNextUniqueID++, nullptr));
2787
2788 MCSymbol *IRelativeSym =
2789 OutStreamer->getContext().createLinkerPrivateSymbol("pauth_ifunc");
2790 OutStreamer->emitLabel(IRelativeSym);
2791 if (isa<MCConstantExpr>(Target)) {
2792 OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi)
2793 .addReg(AArch64::X0)
2794 .addExpr(Target)
2795 .addImm(0),
2796 *STI);
2797 } else {
2798 emitAddress(AArch64::X0, Target, AArch64::X16, IsDSOLocal, *STI);
2799 }
2800 if (HasAddressDiversity) {
2801 auto *PlacePlusDisc = MCBinaryExpr::createAdd(
2802 MCSymbolRefExpr::create(Place, OutStreamer->getContext()),
2803 MCConstantExpr::create(Disc, OutStreamer->getContext()),
2804 OutStreamer->getContext());
2805 emitAddress(AArch64::X1, PlacePlusDisc, AArch64::X16, /*IsDSOLocal=*/true,
2806 *STI);
2807 } else {
2808 if (!isUInt<16>(Disc)) {
2809 OutContext.reportError(SMLoc(), "AArch64 PAC Discriminator '" +
2810 Twine(Disc) +
2811 "' out of range [0, 0xFFFF]");
2812 }
2813 emitMOVZ(AArch64::X1, Disc, 0);
2814 }
2815
2816 if (DSExpr) {
2817 MCSymbol *PrePACInst = OutStreamer->getContext().createTempSymbol();
2818 OutStreamer->emitLabel(PrePACInst);
2819
2820 auto *PrePACInstExpr =
2821 MCSymbolRefExpr::create(PrePACInst, OutStreamer->getContext());
2822 OutStreamer->emitRelocDirective(*PrePACInstExpr, "R_AARCH64_PATCHINST",
2823 DSExpr, SMLoc());
2824 }
2825
2826 // We don't know the subtarget because this is being emitted for a global
2827 // initializer. Because the performance of IFUNC resolvers is unimportant, we
2828 // always call the EmuPAC runtime, which will end up using the PAC instruction
2829 // if the target supports PAC.
2830 MCSymbol *EmuPAC =
2831 OutStreamer->getContext().getOrCreateSymbol("__emupac_pacda");
2832 const MCSymbolRefExpr *EmuPACRef =
2833 MCSymbolRefExpr::create(EmuPAC, OutStreamer->getContext());
2834 OutStreamer->emitInstruction(MCInstBuilder(AArch64::B).addExpr(EmuPACRef),
2835 *STI);
2836
2837 // We need a RET despite the above tail call because the deactivation symbol
2838 // may replace the tail call with a NOP.
2839 if (DSExpr)
2840 OutStreamer->emitInstruction(
2841 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
2842 OutStreamer->popSection();
2843
2845 MCSymbolRefExpr::create(IRelativeSym, OutStreamer->getContext()),
2846 AArch64::S_FUNCINIT, OutStreamer->getContext());
2847}
2848
2849const MCExpr *
2850AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2851 MCContext &Ctx = OutContext;
2852
2853 // Figure out the base symbol and the addend, if any.
2854 APInt Offset(64, 0);
2855 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2856 getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2857
2858 auto *BaseGVB = dyn_cast<GlobalValue>(BaseGV);
2859
2860 const MCExpr *Sym;
2861 if (BaseGVB) {
2862 // If there is an addend, turn that into the appropriate MCExpr.
2863 Sym = MCSymbolRefExpr::create(getSymbol(BaseGVB), Ctx);
2864 if (Offset.sgt(0))
2866 Sym, MCConstantExpr::create(Offset.getSExtValue(), Ctx), Ctx);
2867 else if (Offset.slt(0))
2869 Sym, MCConstantExpr::create((-Offset).getSExtValue(), Ctx), Ctx);
2870 } else if (isa<ConstantPointerNull>(BaseGV)) {
2871 Sym = MCConstantExpr::create(Offset.getSExtValue(), Ctx);
2872 } else {
2873 reportFatalUsageError("unsupported constant expression in ptrauth pointer");
2874 }
2875
2876 const MCExpr *DSExpr = nullptr;
2877 if (auto *DS = dyn_cast<GlobalValue>(CPA.getDeactivationSymbol())) {
2878 if (isa<GlobalAlias>(DS))
2879 return Sym;
2880 DSExpr = MCSymbolRefExpr::create(getSymbol(DS), Ctx);
2881 }
2882
2883 uint64_t KeyID = CPA.getKey()->getZExtValue();
2884 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2885 // AArch64AuthMCExpr::printImpl, so fail fast.
2886 if (KeyID > AArch64PACKey::LAST) {
2887 CPA.getContext().emitError("AArch64 PAC Key ID '" + Twine(KeyID) +
2888 "' out of range [0, " +
2889 Twine((unsigned)AArch64PACKey::LAST) + "]");
2890 KeyID = 0;
2891 }
2892
2893 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2894
2895 // Check if we can represent this with an IRELATIVE and emit it if so.
2896 if (auto *IFuncSym = emitPAuthRelocationAsIRelative(
2897 Sym, Disc, AArch64PACKey::ID(KeyID), CPA.hasAddressDiscriminator(),
2898 BaseGVB && BaseGVB->isDSOLocal(), DSExpr))
2899 return IFuncSym;
2900
2901 if (!isUInt<16>(Disc)) {
2902 CPA.getContext().emitError("AArch64 PAC Discriminator '" + Twine(Disc) +
2903 "' out of range [0, 0xFFFF]");
2904 Disc = 0;
2905 }
2906
2907 if (DSExpr)
2908 report_fatal_error("deactivation symbols unsupported in constant "
2909 "expressions on this target");
2910
2911 // Finally build the complete @AUTH expr.
2912 return AArch64AuthMCExpr::create(Sym, Disc, AArch64PACKey::ID(KeyID),
2913 CPA.hasAddressDiscriminator(), Ctx);
2914}
2915
2916void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2917 unsigned DstReg = MI.getOperand(0).getReg();
2918 const MachineOperand &GAOp = MI.getOperand(1);
2919 const uint64_t KeyC = MI.getOperand(2).getImm();
2920 assert(KeyC <= AArch64PACKey::LAST &&
2921 "key is out of range [0, AArch64PACKey::LAST]");
2922 const auto Key = (AArch64PACKey::ID)KeyC;
2923 const uint64_t Disc = MI.getOperand(3).getImm();
2924 assert(isUInt<16>(Disc) &&
2925 "constant discriminator is out of range [0, 0xffff]");
2926
2927 // Emit instruction sequence like the following:
2928 // ADRP x16, symbol$auth_ptr$key$disc
2929 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2930 //
2931 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2932 // to symbol.
2933 MCSymbol *AuthPtrStubSym;
2934 if (TM.getTargetTriple().isOSBinFormatELF()) {
2935 const auto &TLOF =
2936 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2937
2938 assert(GAOp.getOffset() == 0 &&
2939 "non-zero offset for $auth_ptr$ stub slots is not supported");
2940 const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
2941 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, GASym, Key, Disc);
2942 } else {
2943 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2944 "LOADauthptrstatic is implemented only for MachO/ELF");
2945
2946 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2947 getObjFileLowering());
2948
2949 assert(GAOp.getOffset() == 0 &&
2950 "non-zero offset for $auth_ptr$ stub slots is not supported");
2951 const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
2952 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, GASym, Key, Disc);
2953 }
2954
2955 MachineOperand StubMOHi =
2957 MachineOperand StubMOLo = MachineOperand::CreateMCSymbol(
2958 AuthPtrStubSym, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2959 MCOperand StubMCHi, StubMCLo;
2960
2961 MCInstLowering.lowerOperand(StubMOHi, StubMCHi);
2962 MCInstLowering.lowerOperand(StubMOLo, StubMCLo);
2963
2964 EmitToStreamer(
2965 *OutStreamer,
2966 MCInstBuilder(AArch64::ADRP).addReg(DstReg).addOperand(StubMCHi));
2967
2968 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRXui)
2969 .addReg(DstReg)
2970 .addReg(DstReg)
2971 .addOperand(StubMCLo));
2972}
2973
2974void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2975 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2976 const bool IsELFSignedGOT = MI.getParent()
2977 ->getParent()
2978 ->getInfo<AArch64FunctionInfo>()
2979 ->hasELFSignedGOT();
2980 MachineOperand GAOp = MI.getOperand(0);
2981 const uint64_t KeyC = MI.getOperand(1).getImm();
2982 assert(KeyC <= AArch64PACKey::LAST &&
2983 "key is out of range [0, AArch64PACKey::LAST]");
2984 const auto Key = (AArch64PACKey::ID)KeyC;
2985 const unsigned AddrDisc = MI.getOperand(2).getReg();
2986 const uint64_t Disc = MI.getOperand(3).getImm();
2987
2988 const int64_t Offset = GAOp.getOffset();
2989 GAOp.setOffset(0);
2990
2991 // Emit:
2992 // target materialization:
2993 // - via GOT:
2994 // - unsigned GOT:
2995 // adrp x16, :got:target
2996 // ldr x16, [x16, :got_lo12:target]
2997 // add offset to x16 if offset != 0
2998 // - ELF signed GOT:
2999 // adrp x17, :got:target
3000 // add x17, x17, :got_auth_lo12:target
3001 // ldr x16, [x17]
3002 // aut{i|d}a x16, x17
3003 // check+trap sequence (if no FPAC)
3004 // add offset to x16 if offset != 0
3005 //
3006 // - direct:
3007 // adrp x16, target
3008 // add x16, x16, :lo12:target
3009 // add offset to x16 if offset != 0
3010 //
3011 // add offset to x16:
3012 // - abs(offset) fits 24 bits:
3013 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
3014 // - abs(offset) does not fit 24 bits:
3015 // - offset < 0:
3016 // movn+movk sequence filling x17 register with the offset (up to 4
3017 // instructions)
3018 // add x16, x16, x17
3019 // - offset > 0:
3020 // movz+movk sequence filling x17 register with the offset (up to 4
3021 // instructions)
3022 // add x16, x16, x17
3023 //
3024 // signing:
3025 // - 0 discriminator:
3026 // paciza x16
3027 // - Non-0 discriminator, no address discriminator:
3028 // mov x17, #Disc
3029 // pacia x16, x17
3030 // - address discriminator (with potentially folded immediate discriminator):
3031 // pacia x16, xAddrDisc
3032
3033 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
3034 MCOperand GAMCHi, GAMCLo;
3035
3036 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
3037 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3038 if (IsGOTLoad) {
3039 GAMOHi.addTargetFlag(AArch64II::MO_GOT);
3040 GAMOLo.addTargetFlag(AArch64II::MO_GOT);
3041 }
3042
3043 MCInstLowering.lowerOperand(GAMOHi, GAMCHi);
3044 MCInstLowering.lowerOperand(GAMOLo, GAMCLo);
3045
3046 EmitToStreamer(
3047 MCInstBuilder(AArch64::ADRP)
3048 .addReg(IsGOTLoad && IsELFSignedGOT ? AArch64::X17 : AArch64::X16)
3049 .addOperand(GAMCHi));
3050
3051 if (IsGOTLoad) {
3052 if (IsELFSignedGOT) {
3053 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
3054 .addReg(AArch64::X17)
3055 .addReg(AArch64::X17)
3056 .addOperand(GAMCLo)
3057 .addImm(0));
3058
3059 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
3060 .addReg(AArch64::X16)
3061 .addReg(AArch64::X17)
3062 .addImm(0));
3063
3064 assert(GAOp.isGlobal());
3065 assert(GAOp.getGlobal()->getValueType() != nullptr);
3066
3067 bool IsFunctionTy = GAOp.getGlobal()->getValueType()->isFunctionTy();
3068 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3069 emitAUT(AuthKey, AArch64::X16, AArch64::X17);
3070
3071 if (!STI->hasFPAC())
3072 emitPtrauthCheckAuthenticatedValue(AArch64::X16, AArch64::X17, AuthKey,
3073 AArch64PAuth::AuthCheckMethod::XPAC);
3074 } else {
3075 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
3076 .addReg(AArch64::X16)
3077 .addReg(AArch64::X16)
3078 .addOperand(GAMCLo));
3079 }
3080 } else {
3081 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
3082 .addReg(AArch64::X16)
3083 .addReg(AArch64::X16)
3084 .addOperand(GAMCLo)
3085 .addImm(0));
3086 }
3087
3088 emitAddImm(AArch64::X16, Offset, AArch64::X17);
3089 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, AArch64::X17);
3090
3091 emitPAC(Key, AArch64::X16, DiscReg);
3092}
3093
3094void AArch64AsmPrinter::LowerLOADgotAUTH(const MachineInstr &MI) {
3095 Register DstReg = MI.getOperand(0).getReg();
3096 Register AuthResultReg = STI->hasFPAC() ? DstReg : AArch64::X16;
3097 const MachineOperand &GAMO = MI.getOperand(1);
3098 assert(GAMO.getOffset() == 0);
3099
3100 if (MI.getMF()->getTarget().getCodeModel() == CodeModel::Tiny) {
3101 MCOperand GAMC;
3102 MCInstLowering.lowerOperand(GAMO, GAMC);
3103 EmitToStreamer(
3104 MCInstBuilder(AArch64::ADR).addReg(AArch64::X17).addOperand(GAMC));
3105 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
3106 .addReg(AuthResultReg)
3107 .addReg(AArch64::X17)
3108 .addImm(0));
3109 } else {
3110 MachineOperand GAHiOp(GAMO);
3111 MachineOperand GALoOp(GAMO);
3112 GAHiOp.addTargetFlag(AArch64II::MO_PAGE);
3113 GALoOp.addTargetFlag(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
3114
3115 MCOperand GAMCHi, GAMCLo;
3116 MCInstLowering.lowerOperand(GAHiOp, GAMCHi);
3117 MCInstLowering.lowerOperand(GALoOp, GAMCLo);
3118
3119 EmitToStreamer(
3120 MCInstBuilder(AArch64::ADRP).addReg(AArch64::X17).addOperand(GAMCHi));
3121
3122 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
3123 .addReg(AArch64::X17)
3124 .addReg(AArch64::X17)
3125 .addOperand(GAMCLo)
3126 .addImm(0));
3127
3128 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
3129 .addReg(AuthResultReg)
3130 .addReg(AArch64::X17)
3131 .addImm(0));
3132 }
3133
3134 assert(GAMO.isGlobal());
3135 MCSymbol *UndefWeakSym;
3136 if (GAMO.getGlobal()->hasExternalWeakLinkage()) {
3137 UndefWeakSym = createTempSymbol("undef_weak");
3138 EmitToStreamer(
3139 MCInstBuilder(AArch64::CBZX)
3140 .addReg(AuthResultReg)
3141 .addExpr(MCSymbolRefExpr::create(UndefWeakSym, OutContext)));
3142 }
3143
3144 assert(GAMO.getGlobal()->getValueType() != nullptr);
3145
3146 bool IsFunctionTy = GAMO.getGlobal()->getValueType()->isFunctionTy();
3147 auto AuthKey = IsFunctionTy ? AArch64PACKey::IA : AArch64PACKey::DA;
3148 emitAUT(AuthKey, AuthResultReg, AArch64::X17);
3149
3150 if (GAMO.getGlobal()->hasExternalWeakLinkage())
3151 OutStreamer->emitLabel(UndefWeakSym);
3152
3153 if (!STI->hasFPAC()) {
3154 emitPtrauthCheckAuthenticatedValue(AuthResultReg, AArch64::X17, AuthKey,
3155 AArch64PAuth::AuthCheckMethod::XPAC);
3156
3157 emitMovXReg(DstReg, AuthResultReg);
3158 }
3159}
3160
3161const MCExpr *
3162AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
3163 const MCExpr *BAE = AsmPrinter::lowerBlockAddressConstant(BA);
3164 const Function &Fn = *BA.getFunction();
3165
3166 if (std::optional<uint16_t> BADisc =
3167 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(Fn))
3168 return AArch64AuthMCExpr::create(BAE, *BADisc, AArch64PACKey::IA,
3169 /*HasAddressDiversity=*/false, OutContext);
3170
3171 return BAE;
3172}
3173
3174void AArch64AsmPrinter::emitCBPseudoExpansion(const MachineInstr *MI) {
3175 bool IsImm = false;
3176 unsigned Width = 0;
3177
3178 switch (MI->getOpcode()) {
3179 default:
3180 llvm_unreachable("This is not a CB pseudo instruction");
3181 case AArch64::CBBAssertExt:
3182 IsImm = false;
3183 Width = 8;
3184 break;
3185 case AArch64::CBHAssertExt:
3186 IsImm = false;
3187 Width = 16;
3188 break;
3189 case AArch64::CBWPrr:
3190 Width = 32;
3191 break;
3192 case AArch64::CBXPrr:
3193 Width = 64;
3194 break;
3195 case AArch64::CBWPri:
3196 IsImm = true;
3197 Width = 32;
3198 break;
3199 case AArch64::CBXPri:
3200 IsImm = true;
3201 Width = 64;
3202 break;
3203 }
3204
3206 static_cast<AArch64CC::CondCode>(MI->getOperand(0).getImm());
3207 bool NeedsRegSwap = false;
3208 bool NeedsImmDec = false;
3209 bool NeedsImmInc = false;
3210
3211#define GET_CB_OPC(IsImm, Width, ImmCond, RegCond) \
3212 (IsImm \
3213 ? (Width == 32 ? AArch64::CB##ImmCond##Wri : AArch64::CB##ImmCond##Xri) \
3214 : (Width == 8 \
3215 ? AArch64::CBB##RegCond##Wrr \
3216 : (Width == 16 ? AArch64::CBH##RegCond##Wrr \
3217 : (Width == 32 ? AArch64::CB##RegCond##Wrr \
3218 : AArch64::CB##RegCond##Xrr))))
3219 unsigned MCOpC;
3220
3221 // Decide if we need to either swap register operands or increment/decrement
3222 // immediate operands
3223 switch (CC) {
3224 default:
3225 llvm_unreachable("Invalid CB condition code");
3226 case AArch64CC::EQ:
3227 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ EQ, /* Reg-Reg */ EQ);
3228 break;
3229 case AArch64CC::NE:
3230 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ NE, /* Reg-Reg */ NE);
3231 break;
3232 case AArch64CC::HS:
3233 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HS);
3234 NeedsImmDec = IsImm;
3235 break;
3236 case AArch64CC::LO:
3237 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HI);
3238 NeedsRegSwap = !IsImm;
3239 break;
3240 case AArch64CC::HI:
3241 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ HI, /* Reg-Reg */ HI);
3242 break;
3243 case AArch64CC::LS:
3244 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LO, /* Reg-Reg */ HS);
3245 NeedsRegSwap = !IsImm;
3246 NeedsImmInc = IsImm;
3247 break;
3248 case AArch64CC::GE:
3249 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GE);
3250 NeedsImmDec = IsImm;
3251 break;
3252 case AArch64CC::LT:
3253 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GT);
3254 NeedsRegSwap = !IsImm;
3255 break;
3256 case AArch64CC::GT:
3257 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ GT, /* Reg-Reg */ GT);
3258 break;
3259 case AArch64CC::LE:
3260 MCOpC = GET_CB_OPC(IsImm, Width, /* Reg-Imm */ LT, /* Reg-Reg */ GE);
3261 NeedsRegSwap = !IsImm;
3262 NeedsImmInc = IsImm;
3263 break;
3264 }
3265#undef GET_CB_OPC
3266
3267 MCInst Inst;
3268 Inst.setOpcode(MCOpC);
3269
3270 MCOperand Lhs, Rhs, Trgt;
3271 lowerOperand(MI->getOperand(1), Lhs);
3272 lowerOperand(MI->getOperand(2), Rhs);
3273 lowerOperand(MI->getOperand(3), Trgt);
3274
3275 // Now swap, increment or decrement
3276 if (NeedsRegSwap) {
3277 assert(Lhs.isReg() && "Expected register operand for CB");
3278 assert(Rhs.isReg() && "Expected register operand for CB");
3279 Inst.addOperand(Rhs);
3280 Inst.addOperand(Lhs);
3281 } else if (NeedsImmDec) {
3282 Rhs.setImm(Rhs.getImm() - 1);
3283 Inst.addOperand(Lhs);
3284 Inst.addOperand(Rhs);
3285 } else if (NeedsImmInc) {
3286 Rhs.setImm(Rhs.getImm() + 1);
3287 Inst.addOperand(Lhs);
3288 Inst.addOperand(Rhs);
3289 } else {
3290 Inst.addOperand(Lhs);
3291 Inst.addOperand(Rhs);
3292 }
3293
3294 assert((!IsImm || (Rhs.getImm() >= 0 && Rhs.getImm() < 64)) &&
3295 "CB immediate operand out-of-bounds");
3296
3297 Inst.addOperand(Trgt);
3298 EmitToStreamer(*OutStreamer, Inst);
3299}
3300
3301void AArch64AsmPrinter::emitAtomicHintPseudoExpansion(const MachineInstr *MI) {
3302
3303 unsigned StOpc;
3304 unsigned Relaxed = MI->getOperand(2).getImm();
3305 assert(Relaxed < 2 && "Atomic hint relaxed immediate out-of-bounds");
3306 switch (MI->getOpcode()) {
3307 case AArch64::ATOMIC_STORE_HINT_B:
3308 StOpc = Relaxed ? AArch64::STRBBui : AArch64::STLRB;
3309 break;
3310 case AArch64::ATOMIC_STORE_HINT_H:
3311 StOpc = Relaxed ? AArch64::STRHHui : AArch64::STLRH;
3312 break;
3313 case AArch64::ATOMIC_STORE_HINT_W:
3314 StOpc = Relaxed ? AArch64::STRWui : AArch64::STLRW;
3315 break;
3316 case AArch64::ATOMIC_STORE_HINT_X:
3317 StOpc = Relaxed ? AArch64::STRXui : AArch64::STLRX;
3318 break;
3319 default:
3320 llvm_unreachable("Unexpected atomic hint size.");
3321 }
3322
3323 EmitToStreamer(
3324 MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(3).getImm()));
3325
3326 MCInst Store;
3327 Store.setOpcode(StOpc);
3328 Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
3329 Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
3330 Store.setFlags(MI->getFlags());
3331 if (Relaxed)
3332 Store.addOperand(MCOperand::createImm(0));
3333 EmitToStreamer(*OutStreamer, Store);
3334}
3335
3336void AArch64AsmPrinter::emitAtomicHintPseudoExpansionRO(
3337 const MachineInstr *MI) {
3338 unsigned StOpc;
3339 assert(MI->getOperand(5).getImm() == 1 &&
3340 "Atomic store addressing mode only supports relaxed stores");
3341
3342 switch (MI->getOpcode()) {
3343 case AArch64::ATOMIC_STORE_HINT_BroW:
3344 StOpc = AArch64::STRBBroW;
3345 break;
3346 case AArch64::ATOMIC_STORE_HINT_HroW:
3347 StOpc = AArch64::STRHHroW;
3348 break;
3349 case AArch64::ATOMIC_STORE_HINT_WroW:
3350 StOpc = AArch64::STRWroW;
3351 break;
3352 case AArch64::ATOMIC_STORE_HINT_XroW:
3353 StOpc = AArch64::STRXroW;
3354 break;
3355 case AArch64::ATOMIC_STORE_HINT_SroW:
3356 StOpc = AArch64::STRSroW;
3357 break;
3358 case AArch64::ATOMIC_STORE_HINT_DroW:
3359 StOpc = AArch64::STRDroW;
3360 break;
3361 case AArch64::ATOMIC_STORE_HINT_BroX:
3362 StOpc = AArch64::STRBBroX;
3363 break;
3364 case AArch64::ATOMIC_STORE_HINT_HroX:
3365 StOpc = AArch64::STRHHroX;
3366 break;
3367 case AArch64::ATOMIC_STORE_HINT_WroX:
3368 StOpc = AArch64::STRWroX;
3369 break;
3370 case AArch64::ATOMIC_STORE_HINT_XroX:
3371 StOpc = AArch64::STRXroX;
3372 break;
3373 case AArch64::ATOMIC_STORE_HINT_SroX:
3374 StOpc = AArch64::STRSroX;
3375 break;
3376 case AArch64::ATOMIC_STORE_HINT_DroX:
3377 StOpc = AArch64::STRDroX;
3378 break;
3379 default:
3380 llvm_unreachable("Unexpected atomic hint opcode.");
3381 }
3382
3383 EmitToStreamer(
3384 MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(6).getImm()));
3385
3386 MCInst Store;
3387 Store.setOpcode(StOpc);
3388 Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
3389 Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Rm
3390 Store.addOperand(MCOperand::createReg(MI->getOperand(2).getReg())); // Data
3391 Store.addOperand(MCOperand::createImm(MI->getOperand(3).getImm())); // Signed
3392 Store.addOperand(MCOperand::createImm(MI->getOperand(4).getImm())); // Shift
3393 Store.setFlags(MI->getFlags());
3394 EmitToStreamer(*OutStreamer, Store);
3395}
3396
3397void AArch64AsmPrinter::emitAtomicHintPseudoExpansionImm(
3398 const MachineInstr *MI) {
3399 unsigned StOpc;
3400 assert(MI->getOperand(3).getImm() == 1 &&
3401 "Atomic store addressing mode only supports relaxed stores");
3402
3403 switch (MI->getOpcode()) {
3404 case AArch64::ATOMIC_STORE_HINT_Bui:
3405 StOpc = AArch64::STRBBui;
3406 break;
3407 case AArch64::ATOMIC_STORE_HINT_Hui:
3408 StOpc = AArch64::STRHHui;
3409 break;
3410 case AArch64::ATOMIC_STORE_HINT_Wui:
3411 StOpc = AArch64::STRWui;
3412 break;
3413 case AArch64::ATOMIC_STORE_HINT_Xui:
3414 StOpc = AArch64::STRXui;
3415 break;
3416 case AArch64::ATOMIC_STORE_HINT_Sui:
3417 StOpc = AArch64::STRSui;
3418 break;
3419 case AArch64::ATOMIC_STORE_HINT_Dui:
3420 StOpc = AArch64::STRDui;
3421 break;
3422 case AArch64::ATOMIC_STORE_HINT_Bi:
3423 StOpc = AArch64::STURBBi;
3424 break;
3425 case AArch64::ATOMIC_STORE_HINT_Hi:
3426 StOpc = AArch64::STURHHi;
3427 break;
3428 case AArch64::ATOMIC_STORE_HINT_Wi:
3429 StOpc = AArch64::STURWi;
3430 break;
3431 case AArch64::ATOMIC_STORE_HINT_Xi:
3432 StOpc = AArch64::STURXi;
3433 break;
3434 case AArch64::ATOMIC_STORE_HINT_Si:
3435 StOpc = AArch64::STURSi;
3436 break;
3437 case AArch64::ATOMIC_STORE_HINT_Di:
3438 StOpc = AArch64::STURDi;
3439 break;
3440 default:
3441 llvm_unreachable("Unexpected atomic hint opcode.");
3442 }
3443
3444 EmitToStreamer(
3445 MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm()));
3446
3447 MCInst Store;
3448 Store.setOpcode(StOpc);
3449 Store.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); // Rn
3450 Store.addOperand(MCOperand::createReg(MI->getOperand(1).getReg())); // Data
3451 Store.addOperand(MCOperand::createImm(MI->getOperand(2).getImm())); // Imm
3452 Store.setFlags(MI->getFlags());
3453 EmitToStreamer(*OutStreamer, Store);
3454}
3455
3456void AArch64AsmPrinter::emitAtomicFetchHintPseudoExpansion(
3457 const MachineInstr *MI) {
3458 EmitToStreamer(
3459 MCInstBuilder(AArch64::HINT).addImm(MI->getOperand(4).getImm()));
3460
3461 MCInst Fetch;
3462 Fetch.setOpcode(MI->getOperand(3).getImm());
3463 Fetch.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
3464 Fetch.addOperand(MCOperand::createReg(MI->getOperand(1).getReg()));
3465 Fetch.addOperand(MCOperand::createReg(MI->getOperand(2).getReg()));
3466 Fetch.setFlags(MI->getFlags());
3467 EmitToStreamer(*OutStreamer, Fetch);
3468}
3469
3470// Simple pseudo-instructions have their lowering (with expansion to real
3471// instructions) auto-generated.
3472#include "AArch64GenMCPseudoLowering.inc"
3473
3474void AArch64AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
3475 S.emitInstruction(Inst, *STI);
3476#ifndef NDEBUG
3477 ++InstsEmitted;
3478#endif
3479}
3480
3481void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
3482 emitInstructionImpl(MI);
3483 if (!MI->isBundledWithSucc())
3484 return;
3485
3486 const MachineInstr *EndProc = MI->getNextNode();
3487 assert(MI->isTerminator() && "Unexpected late AArch64 bundle head");
3488 assert(EndProc->getOpcode() == AArch64::SEH_EndProc &&
3489 !EndProc->isBundledWithSucc() &&
3490 "Unexpected late AArch64 bundle member");
3491 emitInstructionImpl(EndProc);
3492}
3493
3494void AArch64AsmPrinter::emitInstructionImpl(const MachineInstr *MI) {
3495 AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
3496
3497#ifndef NDEBUG
3498 InstsEmitted = 0;
3499 llvm::scope_exit CheckMISize([&]() {
3500 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
3501 });
3502#endif
3503
3504 // Do any auto-generated pseudo lowerings.
3505 if (MCInst OutInst; lowerPseudoInstExpansion(MI, OutInst)) {
3506 EmitToStreamer(*OutStreamer, OutInst);
3507 return;
3508 }
3509
3510 if (MI->getOpcode() == AArch64::ADRP) {
3511 for (auto &Opd : MI->operands()) {
3512 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
3513 "swift_async_extendedFramePointerFlags") {
3514 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
3515 }
3516 }
3517 }
3518
3519 if (AArch64FI->getLOHRelated().count(MI)) {
3520 // Generate a label for LOH related instruction
3521 MCSymbol *LOHLabel = createTempSymbol("loh");
3522 // Associate the instruction with the label
3523 LOHInstToLabel[MI] = LOHLabel;
3524 OutStreamer->emitLabel(LOHLabel);
3525 }
3526
3527 AArch64TargetStreamer *TS =
3528 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
3529 // Do any manual lowerings.
3530 switch (MI->getOpcode()) {
3531 default:
3533 "Unhandled tail call instruction");
3534 break;
3535 case AArch64::READ_REGISTER_GPR64:
3536 // Read of a named GPR: emit "mov Xt, Xn" (ORR Xt, XZR, Xn). The source
3537 // register is encoded as an immediate operand so that earlier passes do not
3538 // see a use of an undefined physical register.
3539 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
3540 .addReg(MI->getOperand(0).getReg())
3541 .addReg(AArch64::XZR)
3542 .addReg(MI->getOperand(1).getImm())
3543 .addImm(0));
3544 return;
3545 case AArch64::READ_REGISTER_FPR64:
3546 // Read of a named FP/SIMD d-register: emit "fmov Dt, Dn".
3547 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::FMOVDr)
3548 .addReg(MI->getOperand(0).getReg())
3549 .addReg(MI->getOperand(1).getImm()));
3550 return;
3551 case AArch64::HINT: {
3552 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
3553 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
3554 // non-empty. If MI is the initial BTI, place the
3555 // __patchable_function_entries label after BTI.
3556 if (CurrentPatchableFunctionEntrySym &&
3557 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
3558 MI == &MF->front().front()) {
3559 int64_t Imm = MI->getOperand(0).getImm();
3560 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38) {
3561 MCInst Inst;
3562 MCInstLowering.Lower(MI, Inst);
3563 EmitToStreamer(*OutStreamer, Inst);
3564 CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
3565 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
3566 return;
3567 }
3568 }
3569 break;
3570 }
3571 case AArch64::MOVMCSym: {
3572 Register DestReg = MI->getOperand(0).getReg();
3573 const MachineOperand &MO_Sym = MI->getOperand(1);
3574 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
3575 MCOperand Hi_MCSym, Lo_MCSym;
3576
3577 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
3578 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
3579
3580 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
3581 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
3582
3583 MCInst MovZ;
3584 MovZ.setOpcode(AArch64::MOVZXi);
3585 MovZ.addOperand(MCOperand::createReg(DestReg));
3586 MovZ.addOperand(Hi_MCSym);
3588 EmitToStreamer(*OutStreamer, MovZ);
3589
3590 MCInst MovK;
3591 MovK.setOpcode(AArch64::MOVKXi);
3592 MovK.addOperand(MCOperand::createReg(DestReg));
3593 MovK.addOperand(MCOperand::createReg(DestReg));
3594 MovK.addOperand(Lo_MCSym);
3596 EmitToStreamer(*OutStreamer, MovK);
3597 return;
3598 }
3599 case AArch64::MOVIv2d_ns:
3600 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
3601 // as movi is more efficient across all cores. Newer cores can eliminate
3602 // fmovs early and there is no difference with movi, but this not true for
3603 // all implementations.
3604 //
3605 // The floating-point version doesn't quite work in rare cases on older
3606 // CPUs, so on those targets we lower this instruction to movi.16b instead.
3607 if (STI->hasZeroCycleZeroingFPWorkaround() &&
3608 MI->getOperand(1).getImm() == 0) {
3609 MCInst TmpInst;
3610 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
3611 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
3612 TmpInst.addOperand(MCOperand::createImm(0));
3613 EmitToStreamer(*OutStreamer, TmpInst);
3614 return;
3615 }
3616 break;
3617
3618 case AArch64::DBG_VALUE:
3619 case AArch64::DBG_VALUE_LIST:
3620 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
3621 SmallString<128> TmpStr;
3622 raw_svector_ostream OS(TmpStr);
3623 PrintDebugValueComment(MI, OS);
3624 OutStreamer->emitRawText(StringRef(OS.str()));
3625 }
3626 return;
3627
3628 case AArch64::EMITBKEY:
3629 case AArch64::EMITNONHINTED: {
3630 ExceptionHandling ExceptionHandlingType = getModuleExceptionModel();
3631 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3632 ExceptionHandlingType != ExceptionHandling::ARM)
3633 return;
3634
3635 if (getFunctionCFISectionType(*MF) == CFISection::None)
3636 return;
3637
3638 if (MI->getOpcode() == AArch64::EMITBKEY)
3639 OutStreamer->emitCFIBKeyFrame();
3640 else
3641 OutStreamer->emitCFINonHintPAuthFrame();
3642
3643 return;
3644 }
3645
3646 case AArch64::EMITMTETAGGED: {
3647 ExceptionHandling ExceptionHandlingType = getModuleExceptionModel();
3648 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
3649 ExceptionHandlingType != ExceptionHandling::ARM)
3650 return;
3651
3652 if (getFunctionCFISectionType(*MF) != CFISection::None)
3653 OutStreamer->emitCFIMTETaggedFrame();
3654 return;
3655 }
3656
3657 case AArch64::AUTx16x17: {
3658 const Register Pointer = AArch64::X16;
3659 const Register Scratch = AArch64::X17;
3660
3661 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3662 (AArch64PACKey::ID)MI->getOperand(0).getImm(),
3663 MI->getOperand(1).getImm(), MI->getOperand(2));
3664
3665 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, std::nullopt,
3666 std::nullopt, MI->getDeactivationSymbol());
3667 return;
3668 }
3669
3670 case AArch64::AUTxMxN: {
3671 const Register Pointer = MI->getOperand(0).getReg();
3672 const Register Scratch = MI->getOperand(1).getReg();
3673
3674 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3675 (AArch64PACKey::ID)MI->getOperand(3).getImm(),
3676 MI->getOperand(4).getImm(), MI->getOperand(5));
3677
3678 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, std::nullopt,
3679 std::nullopt, MI->getDeactivationSymbol());
3680 return;
3681 }
3682
3683 case AArch64::AUTPAC: {
3684 const Register Pointer = AArch64::X16;
3685 const Register Scratch = AArch64::X17;
3686
3687 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3688 (AArch64PACKey::ID)MI->getOperand(0).getImm(),
3689 MI->getOperand(1).getImm(), MI->getOperand(2));
3690
3691 auto SignSchema = PtrAuthSchema::CreateImmReg(
3692 (AArch64PACKey::ID)MI->getOperand(3).getImm(),
3693 MI->getOperand(4).getImm(), MI->getOperand(5));
3694
3695 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3696 std::nullopt, MI->getDeactivationSymbol());
3697 return;
3698 }
3699
3700 case AArch64::AUTPCPAC: {
3701 auto AuthSchema = PtrAuthSchema::CreateRegReg(
3702 (AArch64PACKey::ID)MI->getOperand(0).getImm(), AArch64::X16,
3703 AArch64::X15);
3704
3705 auto SignSchema = PtrAuthSchema::CreateImmReg(
3706 (AArch64PACKey::ID)MI->getOperand(1).getImm(),
3707 MI->getOperand(2).getImm(), MI->getOperand(3));
3708
3709 emitPtrauthAuthResign(/*Pointer=*/AArch64::X17, /*Scratch=*/AArch64::X16,
3710 AuthSchema, SignSchema, std::nullopt,
3711 MI->getDeactivationSymbol());
3712 return;
3713 }
3714
3715 case AArch64::AUTRELLOADPAC: {
3716 const Register Pointer = AArch64::X16;
3717 const Register Scratch = AArch64::X17;
3718
3719 auto AuthSchema = PtrAuthSchema::CreateImmReg(
3720 (AArch64PACKey::ID)MI->getOperand(0).getImm(),
3721 MI->getOperand(1).getImm(), MI->getOperand(2));
3722
3723 auto SignSchema = PtrAuthSchema::CreateImmReg(
3724 (AArch64PACKey::ID)MI->getOperand(3).getImm(),
3725 MI->getOperand(4).getImm(), MI->getOperand(5));
3726
3727 emitPtrauthAuthResign(Pointer, Scratch, AuthSchema, SignSchema,
3728 MI->getOperand(6).getImm(),
3729 MI->getDeactivationSymbol());
3730
3731 return;
3732 }
3733
3734 case AArch64::PAC:
3735 emitPtrauthSign(MI);
3736 return;
3737
3738 case AArch64::LOADauthptrstatic:
3739 LowerLOADauthptrstatic(*MI);
3740 return;
3741
3742 case AArch64::LOADgotPAC:
3743 case AArch64::MOVaddrPAC:
3744 LowerMOVaddrPAC(*MI);
3745 return;
3746
3747 case AArch64::LOADgotAUTH:
3748 LowerLOADgotAUTH(*MI);
3749 return;
3750
3751 case AArch64::BRA:
3752 case AArch64::BLRA:
3753 emitPtrauthBranch(MI);
3754 return;
3755
3756 // Tail calls use pseudo instructions so they have the proper code-gen
3757 // attributes (isCall, isReturn, etc.). We lower them to the real
3758 // instruction here.
3759 case AArch64::AUTH_TCRETURN:
3760 case AArch64::AUTH_TCRETURN_BTI: {
3761 Register Callee = MI->getOperand(0).getReg();
3762 const auto Key = (AArch64PACKey::ID)MI->getOperand(2).getImm();
3763 const uint64_t Disc = MI->getOperand(3).getImm();
3764
3765 Register AddrDisc = MI->getOperand(4).getReg();
3766
3767 Register ScratchReg = Callee == AArch64::X16 ? AArch64::X17 : AArch64::X16;
3768
3769 emitPtrauthTailCallHardening(MI);
3770
3771 // See the comments in emitPtrauthBranch.
3772 if (Callee == AddrDisc)
3773 report_fatal_error("Call target is signed with its own value");
3774
3775 // After isX16X17Safer predicate was introduced, emitPtrauthDiscriminator is
3776 // no longer restricted to only reusing AddrDisc when it is X16 or X17
3777 // (which are implicit-def'ed by AUTH_TCRETURN pseudos), thus impose this
3778 // restriction manually not to clobber an unexpected register.
3779 bool AddrDiscIsImplicitDef =
3780 AddrDisc == AArch64::X16 || AddrDisc == AArch64::X17;
3781 Register DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, ScratchReg,
3782 AddrDiscIsImplicitDef);
3783 emitBLRA(/*IsCall*/ false, Key, Callee, DiscReg);
3784 return;
3785 }
3786
3787 case AArch64::TCRETURNri:
3788 case AArch64::TCRETURNrix16x17:
3789 case AArch64::TCRETURNrix17:
3790 case AArch64::TCRETURNrinotx16:
3791 case AArch64::TCRETURNriALL: {
3792 emitPtrauthTailCallHardening(MI);
3793
3794 recordIfImportCall(MI);
3795 MCInst TmpInst;
3796 TmpInst.setOpcode(AArch64::BR);
3797 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
3798 EmitToStreamer(*OutStreamer, TmpInst);
3799 return;
3800 }
3801 case AArch64::TCRETURNdi: {
3802 emitPtrauthTailCallHardening(MI);
3803
3804 MCOperand Dest;
3805 MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
3806 recordIfImportCall(MI);
3807 MCInst TmpInst;
3808 TmpInst.setOpcode(AArch64::B);
3809 TmpInst.addOperand(Dest);
3810 EmitToStreamer(*OutStreamer, TmpInst);
3811 return;
3812 }
3813 case AArch64::SpeculationBarrierISBDSBEndBB: {
3814 // Print DSB SYS + ISB
3815 MCInst TmpInstDSB;
3816 TmpInstDSB.setOpcode(AArch64::DSB);
3817 TmpInstDSB.addOperand(MCOperand::createImm(0xf));
3818 EmitToStreamer(*OutStreamer, TmpInstDSB);
3819 MCInst TmpInstISB;
3820 TmpInstISB.setOpcode(AArch64::ISB);
3821 TmpInstISB.addOperand(MCOperand::createImm(0xf));
3822 EmitToStreamer(*OutStreamer, TmpInstISB);
3823 return;
3824 }
3825 case AArch64::SpeculationBarrierSBEndBB: {
3826 // Print SB
3827 MCInst TmpInstSB;
3828 TmpInstSB.setOpcode(AArch64::SB);
3829 EmitToStreamer(*OutStreamer, TmpInstSB);
3830 return;
3831 }
3832 case AArch64::TLSDESC_AUTH_CALLSEQ: {
3833 /// lower this to:
3834 /// adrp x0, :tlsdesc_auth:var
3835 /// ldr x16, [x0, #:tlsdesc_auth_lo12:var]
3836 /// add x0, x0, #:tlsdesc_auth_lo12:var
3837 /// .tlsauthdesccall var
3838 /// blraa x16, x0
3839 /// (TPIDR_EL0 offset now in x0)
3840 const MachineOperand &MO_Sym = MI->getOperand(0);
3841 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3842 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3843 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3844 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3845 MCInstLowering.lowerOperand(MO_Sym, Sym);
3846 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
3847 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
3848
3849 MCInst Adrp;
3850 Adrp.setOpcode(AArch64::ADRP);
3851 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
3852 Adrp.addOperand(SymTLSDesc);
3853 EmitToStreamer(*OutStreamer, Adrp);
3854
3855 MCInst Ldr;
3856 Ldr.setOpcode(AArch64::LDRXui);
3857 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
3858 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
3859 Ldr.addOperand(SymTLSDescLo12);
3861 EmitToStreamer(*OutStreamer, Ldr);
3862
3863 MCInst Add;
3864 Add.setOpcode(AArch64::ADDXri);
3865 Add.addOperand(MCOperand::createReg(AArch64::X0));
3866 Add.addOperand(MCOperand::createReg(AArch64::X0));
3867 Add.addOperand(SymTLSDescLo12);
3869 EmitToStreamer(*OutStreamer, Add);
3870
3871 // Emit a relocation-annotation. This expands to no code, but requests
3872 // the following instruction gets an R_AARCH64_AUTH_TLSDESC_CALL.
3873 MCInst TLSAuthDescCall;
3874 TLSAuthDescCall.setOpcode(AArch64::TLSAUTHDESCCALL);
3875 TLSAuthDescCall.addOperand(Sym);
3876 EmitToStreamer(*OutStreamer, TLSAuthDescCall);
3877#ifndef NDEBUG
3878 --InstsEmitted; // no code emitted
3879#endif
3880
3881 MCInst Blraa;
3882 Blraa.setOpcode(AArch64::BLRAA);
3883 Blraa.addOperand(MCOperand::createReg(AArch64::X16));
3884 Blraa.addOperand(MCOperand::createReg(AArch64::X0));
3885 EmitToStreamer(*OutStreamer, Blraa);
3886
3887 return;
3888 }
3889 case AArch64::TLSDESC_CALLSEQ: {
3890 /// lower this to:
3891 /// adrp x0, :tlsdesc:var
3892 /// ldr x1, [x0, #:tlsdesc_lo12:var]
3893 /// add x0, x0, #:tlsdesc_lo12:var
3894 /// .tlsdesccall var
3895 /// blr x1
3896 /// (TPIDR_EL0 offset now in x0)
3897 const MachineOperand &MO_Sym = MI->getOperand(0);
3898 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
3899 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
3900 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
3901 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
3902 MCInstLowering.lowerOperand(MO_Sym, Sym);
3903 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
3904 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
3905
3906 MCInst Adrp;
3907 Adrp.setOpcode(AArch64::ADRP);
3908 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
3909 Adrp.addOperand(SymTLSDesc);
3910 EmitToStreamer(*OutStreamer, Adrp);
3911
3912 MCInst Ldr;
3913 if (STI->isTargetILP32()) {
3914 Ldr.setOpcode(AArch64::LDRWui);
3915 Ldr.addOperand(MCOperand::createReg(AArch64::W1));
3916 } else {
3917 Ldr.setOpcode(AArch64::LDRXui);
3918 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
3919 }
3920 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
3921 Ldr.addOperand(SymTLSDescLo12);
3923 EmitToStreamer(*OutStreamer, Ldr);
3924
3925 MCInst Add;
3926 if (STI->isTargetILP32()) {
3927 Add.setOpcode(AArch64::ADDWri);
3928 Add.addOperand(MCOperand::createReg(AArch64::W0));
3929 Add.addOperand(MCOperand::createReg(AArch64::W0));
3930 } else {
3931 Add.setOpcode(AArch64::ADDXri);
3932 Add.addOperand(MCOperand::createReg(AArch64::X0));
3933 Add.addOperand(MCOperand::createReg(AArch64::X0));
3934 }
3935 Add.addOperand(SymTLSDescLo12);
3937 EmitToStreamer(*OutStreamer, Add);
3938
3939 // Emit a relocation-annotation. This expands to no code, but requests
3940 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
3941 MCInst TLSDescCall;
3942 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
3943 TLSDescCall.addOperand(Sym);
3944 EmitToStreamer(*OutStreamer, TLSDescCall);
3945#ifndef NDEBUG
3946 --InstsEmitted; // no code emitted
3947#endif
3948
3949 MCInst Blr;
3950 Blr.setOpcode(AArch64::BLR);
3951 Blr.addOperand(MCOperand::createReg(AArch64::X1));
3952 EmitToStreamer(*OutStreamer, Blr);
3953
3954 return;
3955 }
3956
3957 case AArch64::JumpTableDest32:
3958 case AArch64::JumpTableDest16:
3959 case AArch64::JumpTableDest8:
3960 LowerJumpTableDest(*OutStreamer, *MI);
3961 return;
3962
3963 case AArch64::BR_JumpTable:
3964 LowerHardenedBRJumpTable(*MI);
3965 return;
3966
3967 case AArch64::FMOVH0:
3968 case AArch64::FMOVS0:
3969 case AArch64::FMOVD0:
3970 emitFMov0(*MI);
3971 return;
3972
3973 case AArch64::MOPSMemoryCopyPseudo:
3974 case AArch64::MOPSMemoryMovePseudo:
3975 case AArch64::MOPSMemorySetPseudo:
3976 case AArch64::MOPSMemorySetTaggingPseudo:
3977 LowerMOPS(*OutStreamer, *MI);
3978 return;
3979
3980 case TargetOpcode::STACKMAP:
3981 return LowerSTACKMAP(*OutStreamer, SM, *MI);
3982
3983 case TargetOpcode::PATCHPOINT:
3984 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
3985
3986 case TargetOpcode::STATEPOINT:
3987 return LowerSTATEPOINT(*OutStreamer, SM, *MI);
3988
3989 case TargetOpcode::FAULTING_OP:
3990 return LowerFAULTING_OP(*MI);
3991
3992 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
3993 LowerPATCHABLE_FUNCTION_ENTER(*MI);
3994 return;
3995
3996 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
3997 LowerPATCHABLE_FUNCTION_EXIT(*MI);
3998 return;
3999
4000 case TargetOpcode::PATCHABLE_TAIL_CALL:
4001 LowerPATCHABLE_TAIL_CALL(*MI);
4002 return;
4003 case TargetOpcode::PATCHABLE_EVENT_CALL:
4004 return LowerPATCHABLE_EVENT_CALL(*MI, false);
4005 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
4006 return LowerPATCHABLE_EVENT_CALL(*MI, true);
4007
4008 case AArch64::KCFI_CHECK:
4009 LowerKCFI_CHECK(*MI);
4010 return;
4011
4012 case AArch64::HWASAN_CHECK_MEMACCESS:
4013 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
4014 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
4015 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
4016 LowerHWASAN_CHECK_MEMACCESS(*MI);
4017 return;
4018
4019 case AArch64::SEH_StartProc:
4020#ifndef NDEBUG
4021 assert(!ExplicitWinCFIRangeOpen && "Starting an explicit Windows CFI range "
4022 "before ending the previous one");
4023 ExplicitWinCFIRangeOpen = true;
4024 EmittedExplicitWinCFIRange = true;
4025#endif
4026 OutStreamer->emitWinCFIStartProc(CurrentFnSym);
4027 return;
4028
4029 case AArch64::SEH_EndProc:
4030#ifndef NDEBUG
4031 assert(ExplicitWinCFIRangeOpen &&
4032 "Ending an explicit Windows CFI range without a matching start");
4033 ExplicitWinCFIRangeOpen = false;
4034#endif
4035 OutStreamer->emitWinCFIFuncletOrFuncEnd();
4036 OutStreamer->emitWinCFIEndProc();
4037 return;
4038
4039 case AArch64::SEH_EndC:
4040 TS->emitARM64WinCFIEndC();
4041 return;
4042
4043 case AArch64::SEH_StackAlloc:
4044 TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
4045 return;
4046
4047 case AArch64::SEH_SaveFPLR:
4048 TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
4049 return;
4050
4051 case AArch64::SEH_SaveFPLR_X:
4052 assert(MI->getOperand(0).getImm() < 0 &&
4053 "Pre increment SEH opcode must have a negative offset");
4054 TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
4055 return;
4056
4057 case AArch64::SEH_SaveReg:
4058 TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
4059 MI->getOperand(1).getImm());
4060 return;
4061
4062 case AArch64::SEH_SaveReg_X:
4063 assert(MI->getOperand(1).getImm() < 0 &&
4064 "Pre increment SEH opcode must have a negative offset");
4065 TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
4066 -MI->getOperand(1).getImm());
4067 return;
4068
4069 case AArch64::SEH_SaveRegP:
4070 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
4071 MI->getOperand(0).getImm() <= 28) {
4072 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
4073 "Register paired with LR must be odd");
4074 TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
4075 MI->getOperand(2).getImm());
4076 return;
4077 }
4078 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4079 "Non-consecutive registers not allowed for save_regp");
4080 TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
4081 MI->getOperand(2).getImm());
4082 return;
4083
4084 case AArch64::SEH_SaveRegP_X:
4085 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4086 "Non-consecutive registers not allowed for save_regp_x");
4087 assert(MI->getOperand(2).getImm() < 0 &&
4088 "Pre increment SEH opcode must have a negative offset");
4089 TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
4090 -MI->getOperand(2).getImm());
4091 return;
4092
4093 case AArch64::SEH_SaveFReg:
4094 TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
4095 MI->getOperand(1).getImm());
4096 return;
4097
4098 case AArch64::SEH_SaveFReg_X:
4099 assert(MI->getOperand(1).getImm() < 0 &&
4100 "Pre increment SEH opcode must have a negative offset");
4101 TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
4102 -MI->getOperand(1).getImm());
4103 return;
4104
4105 case AArch64::SEH_SaveFRegP:
4106 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4107 "Non-consecutive registers not allowed for save_regp");
4108 TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
4109 MI->getOperand(2).getImm());
4110 return;
4111
4112 case AArch64::SEH_SaveFRegP_X:
4113 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
4114 "Non-consecutive registers not allowed for save_regp_x");
4115 assert(MI->getOperand(2).getImm() < 0 &&
4116 "Pre increment SEH opcode must have a negative offset");
4117 TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
4118 -MI->getOperand(2).getImm());
4119 return;
4120
4121 case AArch64::SEH_SetFP:
4123 return;
4124
4125 case AArch64::SEH_AddFP:
4126 TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm());
4127 return;
4128
4129 case AArch64::SEH_Nop:
4130 TS->emitARM64WinCFINop();
4131 return;
4132
4133 case AArch64::SEH_PrologEnd:
4135 return;
4136
4137 case AArch64::SEH_EpilogStart:
4139 return;
4140
4141 case AArch64::SEH_EpilogEnd:
4143 return;
4144
4145 case AArch64::SEH_PACSignLR:
4147 return;
4148
4149 case AArch64::SEH_SaveAnyRegI:
4150 assert((isShiftedUInt<6, 3>((uint64_t)MI->getOperand(1).getImm())) &&
4151 "SaveAnyRegI SEH opcode offset must fit into 6 bits");
4152 TS->emitARM64WinCFISaveAnyRegI(MI->getOperand(0).getImm(),
4153 MI->getOperand(1).getImm());
4154 return;
4155
4156 case AArch64::SEH_SaveAnyRegIP:
4157 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4158 "Non-consecutive registers not allowed for save_any_reg");
4159 assert((isShiftedUInt<6, 4>((uint64_t)MI->getOperand(2).getImm())) &&
4160 "SaveAnyRegIP SEH opcode offset must fit into 6 bits");
4161 TS->emitARM64WinCFISaveAnyRegIP(MI->getOperand(0).getImm(),
4162 MI->getOperand(2).getImm());
4163 return;
4164
4165 case AArch64::SEH_SaveAnyRegQ:
4166 assert((isShiftedUInt<6, 4>((uint64_t)MI->getOperand(1).getImm())) &&
4167 "SaveAnyRegQ SEH opcode offset must fit into 6 bits");
4168 TS->emitARM64WinCFISaveAnyRegQ(MI->getOperand(0).getImm(),
4169 MI->getOperand(1).getImm());
4170 return;
4171
4172 case AArch64::SEH_SaveAnyRegQX:
4173 assert((isShiftedUInt<6, 4>(-(uint64_t)MI->getOperand(1).getImm())) &&
4174 "SaveAnyRegQX SEH opcode offset must fit into 6 bits");
4175 TS->emitARM64WinCFISaveAnyRegQX(MI->getOperand(0).getImm(),
4176 -MI->getOperand(1).getImm());
4177 return;
4178
4179 case AArch64::SEH_SaveAnyRegQP:
4180 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4181 "Non-consecutive registers not allowed for save_any_reg");
4182 assert((isShiftedUInt<6, 4>((uint64_t)MI->getOperand(2).getImm())) &&
4183 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
4184 TS->emitARM64WinCFISaveAnyRegQP(MI->getOperand(0).getImm(),
4185 MI->getOperand(2).getImm());
4186 return;
4187
4188 case AArch64::SEH_SaveAnyRegQPX:
4189 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
4190 "Non-consecutive registers not allowed for save_any_reg");
4191 assert((isShiftedUInt<6, 4>(-(uint64_t)MI->getOperand(2).getImm())) &&
4192 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
4193 TS->emitARM64WinCFISaveAnyRegQPX(MI->getOperand(0).getImm(),
4194 -MI->getOperand(2).getImm());
4195 return;
4196
4197 case AArch64::SEH_AllocZ:
4198 assert(MI->getOperand(0).getImm() >= 0 &&
4199 "AllocZ SEH opcode offset must be non-negative");
4200 assert(MI->getOperand(0).getImm() <= 255 &&
4201 "AllocZ SEH opcode offset must fit into 8 bits");
4202 TS->emitARM64WinCFIAllocZ(MI->getOperand(0).getImm());
4203 return;
4204
4205 case AArch64::SEH_SaveZReg:
4206 assert(MI->getOperand(1).getImm() >= 0 &&
4207 "SaveZReg SEH opcode offset must be non-negative");
4208 assert(MI->getOperand(1).getImm() <= 255 &&
4209 "SaveZReg SEH opcode offset must fit into 8 bits");
4210 TS->emitARM64WinCFISaveZReg(MI->getOperand(0).getImm(),
4211 MI->getOperand(1).getImm());
4212 return;
4213
4214 case AArch64::SEH_SavePReg:
4215 assert(MI->getOperand(1).getImm() >= 0 &&
4216 "SavePReg SEH opcode offset must be non-negative");
4217 assert(MI->getOperand(1).getImm() <= 255 &&
4218 "SavePReg SEH opcode offset must fit into 8 bits");
4219 TS->emitARM64WinCFISavePReg(MI->getOperand(0).getImm(),
4220 MI->getOperand(1).getImm());
4221 return;
4222
4223 case AArch64::BLR:
4224 case AArch64::BR: {
4225 recordIfImportCall(MI);
4226 MCInst TmpInst;
4227 MCInstLowering.Lower(MI, TmpInst);
4228 EmitToStreamer(*OutStreamer, TmpInst);
4229 return;
4230 }
4231 case AArch64::CBWPri:
4232 case AArch64::CBXPri:
4233 case AArch64::CBBAssertExt:
4234 case AArch64::CBHAssertExt:
4235 case AArch64::CBWPrr:
4236 case AArch64::CBXPrr:
4237 emitCBPseudoExpansion(MI);
4238 return;
4239 case AArch64::ATOMIC_STORE_HINT_B:
4240 case AArch64::ATOMIC_STORE_HINT_H:
4241 case AArch64::ATOMIC_STORE_HINT_W:
4242 case AArch64::ATOMIC_STORE_HINT_X:
4243 emitAtomicHintPseudoExpansion(MI);
4244 return;
4245 case AArch64::ATOMIC_FETCH_HINT_W:
4246 case AArch64::ATOMIC_FETCH_HINT_X:
4247 emitAtomicFetchHintPseudoExpansion(MI);
4248 return;
4249 case AArch64::ATOMIC_STORE_HINT_BroW:
4250 case AArch64::ATOMIC_STORE_HINT_HroW:
4251 case AArch64::ATOMIC_STORE_HINT_WroW:
4252 case AArch64::ATOMIC_STORE_HINT_XroW:
4253 case AArch64::ATOMIC_STORE_HINT_SroW:
4254 case AArch64::ATOMIC_STORE_HINT_DroW:
4255 case AArch64::ATOMIC_STORE_HINT_BroX:
4256 case AArch64::ATOMIC_STORE_HINT_HroX:
4257 case AArch64::ATOMIC_STORE_HINT_WroX:
4258 case AArch64::ATOMIC_STORE_HINT_XroX:
4259 case AArch64::ATOMIC_STORE_HINT_SroX:
4260 case AArch64::ATOMIC_STORE_HINT_DroX:
4261 emitAtomicHintPseudoExpansionRO(MI);
4262 return;
4263 case AArch64::ATOMIC_STORE_HINT_Bui:
4264 case AArch64::ATOMIC_STORE_HINT_Hui:
4265 case AArch64::ATOMIC_STORE_HINT_Wui:
4266 case AArch64::ATOMIC_STORE_HINT_Xui:
4267 case AArch64::ATOMIC_STORE_HINT_Sui:
4268 case AArch64::ATOMIC_STORE_HINT_Dui:
4269 case AArch64::ATOMIC_STORE_HINT_Bi:
4270 case AArch64::ATOMIC_STORE_HINT_Hi:
4271 case AArch64::ATOMIC_STORE_HINT_Wi:
4272 case AArch64::ATOMIC_STORE_HINT_Xi:
4273 case AArch64::ATOMIC_STORE_HINT_Si:
4274 case AArch64::ATOMIC_STORE_HINT_Di:
4275 emitAtomicHintPseudoExpansionImm(MI);
4276 return;
4277 }
4278
4279 if (emitDeactivationSymbolRelocation(MI->getDeactivationSymbol()))
4280 return;
4281
4282 // Finally, do the automated lowerings for everything else.
4283 MCInst TmpInst;
4284 MCInstLowering.Lower(MI, TmpInst);
4285 EmitToStreamer(*OutStreamer, TmpInst);
4286}
4287
4288void AArch64AsmPrinter::recordIfImportCall(
4289 const llvm::MachineInstr *BranchInst) {
4290 if (!EnableImportCallOptimization)
4291 return;
4292
4293 auto [GV, OpFlags] = BranchInst->getMF()->tryGetCalledGlobal(BranchInst);
4294 if (GV && GV->hasDLLImportStorageClass()) {
4295 auto *CallSiteSymbol = MMI->getContext().createNamedTempSymbol("impcall");
4296 OutStreamer->emitLabel(CallSiteSymbol);
4297
4298 auto *CalledSymbol = MCInstLowering.GetGlobalValueSymbol(GV, OpFlags);
4299 SectionToImportedFunctionCalls[OutStreamer->getCurrentSectionOnly()]
4300 .push_back({CallSiteSymbol, CalledSymbol});
4301 }
4302}
4303
4304void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
4305 MCSymbol *LazyPointer) {
4306 // _ifunc:
4307 // adrp x16, lazy_pointer@GOTPAGE
4308 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
4309 // ldr x16, [x16]
4310 // br x16
4311
4312 {
4313 MCInst Adrp;
4314 Adrp.setOpcode(AArch64::ADRP);
4315 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
4316 MCOperand SymPage;
4317 MCInstLowering.lowerOperand(
4320 SymPage);
4321 Adrp.addOperand(SymPage);
4322 EmitToStreamer(Adrp);
4323 }
4324
4325 {
4326 MCInst Ldr;
4327 Ldr.setOpcode(AArch64::LDRXui);
4328 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
4329 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
4330 MCOperand SymPageOff;
4331 MCInstLowering.lowerOperand(
4334 SymPageOff);
4335 Ldr.addOperand(SymPageOff);
4337 EmitToStreamer(Ldr);
4338 }
4339
4340 EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
4341 .addReg(AArch64::X16)
4342 .addReg(AArch64::X16)
4343 .addImm(0));
4344
4345 EmitToStreamer(MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4346 : AArch64::BR)
4347 .addReg(AArch64::X16));
4348}
4349
4350void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
4351 const GlobalIFunc &GI,
4352 MCSymbol *LazyPointer) {
4353 // These stub helpers are only ever called once, so here we're optimizing for
4354 // minimum size by using the pre-indexed store variants, which saves a few
4355 // bytes of instructions to bump & restore sp.
4356
4357 // _ifunc.stub_helper:
4358 // stp fp, lr, [sp, #-16]!
4359 // mov fp, sp
4360 // stp x1, x0, [sp, #-16]!
4361 // stp x3, x2, [sp, #-16]!
4362 // stp x5, x4, [sp, #-16]!
4363 // stp x7, x6, [sp, #-16]!
4364 // stp d1, d0, [sp, #-16]!
4365 // stp d3, d2, [sp, #-16]!
4366 // stp d5, d4, [sp, #-16]!
4367 // stp d7, d6, [sp, #-16]!
4368 // bl _resolver
4369 // adrp x16, lazy_pointer@GOTPAGE
4370 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
4371 // str x0, [x16]
4372 // mov x16, x0
4373 // ldp d7, d6, [sp], #16
4374 // ldp d5, d4, [sp], #16
4375 // ldp d3, d2, [sp], #16
4376 // ldp d1, d0, [sp], #16
4377 // ldp x7, x6, [sp], #16
4378 // ldp x5, x4, [sp], #16
4379 // ldp x3, x2, [sp], #16
4380 // ldp x1, x0, [sp], #16
4381 // ldp fp, lr, [sp], #16
4382 // br x16
4383
4384 EmitToStreamer(MCInstBuilder(AArch64::STPXpre)
4385 .addReg(AArch64::SP)
4386 .addReg(AArch64::FP)
4387 .addReg(AArch64::LR)
4388 .addReg(AArch64::SP)
4389 .addImm(-2));
4390
4391 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
4392 .addReg(AArch64::FP)
4393 .addReg(AArch64::SP)
4394 .addImm(0)
4395 .addImm(0));
4396
4397 for (int I = 0; I != 4; ++I)
4398 EmitToStreamer(MCInstBuilder(AArch64::STPXpre)
4399 .addReg(AArch64::SP)
4400 .addReg(AArch64::X1 + 2 * I)
4401 .addReg(AArch64::X0 + 2 * I)
4402 .addReg(AArch64::SP)
4403 .addImm(-2));
4404
4405 for (int I = 0; I != 4; ++I)
4406 EmitToStreamer(MCInstBuilder(AArch64::STPDpre)
4407 .addReg(AArch64::SP)
4408 .addReg(AArch64::D1 + 2 * I)
4409 .addReg(AArch64::D0 + 2 * I)
4410 .addReg(AArch64::SP)
4411 .addImm(-2));
4412
4413 EmitToStreamer(
4414 MCInstBuilder(AArch64::BL)
4416
4417 {
4418 MCInst Adrp;
4419 Adrp.setOpcode(AArch64::ADRP);
4420 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
4421 MCOperand SymPage;
4422 MCInstLowering.lowerOperand(
4423 MachineOperand::CreateES(LazyPointer->getName().data() + 1,
4425 SymPage);
4426 Adrp.addOperand(SymPage);
4427 EmitToStreamer(Adrp);
4428 }
4429
4430 {
4431 MCInst Ldr;
4432 Ldr.setOpcode(AArch64::LDRXui);
4433 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
4434 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
4435 MCOperand SymPageOff;
4436 MCInstLowering.lowerOperand(
4437 MachineOperand::CreateES(LazyPointer->getName().data() + 1,
4439 SymPageOff);
4440 Ldr.addOperand(SymPageOff);
4442 EmitToStreamer(Ldr);
4443 }
4444
4445 EmitToStreamer(MCInstBuilder(AArch64::STRXui)
4446 .addReg(AArch64::X0)
4447 .addReg(AArch64::X16)
4448 .addImm(0));
4449
4450 EmitToStreamer(MCInstBuilder(AArch64::ADDXri)
4451 .addReg(AArch64::X16)
4452 .addReg(AArch64::X0)
4453 .addImm(0)
4454 .addImm(0));
4455
4456 for (int I = 3; I != -1; --I)
4457 EmitToStreamer(MCInstBuilder(AArch64::LDPDpost)
4458 .addReg(AArch64::SP)
4459 .addReg(AArch64::D1 + 2 * I)
4460 .addReg(AArch64::D0 + 2 * I)
4461 .addReg(AArch64::SP)
4462 .addImm(2));
4463
4464 for (int I = 3; I != -1; --I)
4465 EmitToStreamer(MCInstBuilder(AArch64::LDPXpost)
4466 .addReg(AArch64::SP)
4467 .addReg(AArch64::X1 + 2 * I)
4468 .addReg(AArch64::X0 + 2 * I)
4469 .addReg(AArch64::SP)
4470 .addImm(2));
4471
4472 EmitToStreamer(MCInstBuilder(AArch64::LDPXpost)
4473 .addReg(AArch64::SP)
4474 .addReg(AArch64::FP)
4475 .addReg(AArch64::LR)
4476 .addReg(AArch64::SP)
4477 .addImm(2));
4478
4479 EmitToStreamer(MCInstBuilder(TM.getTargetTriple().isArm64e() ? AArch64::BRAAZ
4480 : AArch64::BR)
4481 .addReg(AArch64::X16));
4482}
4483
4484const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV,
4485 const Constant *BaseCV,
4486 uint64_t Offset) {
4487 if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
4488 return MCSymbolRefExpr::create(MCInstLowering.GetGlobalValueSymbol(GV, 0),
4489 OutContext);
4490 }
4491
4492 return AsmPrinter::lowerConstant(CV, BaseCV, Offset);
4493}
4494
4495char AArch64AsmPrinter::ID = 0;
4496
4497INITIALIZE_PASS(AArch64AsmPrinter, "aarch64-asm-printer",
4498 "AArch64 Assembly Printer", false, false)
4499
4500// Force static initialization.
4501extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void
4502LLVMInitializeAArch64AsmPrinter() {
4508}
4509
4512 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4513 MAM.getResult<AsmPrinterAnalysis>(M).getPrinter());
4516 return PreservedAnalyses::all();
4517}
4518
4522 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4524 .getCachedResult<AsmPrinterAnalysis>(*MF.getFunction().getParent())
4525 ->getPrinter());
4528 return PreservedAnalyses::all();
4529}
4530
4533 AArch64AsmPrinter &AsmPrinter = static_cast<AArch64AsmPrinter &>(
4534 MAM.getResult<AsmPrinterAnalysis>(M).getPrinter());
4537 return PreservedAnalyses::all();
4538}
#define GET_CB_OPC(IsImm, Width, ImmCond, RegCond)
static void emitAuthenticatedPointer(MCStreamer &OutStreamer, MCSymbol *StubLabel, const MCExpr *StubAuthPtrRef)
static bool getOptionalBooleanModuleFlag(Module &M, StringRef Name)
static bool targetSupportsIRelativeRelocation(const Triple &TT)
static PtrauthCheckMode getCheckMode(const MachineFunction *MF)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static const Function * getParent(const Value *V)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ABI
Definition Compiler.h:211
#define LLVM_EXTERNAL_VISIBILITY
Definition Compiler.h:132
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
print mir2vec MIR2Vec Vocabulary Printer Pass
Definition MIR2Vec.cpp:656
Machine Check Debug Module
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define P(N)
ModuleAnalysisManager MAM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
RISCVCC::CondCode CC
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
This file provides utility classes that use RAII to save and restore values.
This file defines the scope_exit class, which executes user-defined cleanup logic at scope exit.
static bool printOperand(raw_ostream &OS, const SelectionDAG *G, const SDValue Value)
This file defines the SmallString class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
std::unique_ptr< MCStreamer > && Streamer
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static bool printAsmMRegister(const X86AsmPrinter &P, const MachineInstr &MI, const MachineOperand &MO, char Mode, raw_ostream &O)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM)
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
static const AArch64AuthMCExpr * create(const MCExpr *Expr, uint16_t Discriminator, AArch64PACKey::ID Key, bool HasAddressDiversity, MCContext &Ctx, SMLoc Loc=SMLoc())
const SetOfInstructions & getLOHRelated() const
unsigned getJumpTableEntrySize(int Idx) const
MCSymbol * getJumpTableEntryPCRelSymbol(int Idx) const
static bool shouldSignReturnAddress(SignReturnAddress Condition, bool IsLRSpilled)
std::optional< std::string > getOutliningStyle() const
const MILOHContainer & getLOHContainer() const
void setJumpTableEntryInfo(int Idx, unsigned Size, MCSymbol *PCRelSym)
static const char * getRegisterName(MCRegister Reg, unsigned AltIdx=AArch64::NoRegAltName)
static bool isTailCallReturnInst(const MachineInstr &MI)
Returns true if MI is one of the TCRETURN* instructions.
AArch64MCInstLower - This class is used to lower an MachineInstr into an MCInst.
MCSymbol * GetGlobalValueSymbol(const GlobalValue *GV, unsigned TargetFlags) const
void Lower(const MachineInstr *MI, MCInst &OutMI) const
bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const
const AArch64Options & getCLOpts() const
virtual void emitARM64WinCFISaveRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveRegPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQ(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQP(unsigned Reg, int Offset)
virtual void emitAttributesSubsection(StringRef VendorName, AArch64BuildAttributes::SubsectionOptional IsOptional, AArch64BuildAttributes::SubsectionType ParameterType)
Build attributes implementation.
virtual void emitARM64WinCFISavePReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegI(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFIAllocStack(unsigned Size)
virtual void emitARM64WinCFISaveFPLRX(int Offset)
virtual void emitARM64WinCFIAllocZ(int Offset)
virtual void emitAttribute(StringRef VendorName, unsigned Tag, uint64_t Value, std::string String)
virtual void emitDirectiveVariantPCS(MCSymbol *Symbol)
Callback used to implement the .variant_pcs directive.
virtual void emitARM64WinCFIAddFP(unsigned Size)
virtual void emitARM64WinCFISaveFPLR(int Offset)
virtual void emitARM64WinCFISaveFRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveZReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveLRPair(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegIP(unsigned Reg, int Offset)
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
void setPreservesAll()
Set by analyses that do not transform their input at all.
const T & front() const
Get the first element.
Definition ArrayRef.h:144
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
This class is intended to be used as a driving class for all asm writers.
Definition AsmPrinter.h:95
virtual void emitGlobalAlias(const Module &M, const GlobalAlias &GA)
virtual void emitBasicBlockEnd(const MachineBasicBlock &MBB)
Targets can override this to emit stuff at the end of a basic block.
virtual MCSymbol * GetCPISymbol(unsigned CPID) const
Return the symbol for the specified constant pool entry.
virtual const MCExpr * lowerConstant(const Constant *CV, const Constant *BaseCV=nullptr, uint64_t Offset=0)
Lower the specified LLVM Constant to an MCExpr.
bool doInitialization(Module &M) override
Set up the AsmPrinter when we are working on a new module.
void getAnalysisUsage(AnalysisUsage &AU) const override
Record analysis usage.
bool doFinalization(Module &M) override
Shut down the asmprinter.
bool runOnMachineFunction(MachineFunction &MF) override
Emit the specified function out to the OutStreamer.
Definition AsmPrinter.h:478
virtual void emitXXStructor(const DataLayout &DL, const Constant *CV)
Targets can override this to change how global constants that are part of a C++ static/global constru...
Definition AsmPrinter.h:690
virtual void emitFunctionEntryLabel()
EmitFunctionEntryLabel - Emit the label that is the entrypoint for the function.
virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant.
virtual const MCExpr * lowerBlockAddressConstant(const BlockAddress &BA)
Lower the specified BlockAddress to an MCExpr.
Function * getFunction() const
Definition Constants.h:1126
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
static LLVM_ABI ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Constant * getPointer() const
The pointer that is signed in this ptrauth signed pointer.
Definition Constants.h:1251
static LLVM_ABI ConstantPtrAuth * get(Constant *Ptr, ConstantInt *Key, ConstantInt *Disc, Constant *AddrDisc, Constant *DeactivationSymbol)
Return a pointer signed with the specified parameters.
ConstantInt * getKey() const
The Key ID, an i32 constant.
Definition Constants.h:1254
Constant * getDeactivationSymbol() const
Definition Constants.h:1273
bool hasAddressDiscriminator() const
Whether there is any non-null address discriminator.
Definition Constants.h:1269
ConstantInt * getDiscriminator() const
The integer discriminator, an i64 constant, or 0.
Definition Constants.h:1257
LLVM_ABI void recordFaultingOp(FaultKind FaultTy, const MCSymbol *FaultingLabel, const MCSymbol *HandlerLabel)
Definition FaultMaps.cpp:27
LLVM_ABI void serializeToFaultMapSection()
Definition FaultMaps.cpp:44
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
const Constant * getAliasee() const
Definition GlobalAlias.h:87
const Constant * getResolver() const
Definition GlobalIFunc.h:73
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this GlobalObject.
bool hasLocalLinkage() const
bool hasExternalWeakLinkage() const
Type * getValueType() const
static LLVM_ABI IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition Type.cpp:333
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
static const MCBinaryExpr * createLShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition MCExpr.h:422
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc=SMLoc())
Definition MCExpr.h:342
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition MCExpr.h:427
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Definition MCExpr.cpp:212
LLVM_ABI MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
MCSectionELF * getELFSection(const Twine &Section, unsigned Type, unsigned Flags)
Definition MCContext.h:552
LLVM_ABI MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
LLVM_ABI MCSymbol * createLinkerPrivateSymbol(const Twine &Name)
Base class for the full range of assembler expressions which are needed for parsing.
Definition MCExpr.h:34
LLVM_ABI bool evaluateAsRelocatable(MCValue &Res, const MCAssembler *Asm) const
Try to evaluate the expression to a relocatable value, i.e.
Definition MCExpr.cpp:450
void setFlags(unsigned F)
Definition MCInst.h:204
void addOperand(const MCOperand Op)
Definition MCInst.h:215
void setOpcode(unsigned Op)
Definition MCInst.h:201
MCSection * getDataSection() const
void setImm(int64_t Val)
Definition MCInst.h:89
static MCOperand createExpr(const MCExpr *Val)
Definition MCInst.h:166
int64_t getImm() const
Definition MCInst.h:84
static MCOperand createReg(MCRegister Reg)
Definition MCInst.h:138
static MCOperand createImm(int64_t Val)
Definition MCInst.h:145
bool isReg() const
Definition MCInst.h:65
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
uint16_t getEncodingValue(MCRegister Reg) const
Returns the encoding for Reg.
static constexpr unsigned NonUniqueID
Definition MCSection.h:585
static const MCSpecifierExpr * create(const MCExpr *Expr, Spec S, MCContext &Ctx, SMLoc Loc=SMLoc())
Definition MCExpr.cpp:743
Streaming machine code generation interface.
Definition MCStreamer.h:222
virtual void emitCFIBKeyFrame()
virtual bool popSection()
Restore the current and previous section from the section stack.
virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI)
Emit the given Instruction into the current section.
virtual void emitRelocDirective(const MCExpr &Offset, StringRef Name, const MCExpr *Expr, SMLoc Loc={})
Record a relocation described by the .reloc directive.
virtual bool hasRawTextSupport() const
Return true if this asm streamer supports emitting unformatted text to the .s file with EmitRawText.
Definition MCStreamer.h:389
MCContext & getContext() const
Definition MCStreamer.h:330
virtual void AddComment(const Twine &T, bool EOL=true)
Add a textual comment.
Definition MCStreamer.h:408
virtual void emitCFIMTETaggedFrame()
virtual void emitWinCFIStartProc(const MCSymbol *Symbol, SMLoc Loc=SMLoc())
void emitValue(const MCExpr *Value, unsigned Size, SMLoc Loc=SMLoc())
virtual void emitLabel(MCSymbol *Symbol, SMLoc Loc=SMLoc())
Emit a label for Symbol into the current section.
MCTargetStreamer * getTargetStreamer()
Definition MCStreamer.h:340
virtual void emitWinCFIEndProc(SMLoc Loc=SMLoc())
virtual void emitWinCFIFuncletOrFuncEnd(SMLoc Loc=SMLoc())
This is used on platforms, such as Windows on ARM64, that require function or funclet sizes to be emi...
void pushSection()
Save the current and previous section on the section stack.
Definition MCStreamer.h:464
virtual void switchSection(MCSection *Section, uint32_t Subsec=0)
Set the current section where code is being emitted to Section.
MCSection * getCurrentSectionOnly() const
Definition MCStreamer.h:442
void emitRawText(const Twine &String)
If this file is backed by a assembly streamer, this dumps the specified string in the output ....
virtual void emitCFINonHintPAuthFrame()
const FeatureBitset & getFeatureBits() const
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx, SMLoc Loc=SMLoc())
Definition MCExpr.h:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition MCSymbol.h:42
LLVM_ABI void print(raw_ostream &OS, const MCAsmInfo *MAI) const
print - Print the value to the stream OS.
Definition MCSymbol.cpp:60
StringRef getName() const
getName - Get the symbol name.
Definition MCSymbol.h:188
const MCSymbol * getAddSym() const
Definition MCValue.h:49
int64_t getConstant() const
Definition MCValue.h:44
MachineInstrBundleIterator< const MachineInstr > const_iterator
LLVM_ABI MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
CalledGlobalInfo tryGetCalledGlobal(const MachineInstr *MI) const
Tries to get the global and target flags for a call site, if the instruction is a call to a global.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MCContext & getContext() const
Function & getFunction()
Return the LLVM function that this machine code represents.
bool usesWindowsUnwindInfo() const
True if this function describes its unwind information with Windows SEH unwind opcodes (....
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr reads the specified register.
mop_range operands()
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
bool isBundledWithSucc() const
Return true if this instruction is part of a bundle, and it is not the last instruction in the bundle...
const MachineOperand & getOperand(unsigned i) const
const std::vector< MachineJumpTableEntry > & getJumpTables() const
unsigned getSubReg() const
static MachineOperand CreateMCSymbol(MCSymbol *Sym, unsigned TargetFlags=0)
const GlobalValue * getGlobal() const
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
const BlockAddress * getBlockAddress() const
void setOffset(int64_t Offset)
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
@ MO_Immediate
Immediate operand.
@ MO_GlobalAddress
Address of a global value.
@ MO_BlockAddress
Address of a basic block.
@ MO_Register
Register operand.
@ MO_ExternalSymbol
Name of external global symbol.
int64_t getOffset() const
Return the offset from the symbol in this operand.
This class implements a map that also provides access to all stored values in a deterministic order.
Definition MapVector.h:39
Root of the metadata hierarchy.
Definition Metadata.h:64
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
static SectionKind getMetadata()
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
void push_back(const T &Elt)
LLVM_ABI void recordStatepoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a statepoint instruction.
LLVM_ABI void recordPatchPoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a patchpoint instruction.
LLVM_ABI void recordStackMap(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a stackmap instruction.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
virtual MCSection * getSectionForJumpTable(const Function &F, const TargetMachine &TM) const
Primary interface to the complete machine description for the target machine.
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
Triple - Helper class for working with autoconf configuration names.
Definition Triple.h:48
bool isFunctionTy() const
True if this is an instance of FunctionType.
Definition Type.h:268
LLVM Value Representation.
Definition Value.h:75
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
LLVM_ABI const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr, bool LookThroughIntToPtr=false) const
Accumulate the constant offset this value has compared to a base pointer.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
self_iterator getIterator()
Definition ilist_node.h:123
Pass manager infrastructure for declaring and invalidating analyses.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI StringRef getVendorName(unsigned const Vendor)
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_S
MO_S - Indicates that the bits of the symbol operand represented by MO_G0 etc are signed.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
PtrauthCheckMode
Ways to check pointer authentication auth/resign failures.
constexpr AArch64PACKey::ID InitFiniKey
PAuth key to be used with function pointers in .init_array and .fini_array.
AuthCheckMethod
Variants of check performed on an authenticated pointer.
constexpr unsigned InitFiniPointerConstantDiscriminator
Constant discriminator to be used with function pointers in .init_array and .fini_array.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
@ SectionSize
Definition COFF.h:61
SymbolStorageClass
Storage class tells where and what the symbol represents.
Definition COFF.h:218
@ IMAGE_SYM_CLASS_EXTERNAL
External symbol.
Definition COFF.h:224
@ IMAGE_SYM_CLASS_STATIC
Static.
Definition COFF.h:225
@ IMAGE_SYM_DTYPE_FUNCTION
A function that returns a base type.
Definition COFF.h:276
@ SCT_COMPLEX_TYPE_SHIFT
Type is formed as (base + (derived << SCT_COMPLEX_TYPE_SHIFT))
Definition COFF.h:280
@ GNU_PROPERTY_AARCH64_FEATURE_1_BTI
Definition ELF.h:1716
@ GNU_PROPERTY_AARCH64_FEATURE_1_PAC
Definition ELF.h:1717
@ GNU_PROPERTY_AARCH64_FEATURE_1_GCS
Definition ELF.h:1718
@ SHF_ALLOC
Definition ELF.h:1098
@ SHF_GROUP
Definition ELF.h:1120
@ SHF_EXECINSTR
Definition ELF.h:1101
@ AARCH64_PAUTH_PLATFORM_LLVM_LINUX
Definition ELF.h:1725
@ SHT_PROGBITS
Definition ELF.h:996
@ S_REGULAR
S_REGULAR - Regular section.
Definition MachO.h:127
void emitInstruction(MCObjectStreamer &, const MCInst &Inst, const MCSubtargetInfo &STI)
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)
Extract a Value from Metadata, allowing null.
Definition Metadata.h:694
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:679
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
bool empty() const
Definition BasicBlock.h:101
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
LLVM_ABI std::optional< std::string > getArm64ECMangledFunctionName(StringRef Name)
Returns the ARM64EC mangled function name unless the input is already mangled.
Definition Mangler.cpp:292
SaveAndRestore(T &) -> SaveAndRestore< T >
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
OuterAnalysisManagerProxy< ModuleAnalysisManager, MachineFunction > ModuleAnalysisManagerMachineFunctionProxy
Provide the ModuleAnalysisManager to Function proxy.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
scope_exit(Callable) -> scope_exit< Callable >
@ Store
The extracted value is stored (ExtractElement only).
static unsigned getXPACOpcodeForKey(AArch64PACKey::ID K)
Return XPAC opcode to be used for a ptrauth strip using the given key.
Target & getTheAArch64beTarget()
std::string utostr(uint64_t X, bool isNeg=false)
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
static unsigned getBranchOpcodeForKey(bool IsCall, AArch64PACKey::ID K, bool Zero)
Return B(L)RA opcode to be used for an authenticated branch or call using the given key,...
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
Target & getTheAArch64leTarget()
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
Target & getTheAArch64_32Target()
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
LLVM_ABI void setupModuleAsmPrinter(Module &M, ModuleAnalysisManager &MAM, AsmPrinter &AsmPrinter)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
Target & getTheARM64_32Target()
static MCRegister getXRegFromWReg(MCRegister Reg)
@ Add
Sum of integers.
Target & getTheARM64Target()
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
static MCRegister getXRegFromXRegTuple(MCRegister RegTuple)
static unsigned getPACOpcodeForKey(AArch64PACKey::ID K, bool Zero)
Return PAC opcode to be used for a ptrauth sign using the given key, or its PAC*Z variant that doesn'...
static MCRegister getWRegFromXReg(MCRegister Reg)
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1933
ExceptionHandling
Definition CodeGen.h:54
LLVM_ABI void setupMachineFunctionAsmPrinter(MachineFunctionAnalysisManager &MFAM, MachineFunction &MF, AsmPrinter &AsmPrinter)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
static unsigned getAUTOpcodeForKey(AArch64PACKey::ID K, bool Zero)
Return AUT opcode to be used for a ptrauth auth using the given key, or its AUT*Z variant that doesn'...
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
Definition MathExtras.h:199
@ MCSA_Weak
.weak
@ MCSA_WeakAntiDep
.weak_anti_dep (COFF)
@ MCSA_ELF_TypeFunction
.type _foo, STT_FUNC # aka @function
@ MCSA_Hidden
.hidden (ELF)
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
Definition MIRParser.h:39
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:879
#define EQ(a, b)
Definition regexec.c:65
RegisterAsmPrinter - Helper template for registering a target specific assembly printer,...