LLVM 24.0.0git
X86MCLFIRewriter.cpp
Go to the documentation of this file.
1//===- X86MCLFIRewriter.cpp -------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the X86MCLFIRewriter class, which rewrites X86-64
10// instructions for LFI (Lightweight Fault Isolation) sandboxing.
11//
12//===----------------------------------------------------------------------===//
13
14#include "X86MCLFIRewriter.h"
15#include "X86BaseInfo.h"
16#include "X86MCTargetDesc.h"
17#include "llvm/MC/MCContext.h"
18#include "llvm/MC/MCExpr.h"
19#include "llvm/MC/MCInst.h"
21#include "llvm/MC/MCStreamer.h"
23#include <algorithm>
24#include <optional>
25
26using namespace llvm;
27
28// LFI reserved registers.
29static constexpr MCRegister LFIBaseReg = X86::R14;
30static constexpr MCRegister LFIScratchReg = X86::R11;
31static constexpr MCRegister LFICtxReg = X86::R15;
32
33// Segment register whose base always holds the sandbox base address.
34static constexpr MCRegister LFIBaseSeg = X86::GS;
35
36// Byte offset into the context register file (pointed to by R15) where the
37// thread pointer is stored.
38static constexpr int TPOffset = 16;
39
40// Byte offset from the context register where the syscall handler address is
41// stored.
42static constexpr int SyscallOffset = -8;
43
44static bool isPrefix(const MCInst &Inst, const MCInstrInfo &InstInfo) {
45 return X86II::isPrefix(InstInfo.get(Inst.getOpcode()).TSFlags);
46}
47
48// Return true if Inst is a supported stand-alone prefix.
49static bool isSupportedPrefix(const MCInst &Inst) {
50 switch (Inst.getOpcode()) {
51 case X86::FS_PREFIX:
52 case X86::GS_PREFIX:
53 case X86::ADDR16_PREFIX:
54 case X86::ADDR32_PREFIX:
55 return false;
56 default:
57 return true;
58 }
59}
60
61static bool isSyscall(const MCInst &Inst) {
62 return Inst.getOpcode() == X86::SYSCALL;
63}
64
65static bool isDirectCall(const MCInst &Inst) {
66 return Inst.getOpcode() == X86::CALL64pcrel32;
67}
68
69static bool isSupportedIndirectBranch(const MCInst &Inst) {
70 switch (Inst.getOpcode()) {
71 case X86::JMP64r:
72 case X86::JMP64r_NT:
73 case X86::JMP64m:
74 case X86::JMP64m_NT:
75 case X86::CALL64r:
76 case X86::CALL64r_NT:
77 case X86::CALL64m:
78 case X86::CALL64m_NT:
79 return true;
80 default:
81 return false;
82 }
83}
84
85static bool isWriteSegmentBase(unsigned Opcode) {
86 switch (Opcode) {
87 case X86::WRFSBASE:
88 case X86::WRFSBASE64:
89 case X86::WRGSBASE:
90 case X86::WRGSBASE64:
91 return true;
92 default:
93 return false;
94 }
95}
96
97static bool hasNoTrackPrefix(const MCInst &Inst, const MCInstrInfo &InstInfo) {
98 return (InstInfo.get(Inst.getOpcode()).TSFlags & X86II::NOTRACK) ||
100}
101
102// Find the index of the memory operand if it has an %fs segment override.
103// Returns -1 if there is no memory operand or no %fs override.
104static int findFSMemOperand(const MCInst &Inst, const MCInstrInfo &InstInfo) {
105 int MemIdx = X86II::getMemoryOperandIdx(InstInfo.get(Inst.getOpcode()));
106 if (MemIdx < 0)
107 return -1;
108 const MCOperand &Seg = Inst.getOperand(MemIdx + X86::AddrSegmentReg);
109 if (Seg.isReg() && Seg.getReg() == X86::FS)
110 return MemIdx;
111 return -1;
112}
113
114// Return true if the instruction reads from Reg.
115static bool readsRegister(const MCInst &Inst, const MCInstrDesc &Desc,
116 MCRegister Reg, const MCRegisterInfo &RI) {
117 for (unsigned I = Desc.getNumDefs(), E = Inst.getNumOperands(); I < E; ++I) {
118 const MCOperand &Op = Inst.getOperand(I);
119 if (Op.isReg() && Op.getReg() && RI.regsOverlap(Op.getReg(), Reg))
120 return true;
121 }
122 for (MCPhysReg Use : Desc.implicit_uses())
123 if (RI.regsOverlap(Use, Reg))
124 return true;
125 return false;
126}
127
128// Return true if Reg is absent or a 64-bit general-purpose register.
130 return Reg == X86::NoRegister ||
131 getX86MCRegisterClass(X86::GR64RegClassID).contains(Reg);
132}
133
134MCRegister X86::X86MCLFIRewriter::getReg32(MCRegister Reg) const {
135 if (getX86MCRegisterClass(X86::GR32RegClassID).contains(Reg))
136 return Reg;
137 return RegInfo->getSubReg(Reg, X86::sub_32bit);
138}
139
140void X86::X86MCLFIRewriter::emitWithPrefixes(const MCInst &Inst,
141 MCStreamer &Out,
142 const MCSubtargetInfo &STI) {
143 for (const MCInst &Prefix : Prefixes)
144 Out.emitInstruction(Prefix, STI);
145 Prefixes.clear();
146 Out.emitInstruction(Inst, STI);
147}
148
149void X86::X86MCLFIRewriter::discardPrefixes() {
150 for (const MCInst &Prefix : Prefixes)
151 error(Prefix, "unsupported instruction prefix");
152 Prefixes.clear();
153}
154
155// syscall
156// ->
157// .bundle_lock
158// leaq .Ltmp(%rip), %r11
159// jmpq *-8(%r15)
160// .Ltmp:
161// .bundle_unlock
162void X86::X86MCLFIRewriter::rewriteSyscall(const MCInst &Inst, MCStreamer &Out,
163 const MCSubtargetInfo &STI) {
164 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
165
166 MCSymbol *Symbol = Out.getContext().createTempSymbol();
167
168 // leaq .Ltmp(%rip), %r11
169 Out.emitInstruction(
170 MCInstBuilder(X86::LEA64r)
171 .addReg(LFIScratchReg)
172 .addReg(X86::RIP)
173 .addImm(1)
174 .addReg(X86::NoRegister)
175 .addExpr(MCSymbolRefExpr::create(Symbol, Out.getContext()))
176 .addReg(X86::NoRegister),
177 STI);
178
179 // jmpq *-8(%r15)
180 Out.emitInstruction(MCInstBuilder(X86::JMP64m)
181 .addReg(LFICtxReg)
182 .addImm(1)
183 .addReg(X86::NoRegister)
184 .addImm(SyscallOffset)
185 .addReg(X86::NoRegister),
186 STI);
187
188 Out.emitLabel(Symbol);
189 Out.emitBundleUnlock(STI);
190}
191
192// andl $-LFIBundleSize, %eX
193// addq %r14, %rX
194void X86::X86MCLFIRewriter::emitSandboxBranchReg(MCRegister Reg,
195 MCStreamer &Out,
196 const MCSubtargetInfo &STI) {
197 MCRegister Reg32 = RegInfo->getSubReg(Reg, X86::sub_32bit);
198
199 Out.emitInstruction(MCInstBuilder(X86::AND32ri8)
200 .addReg(Reg32)
201 .addReg(Reg32)
202 .addImm(-static_cast<int64_t>(LFIBundleSize)),
203 STI);
204
205 Out.emitInstruction(
206 MCInstBuilder(X86::ADD64rr).addReg(Reg).addReg(Reg).addReg(LFIBaseReg),
207 STI);
208}
209
210// Rewrite an indirect jump or call so that it can only target a bundle
211// boundary inside the sandbox.
212//
213// jmpq *%rX
214// ->
215// .bundle_lock
216// andl $-32, %eX
217// addq %r14, %rX
218// jmpq *%rX
219// .bundle_unlock
220//
221// A branch through memory loads its target into the scratch register first,
222// and then dispatches through it.
223//
224// jmpq *(%rdi)
225// ->
226// movq %gs:(%edi), %r11
227// .bundle_lock
228// andl $-32, %r11d
229// addq %r14, %r11
230// jmpq *%r11
231// .bundle_unlock
232void X86::X86MCLFIRewriter::rewriteIndirectBranch(const MCInst &Inst,
233 MCStreamer &Out,
234 const MCSubtargetInfo &STI) {
235 discardPrefixes();
236
237 MCRegister Target;
238 int MemIdx = X86II::getMemoryOperandIdx(InstInfo->get(Inst.getOpcode()));
239 if (MemIdx >= 0) {
241
242 // Construct the load and then apply the rewriter to it.
243 MCInstBuilder Mov(X86::MOV64rm);
244 Mov.addReg(Target);
245 for (unsigned I = 0; I < X86::AddrNumOperands; ++I)
246 Mov.addOperand(Inst.getOperand(MemIdx + I));
247 Mov.setLoc(Inst.getLoc());
248 doRewriteInst(Mov, Out, STI);
249 } else {
250 Target = Inst.getOperand(0).getReg();
251
252 if (Target == LFIBaseReg || Target == LFICtxReg || Target == X86::RSP)
253 return error(Inst, "indirect branch through reserved register");
254 }
255
256 Out.emitBundleLock(/*AlignToEnd=*/isCall(Inst), STI);
257
258 emitSandboxBranchReg(Target, Out, STI);
259
260 MCInst Branch =
261 MCInstBuilder(isCall(Inst) ? X86::CALL64r : X86::JMP64r).addReg(Target);
262 if (hasNoTrackPrefix(Inst, *InstInfo))
263 Branch.setFlags(Branch.getFlags() | X86::IP_HAS_NOTRACK);
264 Out.emitInstruction(Branch, STI);
265
266 Out.emitBundleUnlock(STI);
267}
268
269// Direct calls are not rewritten, but must be placed at the end of a bundle
270// so that the return address they push is bundle-aligned.
271void X86::X86MCLFIRewriter::rewriteDirectCall(const MCInst &Inst,
272 MCStreamer &Out,
273 const MCSubtargetInfo &STI) {
274 Out.emitBundleLock(/*AlignToEnd=*/true, STI);
275 emitWithPrefixes(Inst, Out, STI);
276 Out.emitBundleUnlock(STI);
277}
278
279// ret
280// ->
281// popq %r11
282// .bundle_lock
283// andl $-32, %r11d
284// addq %r14, %r11
285// jmpq *%r11
286// .bundle_unlock
287//
288// A return with an immediate additionally pops the immediate off the stack
289// after loading the return address.
290//
291// retq $16
292// ->
293// popq %r11
294// addq $16, %rsp
295// .bundle_lock
296// andl $-32, %r11d
297// addq %r14, %r11
298// jmpq *%r11
299// .bundle_unlock
300void X86::X86MCLFIRewriter::rewriteReturn(const MCInst &Inst, MCStreamer &Out,
301 const MCSubtargetInfo &STI) {
302 if (Inst.getOpcode() != X86::RET64 && Inst.getOpcode() != X86::RETI64)
303 return error(Inst, "unsupported return instruction");
304
305 discardPrefixes();
306
307 Out.emitInstruction(MCInstBuilder(X86::POP64r).addReg(LFIScratchReg), STI);
308
309 if (Inst.getOpcode() == X86::RETI64) {
310 // Return with an immediate is rewritten recursively so that the stack
311 // pointer modification goes through the rewriter.
312 doRewriteInst(MCInstBuilder(X86::ADD64ri32)
313 .addReg(X86::RSP)
314 .addReg(X86::RSP)
315 .addOperand(Inst.getOperand(0))
316 .setLoc(Inst.getLoc()),
317 Out, STI);
318 }
319
320 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
321
322 emitSandboxBranchReg(LFIScratchReg, Out, STI);
323
324 Out.emitInstruction(MCInstBuilder(X86::JMP64r).addReg(LFIScratchReg), STI);
325
326 Out.emitBundleUnlock(STI);
327}
328
329// Emit: movq TPOffset(%r15), %Reg
331 const MCSubtargetInfo &STI) {
332 Out.emitInstruction(MCInstBuilder(X86::MOV64rm)
333 .addReg(Reg)
334 .addReg(LFICtxReg)
335 .addImm(1)
336 .addReg(X86::NoRegister)
337 .addImm(TPOffset)
338 .addReg(X86::NoRegister),
339 STI);
340}
341
342bool X86::X86MCLFIRewriter::isFSAccess(const MCInst &Inst) {
343 return (mayLoad(Inst) || mayStore(Inst)) &&
344 findFSMemOperand(Inst, *InstInfo) >= 0;
345}
346
347// Rewrite %fs-segment memory accesses to use the virtual thread pointer stored
348// at TPOffset(%r15). The rewritten access is fed back through the rewriter so
349// that it is sandboxed like other memory accesses. Example rewrites:
350//
351// movq %fs:0, %rax
352// ->
353// movq 16(%r15), %rax
354//
355// movq %fs:(%rdi), %rax
356// ->
357// movq 16(%r15), %rax
358// movq %gs:(%eax,%edi), %rax
359//
360// movq %fs:8(%rdi, %rsi, 2), %rax
361// ->
362// movq 16(%r15), %rax
363// leaq (%rax, %rdi), %rax
364// movq %gs:8(%eax, %esi, 2), %rax
365void X86::X86MCLFIRewriter::rewriteFSAccess(const MCInst &Inst, MCStreamer &Out,
366 const MCSubtargetInfo &STI) {
367 int MemIdx = findFSMemOperand(Inst, *InstInfo);
368 assert(MemIdx >= 0 && "expected an %fs memory operand");
369
370 MCRegister BaseReg = Inst.getOperand(MemIdx + X86::AddrBaseReg).getReg();
371 MCRegister IndexReg = Inst.getOperand(MemIdx + X86::AddrIndexReg).getReg();
372 bool HasBase = BaseReg != X86::NoRegister;
373 bool HasIndex = IndexReg != X86::NoRegister;
374 bool HasDisp = !Inst.getOperand(MemIdx + X86::AddrDisp).isImm() ||
375 Inst.getOperand(MemIdx + X86::AddrDisp).getImm() != 0;
376
377 // %fs:0 -> TPOffset(%r15). The context register always points into the
378 // runtime's thread-local block, so this access is safe.
379 if (!HasBase && !HasIndex && !HasDisp) {
380 MCInst Modified(Inst);
381 Modified.getOperand(MemIdx + X86::AddrBaseReg).setReg(LFICtxReg);
382 Modified.getOperand(MemIdx + X86::AddrDisp).setImm(TPOffset);
383 Modified.getOperand(MemIdx + X86::AddrSegmentReg).setReg(X86::NoRegister);
384 return emitWithPrefixes(Modified, Out, STI);
385 }
386
387 if (!isGR64OrNone(BaseReg) || !isGR64OrNone(IndexReg) ||
388 BaseReg == X86::RSP || BaseReg == X86::RIP)
389 return error(Inst, "unsupported addressing mode for %fs access");
390
391 const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode());
392
393 // Reuse operand 0 as the TP temporary when the instruction writes it without
394 // also reading it, otherwise use %r11.
395 MCRegister TPDest = LFIScratchReg;
396 if (MemIdx > 0 && Inst.getOperand(0).isReg()) {
397 MCRegister DestReg = Inst.getOperand(0).getReg();
398 if (Desc.getNumDefs() > 0 &&
399 getX86MCRegisterClass(X86::GR64RegClassID).contains(DestReg) &&
400 !readsRegister(Inst, Desc, DestReg, *RegInfo))
401 TPDest = DestReg;
402 }
403
404 if (TPDest == LFIScratchReg &&
405 readsRegister(Inst, Desc, LFIScratchReg, *RegInfo))
406 return error(Inst, "%fs access reads reserved register %r11");
407
408 emitTPLoad(TPDest, Out, STI);
409
410 // Both slots occupied: the compute base via lea. For example:
411 //
412 // movq %fs:8(%rdi,%rsi,2), %rax
413 // ->
414 // movq 16(%r15), %rax
415 // leaq (%rax,%rdi), %rax
416 // movq %gs:8(%eax,%esi,2), %rax
417 if (HasBase && HasIndex) {
418 Out.emitInstruction(MCInstBuilder(X86::LEA64r)
419 .addReg(TPDest)
420 .addReg(TPDest)
421 .addImm(1)
422 .addReg(BaseReg)
423 .addImm(0)
424 .addReg(X86::NoRegister),
425 STI);
426 }
427
428 // Emit the access with TPDest as the new base, and the original base
429 // (offset from %fs) as the new index. For example:
430 //
431 // movq %fs:(%rdi), %rax
432 // ->
433 // movq 16(%r15), %rax
434 // movq %gs:(%eax,%edi), %rax
435 MCInst Modified(Inst);
436 Modified.getOperand(MemIdx + X86::AddrBaseReg).setReg(TPDest);
437 if (HasBase && !HasIndex)
438 Modified.getOperand(MemIdx + X86::AddrIndexReg).setReg(BaseReg);
439 Modified.getOperand(MemIdx + X86::AddrSegmentReg).setReg(X86::NoRegister);
440 doRewriteInst(Modified, Out, STI);
441}
442
443// Return true if Reg always contains an address within the sandbox.
445 return Reg == LFIBaseReg || Reg == X86::RSP || Reg == X86::RIP;
446}
447
448// Return true if Inst reads or writes memory through a pointer register
449// instead of an addressing mode, and so cannot be sandboxed by rewriting its
450// operands.
451static bool hasImplicitMemPointer(unsigned Opcode) {
452 switch (Opcode) {
453 // xlat loads from (%rbx,%al).
454 case X86::XLAT:
455 // maskmov stores to (%rdi).
456 case X86::MASKMOVDQU:
457 case X86::MASKMOVDQU64:
458 case X86::VMASKMOVDQU:
459 case X86::VMASKMOVDQU64:
460 case X86::MMX_MASKMOVQ:
461 case X86::MMX_MASKMOVQ64:
462 // clzero zeroes the cache line at (%rax).
463 case X86::CLZERO32r:
464 case X86::CLZERO64r:
465 // movdir64b and enqcmd store to the address in their register operand.
466 case X86::MOVDIR64B32:
467 case X86::MOVDIR64B64:
468 case X86::MOVDIR64B32_EVEX:
469 case X86::MOVDIR64B64_EVEX:
470 case X86::ENQCMD32:
471 case X86::ENQCMD64:
472 case X86::ENQCMD32_EVEX:
473 case X86::ENQCMD64_EVEX:
474 case X86::ENQCMDS32:
475 case X86::ENQCMDS64:
476 case X86::ENQCMDS32_EVEX:
477 case X86::ENQCMDS64_EVEX:
478 return true;
479 default:
480 return false;
481 }
482}
483
484// Return true if Inst is a bit test of memory with a register bit offset. The
485// offset is added to the address of the memory operand, so the bit it accesses
486// may be beyond the address computed by the addressing mode.
487static bool isBitTestWithRegOffset(unsigned Opcode) {
488 switch (Opcode) {
489 case X86::BT16mr:
490 case X86::BT32mr:
491 case X86::BT64mr:
492 case X86::BTC16mr:
493 case X86::BTC32mr:
494 case X86::BTC64mr:
495 case X86::BTR16mr:
496 case X86::BTR32mr:
497 case X86::BTR64mr:
498 case X86::BTS16mr:
499 case X86::BTS32mr:
500 case X86::BTS64mr:
501 case X86::LOCK_BTC_RM16rm:
502 case X86::LOCK_BTC_RM32rm:
503 case X86::LOCK_BTC_RM64rm:
504 case X86::LOCK_BTR_RM16rm:
505 case X86::LOCK_BTR_RM32rm:
506 case X86::LOCK_BTR_RM64rm:
507 case X86::LOCK_BTS_RM16rm:
508 case X86::LOCK_BTS_RM32rm:
509 case X86::LOCK_BTS_RM64rm:
510 return true;
511 default:
512 return false;
513 }
514}
515
516// Return true if Inst has a memory operand that is not dereferenced.
517static bool ignoresMemOperand(unsigned Opcode) {
518 switch (Opcode) {
519 case X86::LEA16r:
520 case X86::LEA32r:
521 case X86::LEA64r:
522 case X86::LEA64_32r:
523 case X86::NOOPW:
524 case X86::NOOPL:
525 case X86::NOOPQ:
526 return true;
527 default:
528 return false;
529 }
530}
531
532// Return true if the index register of the memory operand at MemIdx is a
533// general-purpose register, and so can be truncated along with the base.
534static bool hasGPRIndex(const MCInst &Inst, int MemIdx) {
535 MCRegister Index = Inst.getOperand(MemIdx + X86::AddrIndexReg).getReg();
536 return Index && getX86MCRegisterClass(X86::GR64RegClassID).contains(Index);
537}
538
539// Return true if the memory operand at MemIdx uses a 32-bit address size,
540// which is selected by a 32-bit base or index register.
541static bool has32BitAddress(const MCInst &Inst, int MemIdx) {
543 MCRegister Index = Inst.getOperand(MemIdx + X86::AddrIndexReg).getReg();
544 const MCRegisterClass &GR32 = getX86MCRegisterClass(X86::GR32RegClassID);
545 return Base == X86::EIP || GR32.contains(Base) || Index == X86::EIZ ||
546 GR32.contains(Index);
547}
548
549// Rewrite the addressing mode of the memory operand at MemIdx so that the
550// access it performs stays inside the sandbox.
551//
552// movq 8(%rax,%rcx,4), %rdi
553// ->
554// movq %gs:8(%eax,%ecx,4), %rdi
555void X86::X86MCLFIRewriter::sandboxMemOperand(MCInst &Inst, int MemIdx) const {
556 MCOperand &Base = Inst.getOperand(MemIdx + X86::AddrBaseReg);
557 MCOperand &Scale = Inst.getOperand(MemIdx + X86::AddrScaleAmt);
558 MCOperand &Index = Inst.getOperand(MemIdx + X86::AddrIndexReg);
559 MCOperand &Segment = Inst.getOperand(MemIdx + X86::AddrSegmentReg);
560
561 // %riz only encodes the absence of an index register, so drop it and handle
562 // the access as one without an index.
563 if (Index.getReg() == X86::RIZ) {
564 Index.setReg(X86::NoRegister);
565 Scale.setImm(1);
566 }
567
568 // A base register that always points into the sandbox needs no rewriting, as
569 // long as there is no index register.
570 if (isSandboxBaseReg(Base.getReg()) && !Index.getReg())
571 return;
572
573 // An absolute address has no register to truncate, so make it relative to
574 // the sandbox base instead.
575 if (!Base.getReg() && !Index.getReg()) {
576 Base.setReg(LFIBaseReg);
577 return;
578 }
579
580 // If there is no Base and the Index is not a GPR, then we use r14d (all
581 // zeroes) as the base for the rewritten addressing mode.
582 bool GPRIndex = hasGPRIndex(Inst, MemIdx);
583
584 Segment.setReg(LFIBaseSeg);
585 if (Base.getReg())
586 Base.setReg(getReg32(Base.getReg()));
587 else if (!GPRIndex)
588 Base.setReg(getReg32(LFIBaseReg));
589 if (GPRIndex)
590 Index.setReg(getReg32(Index.getReg()));
591}
592
593void X86::X86MCLFIRewriter::sandboxMemAccess(MCInst &Inst) const {
594 if (ignoresMemOperand(Inst.getOpcode()))
595 return;
596
597 int MemIdx = X86II::getMemoryOperandIdx(InstInfo->get(Inst.getOpcode()));
598 if (MemIdx >= 0)
599 sandboxMemOperand(Inst, MemIdx);
600}
601
602void X86::X86MCLFIRewriter::rewriteMemAccess(const MCInst &Inst,
603 MCStreamer &Out,
604 const MCSubtargetInfo &STI) {
605 MCInst Sandboxed(Inst);
606 sandboxMemAccess(Sandboxed);
607 emitWithPrefixes(Sandboxed, Out, STI);
608}
609
610namespace {
611struct StringOpPointers {
612 bool UsesDest;
613 bool UsesSrc;
614};
615} // namespace
616
617static std::optional<StringOpPointers> getStringOpPointers(unsigned Opcode) {
618 switch (Opcode) {
619 case X86::STOSB:
620 case X86::STOSW:
621 case X86::STOSL:
622 case X86::STOSQ:
623 case X86::REP_STOSB_64:
624 case X86::REP_STOSW_64:
625 case X86::REP_STOSD_64:
626 case X86::REP_STOSQ_64:
627 case X86::SCASB:
628 case X86::SCASW:
629 case X86::SCASL:
630 case X86::SCASQ:
631 return StringOpPointers{true, false};
632 case X86::MOVSB:
633 case X86::MOVSW:
634 case X86::MOVSL:
635 case X86::MOVSQ:
636 case X86::REP_MOVSB_64:
637 case X86::REP_MOVSW_64:
638 case X86::REP_MOVSD_64:
639 case X86::REP_MOVSQ_64:
640 case X86::CMPSB:
641 case X86::CMPSW:
642 case X86::CMPSL:
643 case X86::CMPSQ:
644 return StringOpPointers{true, true};
645 case X86::LODSB:
646 case X86::LODSW:
647 case X86::LODSL:
648 case X86::LODSQ:
649 return StringOpPointers{false, true};
650 default:
651 return std::nullopt;
652 }
653}
654
656 MCStreamer &Out, const MCSubtargetInfo &STI) {
657 Out.emitInstruction(MCInstBuilder(X86::MOV32rr).addReg(Reg32).addReg(Reg32),
658 STI);
659
660 Out.emitInstruction(MCInstBuilder(X86::LEA64r)
661 .addReg(Reg)
662 .addReg(LFIBaseReg)
663 .addImm(1)
664 .addReg(Reg)
665 .addImm(0)
666 .addReg(X86::NoRegister),
667 STI);
668}
669
670// rep movsq
671// ->
672// .bundle_lock
673// movl %edi, %edi
674// leaq (%r14,%rdi), %rdi
675// movl %esi, %esi
676// leaq (%r14,%rsi), %rsi
677// rep movsq
678// .bundle_unlock
679void X86::X86MCLFIRewriter::rewriteStringOperation(const MCInst &Inst,
680 MCStreamer &Out,
681 const MCSubtargetInfo &STI) {
682 std::optional<StringOpPointers> Ptrs = getStringOpPointers(Inst.getOpcode());
683 assert(Ptrs && "expected a string instruction");
684
685 // The guards below sandbox the full 64-bit %rdi and %rsi, so the access must
686 // use them with a 64-bit address size and no %fs segment override.
687 for (const MCOperand &Op : Inst) {
688 if (!Op.isReg())
689 continue;
690 if (Op.getReg() == X86::EDI || Op.getReg() == X86::ESI)
691 return error(Inst, "unsupported address size for string instruction");
692 if (Op.getReg() == X86::FS)
693 return error(Inst, "unsupported segment override for string instruction");
694 }
695
696 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
697
698 if (Ptrs->UsesDest)
699 emitStringPtrFixup(X86::RDI, X86::EDI, Out, STI);
700 if (Ptrs->UsesSrc)
701 emitStringPtrFixup(X86::RSI, X86::ESI, Out, STI);
702
703 emitWithPrefixes(Inst, Out, STI);
704
705 Out.emitBundleUnlock(STI);
706}
707
708// The register form of an instruction that modifies the stack and reads its
709// source from memory, if there is one.
710static std::optional<unsigned> memToRegOpcode(unsigned Opcode) {
711 switch (Opcode) {
712 case X86::MOV64rm:
713 return X86::MOV64rr;
714 case X86::ADD64rm:
715 return X86::ADD64rr;
716 case X86::SUB64rm:
717 return X86::SUB64rr;
718 case X86::AND64rm:
719 return X86::AND64rr;
720 case X86::OR64rm:
721 return X86::OR64rr;
722 case X86::XOR64rm:
723 return X86::XOR64rr;
724 default:
725 return std::nullopt;
726 }
727}
728
729// The 32-bit form of a 64-bit instruction, if it has one that can be used to
730// modify the stack pointer.
731static std::optional<unsigned> demote64BitOpcode(unsigned Opcode) {
732 switch (Opcode) {
733 case X86::MOV64rr:
734 return X86::MOV32rr;
735 case X86::MOV64ri32:
736 return X86::MOV32ri;
737 case X86::LEA64r:
738 return X86::LEA64_32r;
739 case X86::ADD64rr:
740 return X86::ADD32rr;
741 case X86::ADD64ri8:
742 return X86::ADD32ri8;
743 case X86::ADD64ri32:
744 return X86::ADD32ri;
745 case X86::SUB64rr:
746 return X86::SUB32rr;
747 case X86::SUB64ri8:
748 return X86::SUB32ri8;
749 case X86::SUB64ri32:
750 return X86::SUB32ri;
751 case X86::AND64rr:
752 return X86::AND32rr;
753 case X86::AND64ri8:
754 return X86::AND32ri8;
755 case X86::AND64ri32:
756 return X86::AND32ri;
757 case X86::OR64rr:
758 return X86::OR32rr;
759 case X86::OR64ri8:
760 return X86::OR32ri8;
761 case X86::OR64ri32:
762 return X86::OR32ri;
763 case X86::XOR64rr:
764 return X86::XOR32rr;
765 case X86::XOR64ri8:
766 return X86::XOR32ri8;
767 case X86::XOR64ri32:
768 return X86::XOR32ri;
769 default:
770 return std::nullopt;
771 }
772}
773
774MCRegister X86::X86MCLFIRewriter::getWrittenStackReg(const MCInst &Inst) const {
775 const MCInstrDesc &Desc = InstInfo->get(Inst.getOpcode());
776 for (unsigned I = 0, E = Desc.getNumDefs(); I < E; ++I) {
777 const MCOperand &Op = Inst.getOperand(I);
778 if (Op.isReg() && RegInfo->regsOverlap(Op.getReg(), X86::RSP))
779 return Op.getReg();
780 }
781 return X86::NoRegister;
782}
783
784// Emit leaq (%rsp,%r14), %rsp
785static void emitStackFixup(MCStreamer &Out, const MCSubtargetInfo &STI) {
786 Out.emitInstruction(MCInstBuilder(X86::LEA64r)
787 .addReg(X86::RSP)
788 .addReg(X86::RSP)
789 .addImm(1)
790 .addReg(LFIBaseReg)
791 .addImm(0)
792 .addReg(X86::NoRegister),
793 STI);
794}
795
796// addq $8, %rsp
797// ->
798// .bundle_lock
799// addl $8, %esp
800// leaq (%rsp,%r14), %rsp
801// .bundle_unlock
802void X86::X86MCLFIRewriter::rewriteStackModification(
803 MCRegister StackReg, const MCInst &Inst, MCStreamer &Out,
804 const MCSubtargetInfo &STI) {
805 if (StackReg != X86::RSP)
806 return error(Inst, "unsupported modification of the stack pointer");
807
808 discardPrefixes();
809
810 // popq %rsp loads through the stack pointer that it overwrites, so it cannot
811 // simply be demoted. Pop into the scratch register instead, and move the
812 // result into %esp.
813 if (Inst.getOpcode() == X86::POP64r) {
814 Out.emitInstruction(MCInstBuilder(X86::POP64r).addReg(LFIScratchReg), STI);
815
816 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
817 Out.emitInstruction(
818 MCInstBuilder(X86::MOV32rr).addReg(X86::ESP).addReg(X86::R11D), STI);
819 emitStackFixup(Out, STI);
820 Out.emitBundleUnlock(STI);
821 return;
822 }
823
824 // movq 8(%rdi), %rsp
825 // ->
826 // movq %gs:8(%edi), %r11
827 // .bundle_lock
828 // movl %r11d, %esp
829 // leaq (%rsp,%r14), %rsp
830 // .bundle_unlock
831 int MemIdx = X86II::getMemoryOperandIdx(InstInfo->get(Inst.getOpcode()));
832 if (MemIdx >= 0 && !ignoresMemOperand(Inst.getOpcode())) {
833 std::optional<unsigned> RegOpcode = memToRegOpcode(Inst.getOpcode());
834 if (!RegOpcode)
835 return error(Inst, "unsupported modification of the stack pointer");
836
837 MCInstBuilder Load(X86::MOV64rm);
838 Load.addReg(LFIScratchReg);
839 for (unsigned I = 0; I < X86::AddrNumOperands; ++I)
840 Load.addOperand(Inst.getOperand(MemIdx + I));
841 Load.setLoc(Inst.getLoc());
842 doRewriteInst(Load, Out, STI);
843
844 MCInstBuilder RegForm(*RegOpcode);
845 for (int I = 0; I < MemIdx; ++I)
846 RegForm.addOperand(Inst.getOperand(I));
847 RegForm.addReg(LFIScratchReg);
848 RegForm.setLoc(Inst.getLoc());
849 return doRewriteInst(RegForm, Out, STI);
850 }
851
852 std::optional<unsigned> Opcode = demote64BitOpcode(Inst.getOpcode());
853 if (!Opcode)
854 return error(Inst, "unsupported modification of the stack pointer");
855
856 MCInst Modified(Inst);
857 Modified.setOpcode(*Opcode);
858
859 ArrayRef<MCOperandInfo> OpInfo = InstInfo->get(*Opcode).operands();
860 for (unsigned I = 0, E = std::min<unsigned>(OpInfo.size(),
861 Modified.getNumOperands());
862 I < E; ++I) {
863 MCOperand &Op = Modified.getOperand(I);
864 if (OpInfo[I].OperandType != MCOI::OPERAND_REGISTER || !Op.isReg())
865 continue;
866 if (getX86MCRegisterClass(X86::GR64RegClassID).contains(Op.getReg()))
867 Op.setReg(RegInfo->getSubReg(Op.getReg(), X86::sub_32bit));
868 }
869
870 Out.emitBundleLock(/*AlignToEnd=*/false, STI);
871 Out.emitInstruction(Modified, STI);
872 emitStackFixup(Out, STI);
873 Out.emitBundleUnlock(STI);
874}
875
876// leave
877// ->
878// .bundle_lock
879// movl %ebp, %esp
880// leaq (%rsp,%r14), %rsp
881// .bundle_unlock
882// popq %rbp
883void X86::X86MCLFIRewriter::rewriteLeave(const MCInst &Inst, MCStreamer &Out,
884 const MCSubtargetInfo &STI) {
885 doRewriteInst(MCInstBuilder(X86::MOV64rr)
886 .addReg(X86::RSP)
887 .addReg(X86::RBP)
888 .setLoc(Inst.getLoc()),
889 Out, STI);
890 Out.emitInstruction(MCInstBuilder(X86::POP64r).addReg(X86::RBP), STI);
891}
892
893void X86::X86MCLFIRewriter::doRewriteInst(const MCInst &Inst, MCStreamer &Out,
894 const MCSubtargetInfo &STI) {
895 if (!STI.hasFeature(X86::Is64Bit))
896 return error(Inst, "LFI only supports 64-bit mode");
897
898 // Hold a stand-alone prefix back, so it can be applied to the rewritten
899 // instruction.
900 if (isPrefix(Inst, *InstInfo)) {
901 if (!isSupportedPrefix(Inst))
902 return error(Inst, "unsupported instruction prefix");
903 Prefixes.push_back(Inst);
904 return;
905 }
906
907 // A lock prefix is only valid on an instruction that writes memory, and
908 // would otherwise be dropped by the rewrites below.
909 if ((Inst.getFlags() & X86::IP_HAS_LOCK) && !mayStore(Inst))
910 return error(Inst, "unsupported lock prefix");
911
912 if (mayModifyRegister(Inst, LFIBaseReg) ||
913 mayModifyRegister(Inst, LFICtxReg) || mayModifyRegister(Inst, LFIBaseSeg))
914 return error(Inst, "illegal modification of reserved LFI register");
915
916 if (isWriteSegmentBase(Inst.getOpcode()))
917 return error(Inst, "illegal modification of reserved segment base");
918
919 if (isSyscall(Inst))
920 return rewriteSyscall(Inst, Out, STI);
921
922 if (isReturn(Inst))
923 return rewriteReturn(Inst, Out, STI);
924
925 if (isDirectCall(Inst))
926 return rewriteDirectCall(Inst, Out, STI);
927
928 // jmpabs is disallowed since it jumps to an absolute address.
929 if (Inst.getOpcode() == X86::JMPABS64i)
930 return error(Inst, "unsupported branch instruction");
931
932 if (isIndirectBranch(Inst) || isCall(Inst)) {
933 if (!isSupportedIndirectBranch(Inst))
934 return error(Inst, "unsupported indirect branch");
935 return rewriteIndirectBranch(Inst, Out, STI);
936 }
937
938 for (const MCOperand &Op : Inst)
939 if (Op.isReg() && Op.getReg() == LFIBaseSeg)
940 return error(Inst, "invalid use of reserved segment register %gs");
941
942 if (MCRegister StackReg = getWrittenStackReg(Inst))
943 return rewriteStackModification(StackReg, Inst, Out, STI);
944
945 // These accesses cannot be sandboxed by rewriting the addressing mode. The
946 // moffs form of mov encodes an absolute address in the instruction instead
947 // of an addressing mode.
948 if ((InstInfo->get(Inst.getOpcode()).TSFlags & X86II::FormMask) ==
950 hasImplicitMemPointer(Inst.getOpcode()))
951 return error(Inst, "unsupported memory access");
952
953 if (isBitTestWithRegOffset(Inst.getOpcode()))
954 return error(Inst, "unsupported bit test with a register bit offset");
955
956 if (isFSAccess(Inst))
957 return rewriteFSAccess(Inst, Out, STI);
958
959 // The compiler never generates 32-bit addressing in LP64 mode.
960 int MemIdx = X86II::getMemoryOperandIdx(InstInfo->get(Inst.getOpcode()));
961 if (MemIdx >= 0 && !ignoresMemOperand(Inst.getOpcode()) &&
962 has32BitAddress(Inst, MemIdx))
963 return error(Inst, "unsupported 32-bit address size");
964
965 if (getStringOpPointers(Inst.getOpcode()))
966 return rewriteStringOperation(Inst, Out, STI);
967
968 if (Inst.getOpcode() == X86::LEAVE64)
969 return rewriteLeave(Inst, Out, STI);
970
971 if (Inst.getOpcode() == X86::ENTER &&
972 (!Inst.getOperand(1).isImm() ||
973 static_cast<uint64_t>(Inst.getOperand(1).getImm()) > 1))
974 return error(Inst, "unsupported nesting level for enter");
975
976 rewriteMemAccess(Inst, Out, STI);
977}
978
980 const MCSubtargetInfo &STI) {
981 // The guard prevents rewrite-recursion when we emit instructions from inside
982 // the rewriter (such instructions should not be rewritten).
983 if (!Enabled || Guard)
984 return false;
985 Guard = true;
986
987 size_t PrefixesBefore = Prefixes.size();
988
989 doRewriteInst(Inst, Out, STI);
990
991 if (Prefixes.size() == PrefixesBefore)
992 discardPrefixes();
993
994 Guard = false;
995 return true;
996}
997
999 discardPrefixes();
1000}
1001
1002void X86::X86MCLFIRewriter::finish(MCStreamer &Out) { discardPrefixes(); }
static constexpr MCRegister LFIScratchReg
static constexpr MCRegister LFICtxReg
static bool isSyscall(const MCInst &Inst)
static constexpr MCRegister LFIBaseReg
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
#define error(X)
static bool isPrefix(unsigned Opcode, const MCInstrInfo &MCII)
Check if the instruction is a prefix.
static bool isWriteSegmentBase(unsigned Opcode)
static void emitStackFixup(MCStreamer &Out, const MCSubtargetInfo &STI)
static std::optional< unsigned > memToRegOpcode(unsigned Opcode)
static bool has32BitAddress(const MCInst &Inst, int MemIdx)
static bool isDirectCall(const MCInst &Inst)
static void emitTPLoad(MCRegister Reg, MCStreamer &Out, const MCSubtargetInfo &STI)
static bool hasNoTrackPrefix(const MCInst &Inst, const MCInstrInfo &InstInfo)
static bool isGR64OrNone(MCRegister Reg)
static bool isPrefix(const MCInst &Inst, const MCInstrInfo &InstInfo)
static bool isSupportedIndirectBranch(const MCInst &Inst)
static int findFSMemOperand(const MCInst &Inst, const MCInstrInfo &InstInfo)
static bool hasGPRIndex(const MCInst &Inst, int MemIdx)
static bool isSupportedPrefix(const MCInst &Inst)
static constexpr int TPOffset
static void emitStringPtrFixup(MCRegister Reg, MCRegister Reg32, MCStreamer &Out, const MCSubtargetInfo &STI)
static bool isSyscall(const MCInst &Inst)
static bool hasImplicitMemPointer(unsigned Opcode)
static constexpr MCRegister LFIBaseSeg
static bool readsRegister(const MCInst &Inst, const MCInstrDesc &Desc, MCRegister Reg, const MCRegisterInfo &RI)
static bool isSandboxBaseReg(MCRegister Reg)
static std::optional< unsigned > demote64BitOpcode(unsigned Opcode)
static bool ignoresMemOperand(unsigned Opcode)
static bool isBitTestWithRegOffset(unsigned Opcode)
static constexpr int SyscallOffset
static std::optional< StringOpPointers > getStringOpPointers(unsigned Opcode)
size_t size() const
Get the array size.
Definition ArrayRef.h:141
Instances of this class represent a single low-level machine instruction.
Definition MCInst.h:188
unsigned getNumOperands() const
Definition MCInst.h:212
SMLoc getLoc() const
Definition MCInst.h:208
unsigned getFlags() const
Definition MCInst.h:205
unsigned getOpcode() const
Definition MCInst.h:202
const MCOperand & getOperand(unsigned i) const
Definition MCInst.h:210
Describe properties that are true of each instruction in the target description file.
Interface to description of machine instruction set.
Definition MCInstrInfo.h:27
std::unique_ptr< MCRegisterInfo > RegInfo
Instances of this class represent operands of the MCInst class.
Definition MCInst.h:40
void setImm(int64_t Val)
Definition MCInst.h:89
int64_t getImm() const
Definition MCInst.h:84
bool isImm() const
Definition MCInst.h:66
void setReg(MCRegister Reg)
Set the register number.
Definition MCInst.h:79
bool isReg() const
Definition MCInst.h:65
MCRegister getReg() const
Returns the register number.
Definition MCInst.h:73
MCRegisterClass - Base class of TargetRegisterClass.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
bool regsOverlap(MCRegister RegA, MCRegister RegB) const
Returns true if the two registers are equal or alias each other.
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
Streaming machine code generation interface.
Definition MCStreamer.h:222
Generic base class for all target subtargets.
bool hasFeature(unsigned Feature) const
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx, SMLoc Loc=SMLoc())
Definition MCExpr.h:213
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition MCSymbol.h:42
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
void finish(MCStreamer &Out) override
bool rewriteInst(const MCInst &Inst, MCStreamer &Out, const MCSubtargetInfo &STI) override
void onLabel(const MCSymbol *Symbol, MCStreamer &Out) override
bool isPrefix(uint64_t TSFlags)
int getMemoryOperandIdx(const MCInstrDesc &Desc)
@ RawFrmMemOffs
RawFrmMemOffs - This form is for instructions that store an absolute memory offset as an immediate wi...
@ AddrNumOperands
Definition X86BaseInfo.h:37
constexpr unsigned LFIBundleSize
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
This is an optimization pass for GlobalISel generic memory operations.
@ Load
The value being inserted comes from a load (InsertElement only).
Op::Description Desc
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >