LLVM 20.0.0git
x86_64.h
Go to the documentation of this file.
1//===-- x86_64.h - Generic JITLink x86-64 edge kinds, utilities -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Generic utilities for graphs representing x86-64 objects.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_EXECUTIONENGINE_JITLINK_X86_64_H
14#define LLVM_EXECUTIONENGINE_JITLINK_X86_64_H
15
18
19namespace llvm {
20namespace jitlink {
21namespace x86_64 {
22
23/// Represents x86-64 fixups and other x86-64-specific edge kinds.
25
26 /// A plain 64-bit pointer value relocation.
27 ///
28 /// Fixup expression:
29 /// Fixup <- Target + Addend : uint64
30 ///
32
33 /// A plain 32-bit pointer value relocation.
34 ///
35 /// Fixup expression:
36 /// Fixup <- Target + Addend : uint32
37 ///
38 /// Errors:
39 /// - The target must reside in the low 32-bits of the address space,
40 /// otherwise an out-of-range error will be returned.
41 ///
43
44 /// A signed 32-bit pointer value relocation
45 ///
46 /// Fixup expression:
47 /// Fixup <- Target + Addend : int32
48 ///
49 /// Errors:
50 /// - The target must reside in the signed 32-bits([-2**31, 2**32 - 1]) of
51 /// the address space, otherwise an out-of-range error will be returned.
53
54 /// A plain 16-bit pointer value relocation.
55 ///
56 /// Fixup expression:
57 /// Fixup <- Target + Addend : uint16
58 ///
59 /// Errors:
60 /// - The target must reside in the low 16-bits of the address space,
61 /// otherwise an out-of-range error will be returned.
62 ///
64
65 /// A plain 8-bit pointer value relocation.
66 ///
67 /// Fixup expression:
68 /// Fixup <- Target + Addend : uint8
69 ///
70 /// Errors:
71 /// - The target must reside in the low 8-bits of the address space,
72 /// otherwise an out-of-range error will be returned.
73 ///
75
76 /// A 64-bit delta.
77 ///
78 /// Delta from the fixup to the target.
79 ///
80 /// Fixup expression:
81 /// Fixup <- Target - Fixup + Addend : int64
82 ///
84
85 /// A 32-bit delta.
86 ///
87 /// Delta from the fixup to the target.
88 ///
89 /// Fixup expression:
90 /// Fixup <- Target - Fixup + Addend : int32
91 ///
92 /// Errors:
93 /// - The result of the fixup expression must fit into an int32, otherwise
94 /// an out-of-range error will be returned.
95 ///
97
98 /// An 8-bit delta.
99 ///
100 /// Delta from the fixup to the target.
101 ///
102 /// Fixup expression:
103 /// Fixup <- Target - Fixup + Addend : int8
104 ///
105 /// Errors:
106 /// - The result of the fixup expression must fit into an int8, otherwise
107 /// an out-of-range error will be returned.
108 ///
110
111 /// A 64-bit negative delta.
112 ///
113 /// Delta from target back to the fixup.
114 ///
115 /// Fixup expression:
116 /// Fixup <- Fixup - Target + Addend : int64
117 ///
119
120 /// A 32-bit negative delta.
121 ///
122 /// Delta from the target back to the fixup.
123 ///
124 /// Fixup expression:
125 /// Fixup <- Fixup - Target + Addend : int32
126 ///
127 /// Errors:
128 /// - The result of the fixup expression must fit into an int32, otherwise
129 /// an out-of-range error will be returned.
131
132 /// A 64-bit GOT delta.
133 ///
134 /// Delta from the global offset table to the target
135 ///
136 /// Fixup expression:
137 /// Fixup <- Target - GOTSymbol + Addend : int64
138 ///
139 /// Errors:
140 /// - *ASSERTION* Failure to a null pointer GOTSymbol, which the GOT section
141 /// symbol was not been defined.
143
144 /// A 32-bit PC-relative branch.
145 ///
146 /// Represents a PC-relative call or branch to a target. This can be used to
147 /// identify, record, and/or patch call sites.
148 ///
149 /// The fixup expression for this kind includes an implicit offset to account
150 /// for the PC (unlike the Delta edges) so that a Branch32PCRel with a target
151 /// T and addend zero is a call/branch to the start (offset zero) of T.
152 ///
153 /// Fixup expression:
154 /// Fixup <- Target - (Fixup + 4) + Addend : int32
155 ///
156 /// Errors:
157 /// - The result of the fixup expression must fit into an int32, otherwise
158 /// an out-of-range error will be returned.
159 ///
161
162 /// A 32-bit PC-relative relocation.
163 ///
164 /// Represents a data/control flow instruction using PC-relative addressing
165 /// to a target.
166 ///
167 /// The fixup expression for this kind includes an implicit offset to account
168 /// for the PC (unlike the Delta edges) so that a PCRel32 with a target
169 /// T and addend zero is a call/branch to the start (offset zero) of T.
170 ///
171 /// Fixup expression:
172 /// Fixup <- Target - (Fixup + 4) + Addend : int32
173 ///
174 /// Errors:
175 /// - The result of the fixup expression must fit into an int32, otherwise
176 /// an out-of-range error will be returned.
177 ///
179
180 /// A 32-bit PC-relative branch to a pointer jump stub.
181 ///
182 /// The target of this relocation should be a pointer jump stub of the form:
183 ///
184 /// \code{.s}
185 /// .text
186 /// jmpq *tgtptr(%rip)
187 /// ; ...
188 ///
189 /// .data
190 /// tgtptr:
191 /// .quad 0
192 /// \endcode
193 ///
194 /// This edge kind has the same fixup expression as BranchPCRel32, but further
195 /// identifies the call/branch as being to a pointer jump stub. For edges of
196 /// this kind the jump stub should not be bypassed (use
197 /// BranchPCRel32ToPtrJumpStubBypassable for that), but the pointer location
198 /// target may be recorded to allow manipulation at runtime.
199 ///
200 /// Fixup expression:
201 /// Fixup <- Target - Fixup + Addend - 4 : int32
202 ///
203 /// Errors:
204 /// - The result of the fixup expression must fit into an int32, otherwise
205 /// an out-of-range error will be returned.
206 ///
208
209 /// A relaxable version of BranchPCRel32ToPtrJumpStub.
210 ///
211 /// The edge kind has the same fixup expression as BranchPCRel32ToPtrJumpStub,
212 /// but identifies the call/branch as being to a pointer jump stub that may be
213 /// bypassed with a direct jump to the ultimate target if the ultimate target
214 /// is within range of the fixup location.
215 ///
216 /// Fixup expression:
217 /// Fixup <- Target - Fixup + Addend - 4: int32
218 ///
219 /// Errors:
220 /// - The result of the fixup expression must fit into an int32, otherwise
221 /// an out-of-range error will be returned.
222 ///
224
225 /// A GOT entry getter/constructor, transformed to Delta32 pointing at the GOT
226 /// entry for the original target.
227 ///
228 /// Indicates that this edge should be transformed into a Delta32 targeting
229 /// the GOT entry for the edge's current target, maintaining the same addend.
230 /// A GOT entry for the target should be created if one does not already
231 /// exist.
232 ///
233 /// Edges of this kind are usually handled by a GOT builder pass inserted by
234 /// default.
235 ///
236 /// Fixup expression:
237 /// NONE
238 ///
239 /// Errors:
240 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
241 /// phase will result in an assert/unreachable during the fixup phase.
242 ///
244
245 /// A GOT entry getter/constructor, transformed to Delta64 pointing at the GOT
246 /// entry for the original target.
247 ///
248 /// Indicates that this edge should be transformed into a Delta64 targeting
249 /// the GOT entry for the edge's current target, maintaining the same addend.
250 /// A GOT entry for the target should be created if one does not already
251 /// exist.
252 ///
253 /// Edges of this kind are usually handled by a GOT builder pass inserted by
254 /// default.
255 ///
256 /// Fixup expression:
257 /// NONE
258 ///
259 /// Errors:
260 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
261 /// phase will result in an assert/unreachable during the fixup phase.
262 ///
264
265 /// A GOT entry offset within GOT getter/constructor, transformed to
266 /// Delta64FromGOT
267 /// pointing at the GOT entry for the original target
268 ///
269 /// Indicates that this edge should be transformed into a Delta64FromGOT
270 /// targeting
271 /// the GOT entry for the edge's current target, maintaining the same addend.
272 /// A GOT entry for the target should be created if one does not already
273 /// exist.
274 ///
275 /// Edges of this kind are usually handled by a GOT builder pass inserted by
276 /// default
277 ///
278 /// Fixup expression:
279 /// NONE
280 ///
281 /// Errors:
282 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
283 /// phase will result in an assert/unreachable during the fixup phase
285
286 /// A PC-relative load of a GOT entry, relaxable if GOT entry target is
287 /// in-range of the fixup
288 ///
289 /// TODO: Explain the optimization
290 ///
291 /// Fixup expression
292 /// Fixup <- Target - (Fixup + 4) + Addend : int32
293 ///
294 /// Errors:
295 /// - The result of the fixup expression must fit into an int32, otherwise
296 /// an out-of-range error will be returned.
297 //
299
300 /// A PC-relative REX load of a GOT entry, relaxable if GOT entry target
301 /// is in-range of the fixup.
302 ///
303 /// If the GOT entry target is in-range of the fixup then the load from the
304 /// GOT may be replaced with a direct memory address calculation.
305 ///
306 /// Fixup expression:
307 /// Fixup <- Target - (Fixup + 4) + Addend : int32
308 ///
309 /// Errors:
310 /// - The result of the fixup expression must fit into an int32, otherwise
311 /// an out-of-range error will be returned.
312 ///
314
315 /// A GOT entry getter/constructor, transformed to
316 /// PCRel32ToGOTLoadREXRelaxable pointing at the GOT entry for the original
317 /// target.
318 ///
319 /// Indicates that this edge should be lowered to a PC32ToGOTLoadREXRelaxable
320 /// targeting the GOT entry for the edge's current target, maintaining the
321 /// same addend. A GOT entry for the target should be created if one does not
322 /// already exist.
323 ///
324 /// Edges of this kind are usually lowered by a GOT builder pass inserted by
325 /// default.
326 ///
327 /// Fixup expression:
328 /// NONE
329 ///
330 /// Errors:
331 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
332 /// phase will result in an assert/unreachable during the fixup phase.
333 ///
335
336 /// A GOT entry getter/constructor, transformed to
337 /// PCRel32ToGOTLoadRelaxable pointing at the GOT entry for the original
338 /// target.
339 ///
340 /// Indicates that this edge should be lowered to a PC32ToGOTLoadRelaxable
341 /// targeting the GOT entry for the edge's current target, maintaining the
342 /// same addend. A GOT entry for the target should be created if one does not
343 /// already exist.
344 ///
345 /// Edges of this kind are usually lowered by a GOT builder pass inserted by
346 /// default.
347 ///
348 /// Fixup expression:
349 /// NONE
350 ///
351 /// Errors:
352 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
353 /// phase will result in an assert/unreachable during the fixup phase.
354 ///
356
357 /// A PC-relative REX load of a Thread Local Variable Pointer (TLVP) entry,
358 /// relaxable if the TLVP entry target is in-range of the fixup.
359 ///
360 /// If the TLVP entry target is in-range of the fixup then the load from the
361 /// TLVP may be replaced with a direct memory address calculation.
362 ///
363 /// The target of this edge must be a thread local variable entry of the form
364 /// .quad <tlv getter thunk>
365 /// .quad <tlv key>
366 /// .quad <tlv initializer>
367 ///
368 /// Fixup expression:
369 /// Fixup <- Target - (Fixup + 4) + Addend : int32
370 ///
371 /// Errors:
372 /// - The result of the fixup expression must fit into an int32, otherwise
373 /// an out-of-range error will be returned.
374 /// - The target must be either external, or a TLV entry of the required
375 /// form, otherwise a malformed TLV entry error will be returned.
376 ///
378
379 /// TODO: Explain the generic edge kind
381
382 /// A TLVP entry getter/constructor, transformed to
383 /// Delta32ToTLVPLoadREXRelaxable.
384 ///
385 /// Indicates that this edge should be transformed into a
386 /// Delta32ToTLVPLoadREXRelaxable targeting the TLVP entry for the edge's
387 /// current target. A TLVP entry for the target should be created if one does
388 /// not already exist.
389 ///
390 /// Fixup expression:
391 /// NONE
392 ///
393 /// Errors:
394 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
395 /// phase will result in an assert/unreachable during the fixup phase.
396 ///
398 // First platform specific relocation.
401
402/// Returns a string name for the given x86-64 edge. For debugging purposes
403/// only.
404const char *getEdgeKindName(Edge::Kind K);
405
406/// Apply fixup expression for edge to block content.
408 const Symbol *GOTSymbol) {
409 using namespace support;
410
411 char *BlockWorkingMem = B.getAlreadyMutableContent().data();
412 char *FixupPtr = BlockWorkingMem + E.getOffset();
413 auto FixupAddress = B.getAddress() + E.getOffset();
414
415 switch (E.getKind()) {
416
417 case Pointer64: {
418 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
419 *(ulittle64_t *)FixupPtr = Value;
420 break;
421 }
422
423 case Pointer32: {
424 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
425 if (LLVM_LIKELY(isUInt<32>(Value)))
426 *(ulittle32_t *)FixupPtr = Value;
427 else
428 return makeTargetOutOfRangeError(G, B, E);
429 break;
430 }
431 case Pointer32Signed: {
432 int64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
433 if (LLVM_LIKELY(isInt<32>(Value)))
434 *(little32_t *)FixupPtr = Value;
435 else
436 return makeTargetOutOfRangeError(G, B, E);
437 break;
438 }
439
440 case Pointer16: {
441 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
442 if (LLVM_LIKELY(isUInt<16>(Value)))
443 *(ulittle16_t *)FixupPtr = Value;
444 else
445 return makeTargetOutOfRangeError(G, B, E);
446 break;
447 }
448
449 case Pointer8: {
450 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
451 if (LLVM_LIKELY(isUInt<8>(Value)))
452 *(uint8_t *)FixupPtr = Value;
453 else
454 return makeTargetOutOfRangeError(G, B, E);
455 break;
456 }
457
458 case PCRel32:
459 case BranchPCRel32:
465 int64_t Value =
466 E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend();
467 if (LLVM_LIKELY(isInt<32>(Value)))
468 *(little32_t *)FixupPtr = Value;
469 else
470 return makeTargetOutOfRangeError(G, B, E);
471 break;
472 }
473
474 case Delta64: {
475 int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
476 *(little64_t *)FixupPtr = Value;
477 break;
478 }
479
480 case Delta32: {
481 int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
482 if (LLVM_LIKELY(isInt<32>(Value)))
483 *(little32_t *)FixupPtr = Value;
484 else
485 return makeTargetOutOfRangeError(G, B, E);
486 break;
487 }
488
489 case Delta8: {
490 int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
491 if (LLVM_LIKELY(isInt<8>(Value)))
492 *FixupPtr = Value;
493 else
494 return makeTargetOutOfRangeError(G, B, E);
495 break;
496 }
497
498 case NegDelta64: {
499 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
500 *(little64_t *)FixupPtr = Value;
501 break;
502 }
503
504 case NegDelta32: {
505 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
506 if (LLVM_LIKELY(isInt<32>(Value)))
507 *(little32_t *)FixupPtr = Value;
508 else
509 return makeTargetOutOfRangeError(G, B, E);
510 break;
511 }
512 case Delta64FromGOT: {
513 assert(GOTSymbol && "No GOT section symbol");
514 int64_t Value =
515 E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend();
516 *(little64_t *)FixupPtr = Value;
517 break;
518 }
519
520 default:
521 return make_error<JITLinkError>(
522 "In graph " + G.getName() + ", section " + B.getSection().getName() +
523 " unsupported edge kind " + getEdgeKindName(E.getKind()));
524 }
525
526 return Error::success();
527}
528
529/// x86_64 pointer size.
530constexpr uint64_t PointerSize = 8;
531
532/// x86-64 null pointer content.
533extern const char NullPointerContent[PointerSize];
534
535/// x86-64 pointer jump stub content.
536///
537/// Contains the instruction sequence for an indirect jump via an in-memory
538/// pointer:
539/// jmpq *ptr(%rip)
540extern const char PointerJumpStubContent[6];
541
542/// Creates a new pointer block in the given section and returns an anonymous
543/// symbol pointing to it.
544///
545/// If InitialTarget is given then an Pointer64 relocation will be added to the
546/// block pointing at InitialTarget.
547///
548/// The pointer block will have the following default values:
549/// alignment: 64-bit
550/// alignment-offset: 0
551/// address: highest allowable (~7U)
553 Symbol *InitialTarget = nullptr,
554 uint64_t InitialAddend = 0) {
555 auto &B = G.createContentBlock(PointerSection, NullPointerContent,
556 orc::ExecutorAddr(~uint64_t(7)), 8, 0);
557 if (InitialTarget)
558 B.addEdge(Pointer64, 0, *InitialTarget, InitialAddend);
559 return G.addAnonymousSymbol(B, 0, 8, false, false);
560}
561
562/// Create a jump stub block that jumps via the pointer at the given symbol.
563///
564/// The stub block will have the following default values:
565/// alignment: 8-bit
566/// alignment-offset: 0
567/// address: highest allowable: (~5U)
569 Symbol &PointerSymbol) {
570 auto &B = G.createContentBlock(StubSection, PointerJumpStubContent,
571 orc::ExecutorAddr(~uint64_t(5)), 1, 0);
572 B.addEdge(BranchPCRel32, 2, PointerSymbol, 0);
573 return B;
574}
575
576/// Create a jump stub that jumps via the pointer at the given symbol and
577/// an anonymous symbol pointing to it. Return the anonymous symbol.
578///
579/// The stub block will be created by createPointerJumpStubBlock.
581 Section &StubSection,
582 Symbol &PointerSymbol) {
583 return G.addAnonymousSymbol(
584 createPointerJumpStubBlock(G, StubSection, PointerSymbol), 0, 6, true,
585 false);
586}
587
588/// Global Offset Table Builder.
589class GOTTableManager : public TableManager<GOTTableManager> {
590public:
591 static StringRef getSectionName() { return "$__GOT"; }
592
594 Edge::Kind KindToSet = Edge::Invalid;
595 switch (E.getKind()) {
597 // we need to make sure that the GOT section exists, but don't otherwise
598 // need to fix up this edge
599 getGOTSection(G);
600 return false;
601 }
604 break;
607 break;
609 KindToSet = x86_64::Delta64;
610 break;
612 KindToSet = x86_64::Delta64FromGOT;
613 break;
615 KindToSet = x86_64::Delta32;
616 break;
617 default:
618 return false;
619 }
620 assert(KindToSet != Edge::Invalid &&
621 "Fell through switch, but no new kind to set");
622 DEBUG_WITH_TYPE("jitlink", {
623 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
624 << B->getFixupAddress(E) << " (" << B->getAddress() << " + "
625 << formatv("{0:x}", E.getOffset()) << ")\n";
626 });
627 E.setKind(KindToSet);
628 E.setTarget(getEntryForTarget(G, E.getTarget()));
629 return true;
630 }
631
633 return createAnonymousPointer(G, getGOTSection(G), &Target);
634 }
635
636private:
637 Section &getGOTSection(LinkGraph &G) {
638 if (!GOTSection)
639 GOTSection = &G.createSection(getSectionName(), orc::MemProt::Read);
640 return *GOTSection;
641 }
642
643 Section *GOTSection = nullptr;
644};
645
646/// Procedure Linkage Table Builder.
647class PLTTableManager : public TableManager<PLTTableManager> {
648public:
650
651 static StringRef getSectionName() { return "$__STUBS"; }
652
654 if (E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined()) {
655 DEBUG_WITH_TYPE("jitlink", {
656 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
657 << B->getFixupAddress(E) << " (" << B->getAddress() << " + "
658 << formatv("{0:x}", E.getOffset()) << ")\n";
659 });
660 // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to
661 // be optimized when the target is in-range.
663 E.setTarget(getEntryForTarget(G, E.getTarget()));
664 return true;
665 }
666 return false;
667 }
668
672 }
673
674public:
676 if (!PLTSection)
677 PLTSection = &G.createSection(getSectionName(),
679 return *PLTSection;
680 }
681
683 Section *PLTSection = nullptr;
684};
685
686/// Optimize the GOT and Stub relocations if the edge target address is in range
687/// 1. PCRel32GOTLoadRelaxable. For this edge kind, if the target is in range,
688/// then replace GOT load with lea
689/// 2. BranchPCRel32ToPtrJumpStubRelaxable. For this edge kind, if the target is
690/// in range, replace a indirect jump by plt stub with a direct jump to the
691/// target
693
694} // namespace x86_64
695} // end namespace jitlink
696} // end namespace llvm
697
698#endif // LLVM_EXECUTIONENGINE_JITLINK_X86_64_H
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_LIKELY(EXPR)
Definition: Compiler.h:236
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition: Debug.h:64
#define G(x, y, z)
Definition: MD5.cpp:56
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:337
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Target - Wrapper for Target specific information.
LLVM Value Representation.
Definition: Value.h:74
Represents an address in the executor process.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163