LLVM 19.0.0git
x86_64.h
Go to the documentation of this file.
1//===-- x86_64.h - Generic JITLink x86-64 edge kinds, utilities -*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Generic utilities for graphs representing x86-64 objects.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_EXECUTIONENGINE_JITLINK_X86_64_H
14#define LLVM_EXECUTIONENGINE_JITLINK_X86_64_H
15
18
19namespace llvm {
20namespace jitlink {
21namespace x86_64 {
22
23/// Represents x86-64 fixups and other x86-64-specific edge kinds.
25
26 /// A plain 64-bit pointer value relocation.
27 ///
28 /// Fixup expression:
29 /// Fixup <- Target + Addend : uint64
30 ///
32
33 /// A plain 32-bit pointer value relocation.
34 ///
35 /// Fixup expression:
36 /// Fixup <- Target + Addend : uint32
37 ///
38 /// Errors:
39 /// - The target must reside in the low 32-bits of the address space,
40 /// otherwise an out-of-range error will be returned.
41 ///
43
44 /// A signed 32-bit pointer value relocation
45 ///
46 /// Fixup expression:
47 /// Fixup <- Target + Addend : int32
48 ///
49 /// Errors:
50 /// - The target must reside in the signed 32-bits([-2**31, 2**32 - 1]) of
51 /// the address space, otherwise an out-of-range error will be returned.
53
54 /// A plain 16-bit pointer value relocation.
55 ///
56 /// Fixup expression:
57 /// Fixup <- Target + Addend : uint16
58 ///
59 /// Errors:
60 /// - The target must reside in the low 16-bits of the address space,
61 /// otherwise an out-of-range error will be returned.
62 ///
64
65 /// A plain 8-bit pointer value relocation.
66 ///
67 /// Fixup expression:
68 /// Fixup <- Target + Addend : uint8
69 ///
70 /// Errors:
71 /// - The target must reside in the low 8-bits of the address space,
72 /// otherwise an out-of-range error will be returned.
73 ///
75
76 /// A 64-bit delta.
77 ///
78 /// Delta from the fixup to the target.
79 ///
80 /// Fixup expression:
81 /// Fixup <- Target - Fixup + Addend : int64
82 ///
84
85 /// A 32-bit delta.
86 ///
87 /// Delta from the fixup to the target.
88 ///
89 /// Fixup expression:
90 /// Fixup <- Target - Fixup + Addend : int64
91 ///
92 /// Errors:
93 /// - The result of the fixup expression must fit into an int32, otherwise
94 /// an out-of-range error will be returned.
95 ///
97
98 /// A 64-bit negative delta.
99 ///
100 /// Delta from target back to the fixup.
101 ///
102 /// Fixup expression:
103 /// Fixup <- Fixup - Target + Addend : int64
104 ///
106
107 /// A 32-bit negative delta.
108 ///
109 /// Delta from the target back to the fixup.
110 ///
111 /// Fixup expression:
112 /// Fixup <- Fixup - Target + Addend : int32
113 ///
114 /// Errors:
115 /// - The result of the fixup expression must fit into an int32, otherwise
116 /// an out-of-range error will be returned.
118
119 /// A 64-bit GOT delta.
120 ///
121 /// Delta from the global offset table to the target
122 ///
123 /// Fixup expression:
124 /// Fixup <- Target - GOTSymbol + Addend : int64
125 ///
126 /// Errors:
127 /// - *ASSERTION* Failure to a null pointer GOTSymbol, which the GOT section
128 /// symbol was not been defined.
130
131 /// A 32-bit PC-relative branch.
132 ///
133 /// Represents a PC-relative call or branch to a target. This can be used to
134 /// identify, record, and/or patch call sites.
135 ///
136 /// The fixup expression for this kind includes an implicit offset to account
137 /// for the PC (unlike the Delta edges) so that a Branch32PCRel with a target
138 /// T and addend zero is a call/branch to the start (offset zero) of T.
139 ///
140 /// Fixup expression:
141 /// Fixup <- Target - (Fixup + 4) + Addend : int32
142 ///
143 /// Errors:
144 /// - The result of the fixup expression must fit into an int32, otherwise
145 /// an out-of-range error will be returned.
146 ///
148
149 /// A 32-bit PC-relative relocation.
150 ///
151 /// Represents a data/control flow instruction using PC-relative addressing
152 /// to a target.
153 ///
154 /// The fixup expression for this kind includes an implicit offset to account
155 /// for the PC (unlike the Delta edges) so that a PCRel32 with a target
156 /// T and addend zero is a call/branch to the start (offset zero) of T.
157 ///
158 /// Fixup expression:
159 /// Fixup <- Target - (Fixup + 4) + Addend : int32
160 ///
161 /// Errors:
162 /// - The result of the fixup expression must fit into an int32, otherwise
163 /// an out-of-range error will be returned.
164 ///
166
167 /// A 32-bit PC-relative branch to a pointer jump stub.
168 ///
169 /// The target of this relocation should be a pointer jump stub of the form:
170 ///
171 /// \code{.s}
172 /// .text
173 /// jmpq *tgtptr(%rip)
174 /// ; ...
175 ///
176 /// .data
177 /// tgtptr:
178 /// .quad 0
179 /// \endcode
180 ///
181 /// This edge kind has the same fixup expression as BranchPCRel32, but further
182 /// identifies the call/branch as being to a pointer jump stub. For edges of
183 /// this kind the jump stub should not be bypassed (use
184 /// BranchPCRel32ToPtrJumpStubBypassable for that), but the pointer location
185 /// target may be recorded to allow manipulation at runtime.
186 ///
187 /// Fixup expression:
188 /// Fixup <- Target - Fixup + Addend - 4 : int32
189 ///
190 /// Errors:
191 /// - The result of the fixup expression must fit into an int32, otherwise
192 /// an out-of-range error will be returned.
193 ///
195
196 /// A relaxable version of BranchPCRel32ToPtrJumpStub.
197 ///
198 /// The edge kind has the same fixup expression as BranchPCRel32ToPtrJumpStub,
199 /// but identifies the call/branch as being to a pointer jump stub that may be
200 /// bypassed with a direct jump to the ultimate target if the ultimate target
201 /// is within range of the fixup location.
202 ///
203 /// Fixup expression:
204 /// Fixup <- Target - Fixup + Addend - 4: int32
205 ///
206 /// Errors:
207 /// - The result of the fixup expression must fit into an int32, otherwise
208 /// an out-of-range error will be returned.
209 ///
211
212 /// A GOT entry getter/constructor, transformed to Delta32 pointing at the GOT
213 /// entry for the original target.
214 ///
215 /// Indicates that this edge should be transformed into a Delta32 targeting
216 /// the GOT entry for the edge's current target, maintaining the same addend.
217 /// A GOT entry for the target should be created if one does not already
218 /// exist.
219 ///
220 /// Edges of this kind are usually handled by a GOT builder pass inserted by
221 /// default.
222 ///
223 /// Fixup expression:
224 /// NONE
225 ///
226 /// Errors:
227 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
228 /// phase will result in an assert/unreachable during the fixup phase.
229 ///
231
232 /// A GOT entry getter/constructor, transformed to Delta64 pointing at the GOT
233 /// entry for the original target.
234 ///
235 /// Indicates that this edge should be transformed into a Delta64 targeting
236 /// the GOT entry for the edge's current target, maintaining the same addend.
237 /// A GOT entry for the target should be created if one does not already
238 /// exist.
239 ///
240 /// Edges of this kind are usually handled by a GOT builder pass inserted by
241 /// default.
242 ///
243 /// Fixup expression:
244 /// NONE
245 ///
246 /// Errors:
247 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
248 /// phase will result in an assert/unreachable during the fixup phase.
249 ///
251
252 /// A GOT entry offset within GOT getter/constructor, transformed to
253 /// Delta64FromGOT
254 /// pointing at the GOT entry for the original target
255 ///
256 /// Indicates that this edge should be transformed into a Delta64FromGOT
257 /// targeting
258 /// the GOT entry for the edge's current target, maintaining the same addend.
259 /// A GOT entry for the target should be created if one does not already
260 /// exist.
261 ///
262 /// Edges of this kind are usually handled by a GOT builder pass inserted by
263 /// default
264 ///
265 /// Fixup expression:
266 /// NONE
267 ///
268 /// Errors:
269 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
270 /// phase will result in an assert/unreachable during the fixup phase
272
273 /// A PC-relative load of a GOT entry, relaxable if GOT entry target is
274 /// in-range of the fixup
275 ///
276 /// TODO: Explain the optimization
277 ///
278 /// Fixup expression
279 /// Fixup <- Target - (Fixup + 4) + Addend : int32
280 ///
281 /// Errors:
282 /// - The result of the fixup expression must fit into an int32, otherwise
283 /// an out-of-range error will be returned.
284 //
286
287 /// A PC-relative REX load of a GOT entry, relaxable if GOT entry target
288 /// is in-range of the fixup.
289 ///
290 /// If the GOT entry target is in-range of the fixup then the load from the
291 /// GOT may be replaced with a direct memory address calculation.
292 ///
293 /// Fixup expression:
294 /// Fixup <- Target - (Fixup + 4) + Addend : int32
295 ///
296 /// Errors:
297 /// - The result of the fixup expression must fit into an int32, otherwise
298 /// an out-of-range error will be returned.
299 ///
301
302 /// A GOT entry getter/constructor, transformed to
303 /// PCRel32ToGOTLoadREXRelaxable pointing at the GOT entry for the original
304 /// target.
305 ///
306 /// Indicates that this edge should be lowered to a PC32ToGOTLoadREXRelaxable
307 /// targeting the GOT entry for the edge's current target, maintaining the
308 /// same addend. A GOT entry for the target should be created if one does not
309 /// already exist.
310 ///
311 /// Edges of this kind are usually lowered by a GOT builder pass inserted by
312 /// default.
313 ///
314 /// Fixup expression:
315 /// NONE
316 ///
317 /// Errors:
318 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
319 /// phase will result in an assert/unreachable during the fixup phase.
320 ///
322
323 /// A GOT entry getter/constructor, transformed to
324 /// PCRel32ToGOTLoadRelaxable pointing at the GOT entry for the original
325 /// target.
326 ///
327 /// Indicates that this edge should be lowered to a PC32ToGOTLoadRelaxable
328 /// targeting the GOT entry for the edge's current target, maintaining the
329 /// same addend. A GOT entry for the target should be created if one does not
330 /// already exist.
331 ///
332 /// Edges of this kind are usually lowered by a GOT builder pass inserted by
333 /// default.
334 ///
335 /// Fixup expression:
336 /// NONE
337 ///
338 /// Errors:
339 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
340 /// phase will result in an assert/unreachable during the fixup phase.
341 ///
343
344 /// A PC-relative REX load of a Thread Local Variable Pointer (TLVP) entry,
345 /// relaxable if the TLVP entry target is in-range of the fixup.
346 ///
347 /// If the TLVP entry target is in-range of the fixup then the load from the
348 /// TLVP may be replaced with a direct memory address calculation.
349 ///
350 /// The target of this edge must be a thread local variable entry of the form
351 /// .quad <tlv getter thunk>
352 /// .quad <tlv key>
353 /// .quad <tlv initializer>
354 ///
355 /// Fixup expression:
356 /// Fixup <- Target - (Fixup + 4) + Addend : int32
357 ///
358 /// Errors:
359 /// - The result of the fixup expression must fit into an int32, otherwise
360 /// an out-of-range error will be returned.
361 /// - The target must be either external, or a TLV entry of the required
362 /// form, otherwise a malformed TLV entry error will be returned.
363 ///
365
366 /// TODO: Explain the generic edge kind
368
369 /// A TLVP entry getter/constructor, transformed to
370 /// Delta32ToTLVPLoadREXRelaxable.
371 ///
372 /// Indicates that this edge should be transformed into a
373 /// Delta32ToTLVPLoadREXRelaxable targeting the TLVP entry for the edge's
374 /// current target. A TLVP entry for the target should be created if one does
375 /// not already exist.
376 ///
377 /// Fixup expression:
378 /// NONE
379 ///
380 /// Errors:
381 /// - *ASSERTION* Failure to handle edges of this kind prior to the fixup
382 /// phase will result in an assert/unreachable during the fixup phase.
383 ///
385 // First platform specific relocation.
388
389/// Returns a string name for the given x86-64 edge. For debugging purposes
390/// only.
391const char *getEdgeKindName(Edge::Kind K);
392
393/// Apply fixup expression for edge to block content.
395 const Symbol *GOTSymbol) {
396 using namespace support;
397
398 char *BlockWorkingMem = B.getAlreadyMutableContent().data();
399 char *FixupPtr = BlockWorkingMem + E.getOffset();
400 auto FixupAddress = B.getAddress() + E.getOffset();
401
402 switch (E.getKind()) {
403
404 case Pointer64: {
405 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
406 *(ulittle64_t *)FixupPtr = Value;
407 break;
408 }
409
410 case Pointer32: {
411 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
412 if (LLVM_LIKELY(isUInt<32>(Value)))
413 *(ulittle32_t *)FixupPtr = Value;
414 else
415 return makeTargetOutOfRangeError(G, B, E);
416 break;
417 }
418 case Pointer32Signed: {
419 int64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
420 if (LLVM_LIKELY(isInt<32>(Value)))
421 *(little32_t *)FixupPtr = Value;
422 else
423 return makeTargetOutOfRangeError(G, B, E);
424 break;
425 }
426
427 case Pointer16: {
428 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
429 if (LLVM_LIKELY(isUInt<16>(Value)))
430 *(ulittle16_t *)FixupPtr = Value;
431 else
432 return makeTargetOutOfRangeError(G, B, E);
433 break;
434 }
435
436 case Pointer8: {
437 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();
438 if (LLVM_LIKELY(isUInt<8>(Value)))
439 *(uint8_t *)FixupPtr = Value;
440 else
441 return makeTargetOutOfRangeError(G, B, E);
442 break;
443 }
444
445 case PCRel32:
446 case BranchPCRel32:
452 int64_t Value =
453 E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend();
454 if (LLVM_LIKELY(isInt<32>(Value)))
455 *(little32_t *)FixupPtr = Value;
456 else
457 return makeTargetOutOfRangeError(G, B, E);
458 break;
459 }
460
461 case Delta64: {
462 int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
463 *(little64_t *)FixupPtr = Value;
464 break;
465 }
466
467 case Delta32: {
468 int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
469 if (LLVM_LIKELY(isInt<32>(Value)))
470 *(little32_t *)FixupPtr = Value;
471 else
472 return makeTargetOutOfRangeError(G, B, E);
473 break;
474 }
475
476 case NegDelta64: {
477 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
478 *(little64_t *)FixupPtr = Value;
479 break;
480 }
481
482 case NegDelta32: {
483 int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
484 if (LLVM_LIKELY(isInt<32>(Value)))
485 *(little32_t *)FixupPtr = Value;
486 else
487 return makeTargetOutOfRangeError(G, B, E);
488 break;
489 }
490 case Delta64FromGOT: {
491 assert(GOTSymbol && "No GOT section symbol");
492 int64_t Value =
493 E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend();
494 *(little64_t *)FixupPtr = Value;
495 break;
496 }
497
498 default:
499 return make_error<JITLinkError>(
500 "In graph " + G.getName() + ", section " + B.getSection().getName() +
501 " unsupported edge kind " + getEdgeKindName(E.getKind()));
502 }
503
504 return Error::success();
505}
506
507/// x86_64 pointer size.
508constexpr uint64_t PointerSize = 8;
509
510/// x86-64 null pointer content.
511extern const char NullPointerContent[PointerSize];
512
513/// x86-64 pointer jump stub content.
514///
515/// Contains the instruction sequence for an indirect jump via an in-memory
516/// pointer:
517/// jmpq *ptr(%rip)
518extern const char PointerJumpStubContent[6];
519
520/// Creates a new pointer block in the given section and returns an anonymous
521/// symbol pointing to it.
522///
523/// If InitialTarget is given then an Pointer64 relocation will be added to the
524/// block pointing at InitialTarget.
525///
526/// The pointer block will have the following default values:
527/// alignment: 64-bit
528/// alignment-offset: 0
529/// address: highest allowable (~7U)
531 Symbol *InitialTarget = nullptr,
532 uint64_t InitialAddend = 0) {
533 auto &B = G.createContentBlock(PointerSection, NullPointerContent,
534 orc::ExecutorAddr(~uint64_t(7)), 8, 0);
535 if (InitialTarget)
536 B.addEdge(Pointer64, 0, *InitialTarget, InitialAddend);
537 return G.addAnonymousSymbol(B, 0, 8, false, false);
538}
539
540/// Create a jump stub block that jumps via the pointer at the given symbol.
541///
542/// The stub block will have the following default values:
543/// alignment: 8-bit
544/// alignment-offset: 0
545/// address: highest allowable: (~5U)
547 Symbol &PointerSymbol) {
548 auto &B = G.createContentBlock(StubSection, PointerJumpStubContent,
549 orc::ExecutorAddr(~uint64_t(5)), 1, 0);
550 B.addEdge(BranchPCRel32, 2, PointerSymbol, 0);
551 return B;
552}
553
554/// Create a jump stub that jumps via the pointer at the given symbol and
555/// an anonymous symbol pointing to it. Return the anonymous symbol.
556///
557/// The stub block will be created by createPointerJumpStubBlock.
559 Section &StubSection,
560 Symbol &PointerSymbol) {
561 return G.addAnonymousSymbol(
562 createPointerJumpStubBlock(G, StubSection, PointerSymbol), 0, 6, true,
563 false);
564}
565
566/// Global Offset Table Builder.
567class GOTTableManager : public TableManager<GOTTableManager> {
568public:
569 static StringRef getSectionName() { return "$__GOT"; }
570
572 Edge::Kind KindToSet = Edge::Invalid;
573 switch (E.getKind()) {
575 // we need to make sure that the GOT section exists, but don't otherwise
576 // need to fix up this edge
577 getGOTSection(G);
578 return false;
579 }
582 break;
585 break;
587 KindToSet = x86_64::Delta64;
588 break;
590 KindToSet = x86_64::Delta64FromGOT;
591 break;
593 KindToSet = x86_64::Delta32;
594 break;
595 default:
596 return false;
597 }
598 assert(KindToSet != Edge::Invalid &&
599 "Fell through switch, but no new kind to set");
600 DEBUG_WITH_TYPE("jitlink", {
601 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
602 << B->getFixupAddress(E) << " (" << B->getAddress() << " + "
603 << formatv("{0:x}", E.getOffset()) << ")\n";
604 });
605 E.setKind(KindToSet);
606 E.setTarget(getEntryForTarget(G, E.getTarget()));
607 return true;
608 }
609
611 return createAnonymousPointer(G, getGOTSection(G), &Target);
612 }
613
614private:
615 Section &getGOTSection(LinkGraph &G) {
616 if (!GOTSection)
617 GOTSection = &G.createSection(getSectionName(), orc::MemProt::Read);
618 return *GOTSection;
619 }
620
621 Section *GOTSection = nullptr;
622};
623
624/// Procedure Linkage Table Builder.
625class PLTTableManager : public TableManager<PLTTableManager> {
626public:
628
629 static StringRef getSectionName() { return "$__STUBS"; }
630
632 if (E.getKind() == x86_64::BranchPCRel32 && !E.getTarget().isDefined()) {
633 DEBUG_WITH_TYPE("jitlink", {
634 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "
635 << B->getFixupAddress(E) << " (" << B->getAddress() << " + "
636 << formatv("{0:x}", E.getOffset()) << ")\n";
637 });
638 // Set the edge kind to Branch32ToPtrJumpStubBypassable to enable it to
639 // be optimized when the target is in-range.
641 E.setTarget(getEntryForTarget(G, E.getTarget()));
642 return true;
643 }
644 return false;
645 }
646
650 }
651
652public:
654 if (!PLTSection)
655 PLTSection = &G.createSection(getSectionName(),
657 return *PLTSection;
658 }
659
661 Section *PLTSection = nullptr;
662};
663
664/// Optimize the GOT and Stub relocations if the edge target address is in range
665/// 1. PCRel32GOTLoadRelaxable. For this edge kind, if the target is in range,
666/// then replace GOT load with lea
667/// 2. BranchPCRel32ToPtrJumpStubRelaxable. For this edge kind, if the target is
668/// in range, replace a indirect jump by plt stub with a direct jump to the
669/// target
671
672} // namespace x86_64
673} // end namespace jitlink
674} // end namespace llvm
675
676#endif // LLVM_EXECUTIONENGINE_JITLINK_X86_64_H
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_LIKELY(EXPR)
Definition: Compiler.h:240
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
Definition: Debug.h:64
#define G(x, y, z)
Definition: MD5.cpp:56
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Target - Wrapper for Target specific information.
LLVM Value Representation.
Definition: Value.h:74
Represents an address in the executor process.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto formatv(const char *Fmt, Ts &&... Vals) -> formatv_object< decltype(std::make_tuple(detail::build_format_adapter(std::forward< Ts >(Vals))...))>
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163