LLVM 19.0.0git
AMDGPUTargetStreamer.cpp
Go to the documentation of this file.
1//===-- AMDGPUTargetStreamer.cpp - Mips Target Streamer Methods -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides AMDGPU specific target streamer methods.
10//
11//===----------------------------------------------------------------------===//
12
14#include "AMDGPUPTNote.h"
15#include "AMDKernelCodeT.h"
20#include "llvm/MC/MCAssembler.h"
21#include "llvm/MC/MCContext.h"
32
33using namespace llvm;
34using namespace llvm::AMDGPU;
35
36//===----------------------------------------------------------------------===//
37// AMDGPUTargetStreamer
38//===----------------------------------------------------------------------===//
39
41 ForceGenericVersion("amdgpu-force-generic-version",
42 cl::desc("Force a specific generic_v<N> flag to be "
43 "added. For testing purposes only."),
45
47 msgpack::Document HSAMetadataDoc;
48 if (!HSAMetadataDoc.fromYAML(HSAMetadataString))
49 return false;
50 return EmitHSAMetadata(HSAMetadataDoc, false);
51}
52
55
56 // clang-format off
57 switch (ElfMach) {
58 case ELF::EF_AMDGPU_MACH_R600_R600: AK = GK_R600; break;
59 case ELF::EF_AMDGPU_MACH_R600_R630: AK = GK_R630; break;
69 case ELF::EF_AMDGPU_MACH_R600_SUMO: AK = GK_SUMO; break;
122 case ELF::EF_AMDGPU_MACH_NONE: AK = GK_NONE; break;
123 default: AK = GK_NONE; break;
124 }
125 // clang-format on
126
127 StringRef GPUName = getArchNameAMDGCN(AK);
128 if (GPUName != "")
129 return GPUName;
130 return getArchNameR600(AK);
131}
132
135 if (AK == AMDGPU::GPUKind::GK_NONE)
136 AK = parseArchR600(GPU);
137
138 // clang-format off
139 switch (AK) {
205 }
206 // clang-format on
207
208 llvm_unreachable("unknown GPU");
209}
210
211//===----------------------------------------------------------------------===//
212// AMDGPUTargetAsmStreamer
213//===----------------------------------------------------------------------===//
214
217 : AMDGPUTargetStreamer(S), OS(OS) { }
218
219// A hook for emitting stuff at the end.
220// We use it for emitting the accumulated PAL metadata as directives.
221// The PAL metadata is reset after it is emitted.
223 std::string S;
225 OS << S;
226
227 // Reset the pal metadata so its data will not affect a compilation that
228 // reuses this object.
230}
231
233 OS << "\t.amdgcn_target \"" << getTargetID()->toString() << "\"\n";
234}
235
237 unsigned COV) {
239 OS << "\t.amdhsa_code_object_version " << COV << '\n';
240}
241
242void
244 OS << "\t.amd_kernel_code_t\n";
245 dumpAmdKernelCode(&Header, OS, "\t\t");
246 OS << "\t.end_amd_kernel_code_t\n";
247}
248
250 unsigned Type) {
251 switch (Type) {
252 default: llvm_unreachable("Invalid AMDGPU symbol type");
254 OS << "\t.amdgpu_hsa_kernel " << SymbolName << '\n' ;
255 break;
256 }
257}
258
260 Align Alignment) {
261 OS << "\t.amdgpu_lds " << Symbol->getName() << ", " << Size << ", "
262 << Alignment.value() << '\n';
263}
264
266 OS << "\t.amd_amdgpu_isa \"" << getTargetID()->toString() << "\"\n";
267 return true;
268}
269
271 msgpack::Document &HSAMetadataDoc, bool Strict) {
273 if (!Verifier.verify(HSAMetadataDoc.getRoot()))
274 return false;
275
276 std::string HSAMetadataString;
277 raw_string_ostream StrOS(HSAMetadataString);
278 HSAMetadataDoc.toYAML(StrOS);
279
280 OS << '\t' << HSAMD::V3::AssemblerDirectiveBegin << '\n';
281 OS << StrOS.str() << '\n';
282 OS << '\t' << HSAMD::V3::AssemblerDirectiveEnd << '\n';
283 return true;
284}
285
287 const uint32_t Encoded_s_code_end = 0xbf9f0000;
288 const uint32_t Encoded_s_nop = 0xbf800000;
289 uint32_t Encoded_pad = Encoded_s_code_end;
290
291 // Instruction cache line size in bytes.
292 const unsigned Log2CacheLineSize = AMDGPU::isGFX11Plus(STI) ? 7 : 6;
293 const unsigned CacheLineSize = 1u << Log2CacheLineSize;
294
295 // Extra padding amount in bytes to support prefetch mode 3.
296 unsigned FillSize = 3 * CacheLineSize;
297
298 if (AMDGPU::isGFX90A(STI)) {
299 Encoded_pad = Encoded_s_nop;
300 FillSize = 16 * CacheLineSize;
301 }
302
303 OS << "\t.p2alignl " << Log2CacheLineSize << ", " << Encoded_pad << '\n';
304 OS << "\t.fill " << (FillSize / 4) << ", 4, " << Encoded_pad << '\n';
305 return true;
306}
307
309 const MCSubtargetInfo &STI, StringRef KernelName,
310 const amdhsa::kernel_descriptor_t &KD, uint64_t NextVGPR, uint64_t NextSGPR,
311 bool ReserveVCC, bool ReserveFlatScr) {
312 IsaVersion IVersion = getIsaVersion(STI.getCPU());
313
314 OS << "\t.amdhsa_kernel " << KernelName << '\n';
315
316#define PRINT_FIELD(STREAM, DIRECTIVE, KERNEL_DESC, MEMBER_NAME, FIELD_NAME) \
317 STREAM << "\t\t" << DIRECTIVE << " " \
318 << AMDHSA_BITS_GET(KERNEL_DESC.MEMBER_NAME, FIELD_NAME) << '\n';
319
320 OS << "\t\t.amdhsa_group_segment_fixed_size " << KD.group_segment_fixed_size
321 << '\n';
322 OS << "\t\t.amdhsa_private_segment_fixed_size "
323 << KD.private_segment_fixed_size << '\n';
324 OS << "\t\t.amdhsa_kernarg_size " << KD.kernarg_size << '\n';
325
326 PRINT_FIELD(OS, ".amdhsa_user_sgpr_count", KD,
327 compute_pgm_rsrc2,
328 amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT);
329
332 OS, ".amdhsa_user_sgpr_private_segment_buffer", KD,
333 kernel_code_properties,
334 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER);
335 PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_ptr", KD,
336 kernel_code_properties,
337 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR);
338 PRINT_FIELD(OS, ".amdhsa_user_sgpr_queue_ptr", KD,
339 kernel_code_properties,
340 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR);
341 PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_segment_ptr", KD,
342 kernel_code_properties,
343 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR);
344 PRINT_FIELD(OS, ".amdhsa_user_sgpr_dispatch_id", KD,
345 kernel_code_properties,
346 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID);
348 PRINT_FIELD(OS, ".amdhsa_user_sgpr_flat_scratch_init", KD,
349 kernel_code_properties,
350 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT);
351 if (hasKernargPreload(STI)) {
352 PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_length ", KD,
353 kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_LENGTH);
354 PRINT_FIELD(OS, ".amdhsa_user_sgpr_kernarg_preload_offset ", KD,
355 kernarg_preload, amdhsa::KERNARG_PRELOAD_SPEC_OFFSET);
356 }
357 PRINT_FIELD(OS, ".amdhsa_user_sgpr_private_segment_size", KD,
358 kernel_code_properties,
359 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE);
360 if (IVersion.Major >= 10)
361 PRINT_FIELD(OS, ".amdhsa_wavefront_size32", KD,
362 kernel_code_properties,
363 amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32);
365 PRINT_FIELD(OS, ".amdhsa_uses_dynamic_stack", KD, kernel_code_properties,
366 amdhsa::KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK);
367 PRINT_FIELD(OS,
369 ? ".amdhsa_enable_private_segment"
370 : ".amdhsa_system_sgpr_private_segment_wavefront_offset"),
371 KD, compute_pgm_rsrc2,
372 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT);
373 PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_x", KD,
374 compute_pgm_rsrc2,
375 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X);
376 PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_y", KD,
377 compute_pgm_rsrc2,
378 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y);
379 PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_id_z", KD,
380 compute_pgm_rsrc2,
381 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z);
382 PRINT_FIELD(OS, ".amdhsa_system_sgpr_workgroup_info", KD,
383 compute_pgm_rsrc2,
384 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO);
385 PRINT_FIELD(OS, ".amdhsa_system_vgpr_workitem_id", KD,
386 compute_pgm_rsrc2,
387 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID);
388
389 // These directives are required.
390 OS << "\t\t.amdhsa_next_free_vgpr " << NextVGPR << '\n';
391 OS << "\t\t.amdhsa_next_free_sgpr " << NextSGPR << '\n';
392
393 if (AMDGPU::isGFX90A(STI))
394 OS << "\t\t.amdhsa_accum_offset " <<
396 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET) + 1) * 4
397 << '\n';
398
399 if (!ReserveVCC)
400 OS << "\t\t.amdhsa_reserve_vcc " << ReserveVCC << '\n';
401 if (IVersion.Major >= 7 && !ReserveFlatScr && !hasArchitectedFlatScratch(STI))
402 OS << "\t\t.amdhsa_reserve_flat_scratch " << ReserveFlatScr << '\n';
403
404 switch (CodeObjectVersion) {
405 default:
406 break;
409 if (getTargetID()->isXnackSupported())
410 OS << "\t\t.amdhsa_reserve_xnack_mask " << getTargetID()->isXnackOnOrAny() << '\n';
411 break;
412 }
413
414 PRINT_FIELD(OS, ".amdhsa_float_round_mode_32", KD,
415 compute_pgm_rsrc1,
416 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32);
417 PRINT_FIELD(OS, ".amdhsa_float_round_mode_16_64", KD,
418 compute_pgm_rsrc1,
419 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64);
420 PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_32", KD,
421 compute_pgm_rsrc1,
422 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32);
423 PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_16_64", KD,
424 compute_pgm_rsrc1,
425 amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64);
426 if (IVersion.Major < 12) {
427 PRINT_FIELD(OS, ".amdhsa_dx10_clamp", KD, compute_pgm_rsrc1,
428 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP);
429 PRINT_FIELD(OS, ".amdhsa_ieee_mode", KD, compute_pgm_rsrc1,
430 amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE);
431 }
432 if (IVersion.Major >= 9)
433 PRINT_FIELD(OS, ".amdhsa_fp16_overflow", KD,
434 compute_pgm_rsrc1,
435 amdhsa::COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL);
436 if (AMDGPU::isGFX90A(STI))
437 PRINT_FIELD(OS, ".amdhsa_tg_split", KD,
438 compute_pgm_rsrc3,
439 amdhsa::COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT);
440 if (IVersion.Major >= 10) {
441 PRINT_FIELD(OS, ".amdhsa_workgroup_processor_mode", KD,
442 compute_pgm_rsrc1,
443 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE);
444 PRINT_FIELD(OS, ".amdhsa_memory_ordered", KD,
445 compute_pgm_rsrc1,
446 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED);
447 PRINT_FIELD(OS, ".amdhsa_forward_progress", KD,
448 compute_pgm_rsrc1,
449 amdhsa::COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS);
450 }
451 if (IVersion.Major >= 10 && IVersion.Major < 12) {
452 PRINT_FIELD(OS, ".amdhsa_shared_vgpr_count", KD, compute_pgm_rsrc3,
453 amdhsa::COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT);
454 }
455 if (IVersion.Major >= 12)
456 PRINT_FIELD(OS, ".amdhsa_round_robin_scheduling", KD, compute_pgm_rsrc1,
457 amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN);
459 OS, ".amdhsa_exception_fp_ieee_invalid_op", KD,
460 compute_pgm_rsrc2,
461 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION);
462 PRINT_FIELD(OS, ".amdhsa_exception_fp_denorm_src", KD,
463 compute_pgm_rsrc2,
464 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE);
466 OS, ".amdhsa_exception_fp_ieee_div_zero", KD,
467 compute_pgm_rsrc2,
468 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO);
469 PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_overflow", KD,
470 compute_pgm_rsrc2,
471 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW);
472 PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_underflow", KD,
473 compute_pgm_rsrc2,
474 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW);
475 PRINT_FIELD(OS, ".amdhsa_exception_fp_ieee_inexact", KD,
476 compute_pgm_rsrc2,
477 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT);
478 PRINT_FIELD(OS, ".amdhsa_exception_int_div_zero", KD,
479 compute_pgm_rsrc2,
480 amdhsa::COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO);
481#undef PRINT_FIELD
482
483 OS << "\t.end_amdhsa_kernel\n";
484}
485
486//===----------------------------------------------------------------------===//
487// AMDGPUTargetELFStreamer
488//===----------------------------------------------------------------------===//
489
491 const MCSubtargetInfo &STI)
492 : AMDGPUTargetStreamer(S), STI(STI), Streamer(S) {}
493
495 return static_cast<MCELFStreamer &>(Streamer);
496}
497
498// A hook for emitting stuff at the end.
499// We use it for emitting the accumulated PAL metadata as a .note record.
500// The PAL metadata is reset after it is emitted.
503 MCA.setELFHeaderEFlags(getEFlags());
506
507 std::string Blob;
508 const char *Vendor = getPALMetadata()->getVendor();
509 unsigned Type = getPALMetadata()->getType();
510 getPALMetadata()->toBlob(Type, Blob);
511 if (Blob.empty())
512 return;
513 EmitNote(Vendor, MCConstantExpr::create(Blob.size(), getContext()), Type,
514 [&](MCELFStreamer &OS) { OS.emitBytes(Blob); });
515
516 // Reset the pal metadata so its data will not affect a compilation that
517 // reuses this object.
519}
520
521void AMDGPUTargetELFStreamer::EmitNote(
522 StringRef Name, const MCExpr *DescSZ, unsigned NoteType,
523 function_ref<void(MCELFStreamer &)> EmitDesc) {
524 auto &S = getStreamer();
525 auto &Context = S.getContext();
526
527 auto NameSZ = Name.size() + 1;
528
529 unsigned NoteFlags = 0;
530 // TODO Apparently, this is currently needed for OpenCL as mentioned in
531 // https://reviews.llvm.org/D74995
532 if (isHsaAbi(STI))
533 NoteFlags = ELF::SHF_ALLOC;
534
535 S.pushSection();
536 S.switchSection(
537 Context.getELFSection(ElfNote::SectionName, ELF::SHT_NOTE, NoteFlags));
538 S.emitInt32(NameSZ); // namesz
539 S.emitValue(DescSZ, 4); // descz
540 S.emitInt32(NoteType); // type
541 S.emitBytes(Name); // name
542 S.emitValueToAlignment(Align(4), 0, 1, 0); // padding 0
543 EmitDesc(S); // desc
544 S.emitValueToAlignment(Align(4), 0, 1, 0); // padding 0
545 S.popSection();
546}
547
548unsigned AMDGPUTargetELFStreamer::getEFlags() {
549 switch (STI.getTargetTriple().getArch()) {
550 default:
551 llvm_unreachable("Unsupported Arch");
552 case Triple::r600:
553 return getEFlagsR600();
554 case Triple::amdgcn:
555 return getEFlagsAMDGCN();
556 }
557}
558
559unsigned AMDGPUTargetELFStreamer::getEFlagsR600() {
561
562 return getElfMach(STI.getCPU());
563}
564
565unsigned AMDGPUTargetELFStreamer::getEFlagsAMDGCN() {
567
568 switch (STI.getTargetTriple().getOS()) {
569 default:
570 // TODO: Why are some tests have "mingw" listed as OS?
571 // llvm_unreachable("Unsupported OS");
573 return getEFlagsUnknownOS();
574 case Triple::AMDHSA:
575 return getEFlagsAMDHSA();
576 case Triple::AMDPAL:
577 return getEFlagsAMDPAL();
578 case Triple::Mesa3D:
579 return getEFlagsMesa3D();
580 }
581}
582
583unsigned AMDGPUTargetELFStreamer::getEFlagsUnknownOS() {
584 // TODO: Why are some tests have "mingw" listed as OS?
585 // assert(STI.getTargetTriple().getOS() == Triple::UnknownOS);
586
587 return getEFlagsV3();
588}
589
590unsigned AMDGPUTargetELFStreamer::getEFlagsAMDHSA() {
591 assert(isHsaAbi(STI));
592
593 if (CodeObjectVersion >= 6)
594 return getEFlagsV6();
595 return getEFlagsV4();
596}
597
598unsigned AMDGPUTargetELFStreamer::getEFlagsAMDPAL() {
600
601 return getEFlagsV3();
602}
603
604unsigned AMDGPUTargetELFStreamer::getEFlagsMesa3D() {
606
607 return getEFlagsV3();
608}
609
610unsigned AMDGPUTargetELFStreamer::getEFlagsV3() {
611 unsigned EFlagsV3 = 0;
612
613 // mach.
614 EFlagsV3 |= getElfMach(STI.getCPU());
615
616 // xnack.
617 if (getTargetID()->isXnackOnOrAny())
619 // sramecc.
620 if (getTargetID()->isSramEccOnOrAny())
622
623 return EFlagsV3;
624}
625
626unsigned AMDGPUTargetELFStreamer::getEFlagsV4() {
627 unsigned EFlagsV4 = 0;
628
629 // mach.
630 EFlagsV4 |= getElfMach(STI.getCPU());
631
632 // xnack.
633 switch (getTargetID()->getXnackSetting()) {
636 break;
639 break;
642 break;
645 break;
646 }
647 // sramecc.
648 switch (getTargetID()->getSramEccSetting()) {
651 break;
654 break;
657 break;
660 break;
661 }
662
663 return EFlagsV4;
664}
665
666unsigned AMDGPUTargetELFStreamer::getEFlagsV6() {
667 unsigned Flags = getEFlagsV4();
668
669 unsigned Version = ForceGenericVersion;
670 if (!Version) {
671 switch (parseArchAMDGCN(STI.getCPU())) {
674 break;
677 break;
680 break;
683 break;
684 default:
685 break;
686 }
687 }
688
689 // Versions start at 1.
690 if (Version) {
692 report_fatal_error("Cannot encode generic code object version " +
693 Twine(Version) +
694 " - no ELF flag can represent this version!");
696 }
697
698 return Flags;
699}
700
702
703void
705
707 OS.pushSection();
708 OS.emitBytes(StringRef((const char*)&Header, sizeof(Header)));
709 OS.popSection();
710}
711
713 unsigned Type) {
714 MCSymbolELF *Symbol = cast<MCSymbolELF>(
715 getStreamer().getContext().getOrCreateSymbol(SymbolName));
716 Symbol->setType(Type);
717}
718
720 Align Alignment) {
721 MCSymbolELF *SymbolELF = cast<MCSymbolELF>(Symbol);
722 SymbolELF->setType(ELF::STT_OBJECT);
723
724 if (!SymbolELF->isBindingSet()) {
725 SymbolELF->setBinding(ELF::STB_GLOBAL);
726 SymbolELF->setExternal(true);
727 }
728
729 if (SymbolELF->declareCommon(Size, Alignment, true)) {
730 report_fatal_error("Symbol: " + Symbol->getName() +
731 " redeclared as different type");
732 }
733
734 SymbolELF->setIndex(ELF::SHN_AMDGPU_LDS);
736}
737
739 // Create two labels to mark the beginning and end of the desc field
740 // and a MCExpr to calculate the size of the desc field.
741 auto &Context = getContext();
742 auto *DescBegin = Context.createTempSymbol();
743 auto *DescEnd = Context.createTempSymbol();
744 auto *DescSZ = MCBinaryExpr::createSub(
747
749 [&](MCELFStreamer &OS) {
750 OS.emitLabel(DescBegin);
751 OS.emitBytes(getTargetID()->toString());
752 OS.emitLabel(DescEnd);
753 });
754 return true;
755}
756
758 bool Strict) {
760 if (!Verifier.verify(HSAMetadataDoc.getRoot()))
761 return false;
762
763 std::string HSAMetadataString;
764 HSAMetadataDoc.writeToBlob(HSAMetadataString);
765
766 // Create two labels to mark the beginning and end of the desc field
767 // and a MCExpr to calculate the size of the desc field.
768 auto &Context = getContext();
769 auto *DescBegin = Context.createTempSymbol();
770 auto *DescEnd = Context.createTempSymbol();
771 auto *DescSZ = MCBinaryExpr::createSub(
774
776 [&](MCELFStreamer &OS) {
777 OS.emitLabel(DescBegin);
778 OS.emitBytes(HSAMetadataString);
779 OS.emitLabel(DescEnd);
780 });
781 return true;
782}
783
785 const MCSubtargetInfo &STI, bool TrapEnabled) {
786 const char *TrapInstr = TrapEnabled ? "\ts_trap 2" : "\ts_endpgm";
787 OS << TrapInstr
788 << " ; Trap with incompatible firmware that doesn't "
789 "support preloading kernel arguments.\n";
790 for (int i = 0; i < 63; ++i) {
791 OS << "\ts_nop 0\n";
792 }
793 return true;
794}
795
797 const MCSubtargetInfo &STI, bool TrapEnabled) {
798 const uint32_t Encoded_s_nop = 0xbf800000;
799 const uint32_t Encoded_s_trap = 0xbf920002;
800 const uint32_t Encoded_s_endpgm = 0xbf810000;
801 const uint32_t TrapInstr = TrapEnabled ? Encoded_s_trap : Encoded_s_endpgm;
803 OS.emitInt32(TrapInstr);
804 for (int i = 0; i < 63; ++i) {
805 OS.emitInt32(Encoded_s_nop);
806 }
807 return true;
808}
809
811 const uint32_t Encoded_s_code_end = 0xbf9f0000;
812 const uint32_t Encoded_s_nop = 0xbf800000;
813 uint32_t Encoded_pad = Encoded_s_code_end;
814
815 // Instruction cache line size in bytes.
816 const unsigned Log2CacheLineSize = AMDGPU::isGFX11Plus(STI) ? 7 : 6;
817 const unsigned CacheLineSize = 1u << Log2CacheLineSize;
818
819 // Extra padding amount in bytes to support prefetch mode 3.
820 unsigned FillSize = 3 * CacheLineSize;
821
822 if (AMDGPU::isGFX90A(STI)) {
823 Encoded_pad = Encoded_s_nop;
824 FillSize = 16 * CacheLineSize;
825 }
826
828 OS.pushSection();
829 OS.emitValueToAlignment(Align(CacheLineSize), Encoded_pad, 4);
830 for (unsigned I = 0; I < FillSize; I += 4)
831 OS.emitInt32(Encoded_pad);
832 OS.popSection();
833 return true;
834}
835
837 const MCSubtargetInfo &STI, StringRef KernelName,
838 const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR,
839 uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) {
840 auto &Streamer = getStreamer();
841 auto &Context = Streamer.getContext();
842
843 MCSymbolELF *KernelCodeSymbol = cast<MCSymbolELF>(
844 Context.getOrCreateSymbol(Twine(KernelName)));
845 MCSymbolELF *KernelDescriptorSymbol = cast<MCSymbolELF>(
846 Context.getOrCreateSymbol(Twine(KernelName) + Twine(".kd")));
847
848 // Copy kernel descriptor symbol's binding, other and visibility from the
849 // kernel code symbol.
850 KernelDescriptorSymbol->setBinding(KernelCodeSymbol->getBinding());
851 KernelDescriptorSymbol->setOther(KernelCodeSymbol->getOther());
852 KernelDescriptorSymbol->setVisibility(KernelCodeSymbol->getVisibility());
853 // Kernel descriptor symbol's type and size are fixed.
854 KernelDescriptorSymbol->setType(ELF::STT_OBJECT);
855 KernelDescriptorSymbol->setSize(
856 MCConstantExpr::create(sizeof(KernelDescriptor), Context));
857
858 // The visibility of the kernel code symbol must be protected or less to allow
859 // static relocations from the kernel descriptor to be used.
860 if (KernelCodeSymbol->getVisibility() == ELF::STV_DEFAULT)
861 KernelCodeSymbol->setVisibility(ELF::STV_PROTECTED);
862
863 Streamer.emitLabel(KernelDescriptorSymbol);
864 Streamer.emitInt32(KernelDescriptor.group_segment_fixed_size);
865 Streamer.emitInt32(KernelDescriptor.private_segment_fixed_size);
866 Streamer.emitInt32(KernelDescriptor.kernarg_size);
867
868 for (uint8_t Res : KernelDescriptor.reserved0)
869 Streamer.emitInt8(Res);
870
871 // FIXME: Remove the use of VK_AMDGPU_REL64 in the expression below. The
872 // expression being created is:
873 // (start of kernel code) - (start of kernel descriptor)
874 // It implies R_AMDGPU_REL64, but ends up being R_AMDGPU_ABS64.
877 KernelCodeSymbol, MCSymbolRefExpr::VK_AMDGPU_REL64, Context),
879 KernelDescriptorSymbol, MCSymbolRefExpr::VK_None, Context),
880 Context),
881 sizeof(KernelDescriptor.kernel_code_entry_byte_offset));
882 for (uint8_t Res : KernelDescriptor.reserved1)
883 Streamer.emitInt8(Res);
884 Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc3);
885 Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc1);
886 Streamer.emitInt32(KernelDescriptor.compute_pgm_rsrc2);
887 Streamer.emitInt16(KernelDescriptor.kernel_code_properties);
888 Streamer.emitInt16(KernelDescriptor.kernarg_preload);
889 for (uint8_t Res : KernelDescriptor.reserved3)
890 Streamer.emitInt8(Res);
891}
This is a verifier for AMDGPU HSA metadata, which can verify both well-typed metadata and untyped met...
AMDGPU metadata definitions and in-memory representations.
Enums and constants for AMDGPU PT_NOTE sections.
static cl::opt< unsigned > ForceGenericVersion("amdgpu-force-generic-version", cl::desc("Force a specific generic_v<N> flag to be " "added. For testing purposes only."), cl::ReallyHidden, cl::init(0))
#define PRINT_FIELD(STREAM, DIRECTIVE, KERNEL_DESC, MEMBER_NAME, FIELD_NAME)
AMDHSA kernel descriptor definitions.
#define AMDHSA_BITS_GET(SRC, MSK)
std::string Name
uint64_t Size
#define I(x, y, z)
Definition: MD5.cpp:58
LLVMContext & Context
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
verify safepoint Safepoint IR Verifier
raw_pwrite_stream & OS
static cl::opt< unsigned > CacheLineSize("cache-line-size", cl::init(0), cl::Hidden, cl::desc("Use this to override the target cache line size when " "specified by the user."))
const char * getVendor() const
void toBlob(unsigned Type, std::string &S)
void toString(std::string &S)
void EmitAMDKernelCodeT(const amd_kernel_code_t &Header) override
AMDGPUTargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS)
bool EmitHSAMetadata(msgpack::Document &HSAMetadata, bool Strict) override
void EmitAMDGPUSymbolType(StringRef SymbolName, unsigned Type) override
void EmitDirectiveAMDHSACodeObjectVersion(unsigned COV) override
void EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override
bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override
bool EmitCodeEnd(const MCSubtargetInfo &STI) override
void emitAMDGPULDS(MCSymbol *Sym, unsigned Size, Align Alignment) override
void EmitAmdhsaKernelDescriptor(const MCSubtargetInfo &STI, StringRef KernelName, const amdhsa::kernel_descriptor_t &KernelDescriptor, uint64_t NextVGPR, uint64_t NextSGPR, bool ReserveVCC, bool ReserveFlatScr) override
bool EmitCodeEnd(const MCSubtargetInfo &STI) override
void EmitAMDKernelCodeT(const amd_kernel_code_t &Header) override
bool EmitHSAMetadata(msgpack::Document &HSAMetadata, bool Strict) override
AMDGPUTargetELFStreamer(MCStreamer &S, const MCSubtargetInfo &STI)
void emitAMDGPULDS(MCSymbol *Sym, unsigned Size, Align Alignment) override
void EmitAMDGPUSymbolType(StringRef SymbolName, unsigned Type) override
bool EmitKernargPreloadHeader(const MCSubtargetInfo &STI, bool TrapEnabled) override
virtual bool EmitHSAMetadata(msgpack::Document &HSAMetadata, bool Strict)
Emit HSA Metadata.
AMDGPUPALMetadata * getPALMetadata()
virtual void EmitDirectiveAMDHSACodeObjectVersion(unsigned COV)
virtual bool EmitHSAMetadataV3(StringRef HSAMetadataString)
static unsigned getElfMach(StringRef GPU)
MCContext & getContext() const
static StringRef getArchNameFromElfMach(unsigned ElfMach)
const std::optional< AMDGPU::IsaInfo::AMDGPUTargetID > & getTargetID() const
MCObjectWriter & getWriter() const
Definition: MCAssembler.h:338
void setELFHeaderEFlags(unsigned Flags)
Definition: MCAssembler.h:282
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:621
static const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Definition: MCExpr.cpp:194
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:35
MCAssembler & getAssembler()
virtual void setOverrideABIVersion(uint8_t ABIVersion)
ELF only, override the default ABIVersion in the ELF header.
Streaming machine code generation interface.
Definition: MCStreamer.h:212
MCContext & getContext() const
Definition: MCStreamer.h:297
void emitValue(const MCExpr *Value, unsigned Size, SMLoc Loc=SMLoc())
Definition: MCStreamer.cpp:180
virtual void emitLabel(MCSymbol *Symbol, SMLoc Loc=SMLoc())
Emit a label for Symbol into the current section.
Definition: MCStreamer.cpp:424
void emitInt16(uint64_t Value)
Definition: MCStreamer.h:753
void emitInt32(uint64_t Value)
Definition: MCStreamer.h:754
void emitInt8(uint64_t Value)
Definition: MCStreamer.h:752
Generic base class for all target subtargets.
const Triple & getTargetTriple() const
StringRef getCPU() const
unsigned getOther() const
void setVisibility(unsigned Visibility)
void setSize(const MCExpr *SS)
Definition: MCSymbolELF.h:22
bool isBindingSet() const
void setBinding(unsigned Binding) const
Definition: MCSymbolELF.cpp:43
unsigned getVisibility() const
unsigned getBinding() const
Definition: MCSymbolELF.cpp:66
void setType(unsigned Type) const
Definition: MCSymbolELF.cpp:94
void setOther(unsigned Other)
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:397
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
void setExternal(bool Value) const
Definition: MCSymbol.h:407
void setIndex(uint32_t Value) const
Set the (implementation defined) index.
Definition: MCSymbol.h:321
bool declareCommon(uint64_t Size, Align Alignment, bool Target=false)
Declare this symbol as being 'common'.
Definition: MCSymbol.h:375
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
OSType getOS() const
Get the parsed operating system type of this triple.
Definition: Triple.h:370
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:361
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
formatted_raw_ostream - A raw_ostream that wraps another one and keeps track of line and column posit...
An efficient, type-erasing, non-owning reference to a callable.
Simple in-memory representation of a document of msgpack objects with ability to find and create arra...
DocNode & getRoot()
Get ref to the document's root element.
void toYAML(raw_ostream &OS)
Convert MsgPack Document to YAML text.
void writeToBlob(std::string &Blob)
Write a MsgPack document to a binary MsgPack blob.
bool fromYAML(StringRef S)
Read YAML text into the MsgPack document. Returns false on failure.
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:678
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const char NoteNameV2[]
Definition: AMDGPUPTNote.h:26
const char SectionName[]
Definition: AMDGPUPTNote.h:24
const char NoteNameV3[]
Definition: AMDGPUPTNote.h:27
static constexpr unsigned GFX10_1
static constexpr unsigned GFX10_3
static constexpr unsigned GFX11
static constexpr unsigned GFX9
constexpr char AssemblerDirectiveBegin[]
HSA metadata beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
HSA metadata ending assembler directive.
StringRef getArchNameR600(GPUKind AK)
GPUKind
GPU kinds supported by the AMDGPU target.
Definition: TargetParser.h:35
bool isHsaAbi(const MCSubtargetInfo &STI)
IsaVersion getIsaVersion(StringRef GPU)
bool isGFX90A(const MCSubtargetInfo &STI)
GPUKind parseArchAMDGCN(StringRef CPU)
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI)
bool isGFX11Plus(const MCSubtargetInfo &STI)
StringRef getArchNameAMDGCN(GPUKind AK)
unsigned hasKernargPreload(const MCSubtargetInfo &STI)
uint8_t getELFABIVersion(const Triple &T, unsigned CodeObjectVersion)
GPUKind parseArchR600(StringRef CPU)
@ SHF_ALLOC
Definition: ELF.h:1155
@ STV_PROTECTED
Definition: ELF.h:1341
@ STV_DEFAULT
Definition: ELF.h:1338
@ SHN_AMDGPU_LDS
Definition: ELF.h:1832
@ EF_AMDGPU_GENERIC_VERSION_MAX
Definition: ELF.h:857
@ EF_AMDGPU_FEATURE_XNACK_ANY_V4
Definition: ELF.h:834
@ EF_AMDGPU_MACH_AMDGCN_GFX703
Definition: ELF.h:750
@ EF_AMDGPU_MACH_AMDGCN_GFX1035
Definition: ELF.h:774
@ EF_AMDGPU_FEATURE_SRAMECC_V3
Definition: ELF.h:825
@ EF_AMDGPU_MACH_AMDGCN_GFX1031
Definition: ELF.h:768
@ EF_AMDGPU_GENERIC_VERSION_OFFSET
Definition: ELF.h:855
@ EF_AMDGPU_MACH_R600_CAYMAN
Definition: ELF.h:732
@ EF_AMDGPU_FEATURE_SRAMECC_UNSUPPORTED_V4
Definition: ELF.h:845
@ EF_AMDGPU_MACH_AMDGCN_GFX704
Definition: ELF.h:751
@ EF_AMDGPU_MACH_AMDGCN_GFX902
Definition: ELF.h:758
@ EF_AMDGPU_MACH_AMDGCN_GFX810
Definition: ELF.h:756
@ EF_AMDGPU_MACH_AMDGCN_GFX1036
Definition: ELF.h:782
@ EF_AMDGPU_MACH_AMDGCN_GFX1102
Definition: ELF.h:784
@ EF_AMDGPU_MACH_R600_RV730
Definition: ELF.h:721
@ EF_AMDGPU_MACH_R600_RV710
Definition: ELF.h:720
@ EF_AMDGPU_MACH_AMDGCN_GFX908
Definition: ELF.h:761
@ EF_AMDGPU_MACH_AMDGCN_GFX1011
Definition: ELF.h:765
@ EF_AMDGPU_MACH_R600_CYPRESS
Definition: ELF.h:725
@ EF_AMDGPU_MACH_AMDGCN_GFX1032
Definition: ELF.h:769
@ EF_AMDGPU_MACH_R600_R600
Definition: ELF.h:715
@ EF_AMDGPU_MACH_AMDGCN_GFX940
Definition: ELF.h:777
@ EF_AMDGPU_MACH_AMDGCN_GFX941
Definition: ELF.h:788
@ EF_AMDGPU_MACH_R600_TURKS
Definition: ELF.h:733
@ EF_AMDGPU_MACH_R600_JUNIPER
Definition: ELF.h:726
@ EF_AMDGPU_FEATURE_SRAMECC_OFF_V4
Definition: ELF.h:849
@ EF_AMDGPU_FEATURE_XNACK_UNSUPPORTED_V4
Definition: ELF.h:832
@ EF_AMDGPU_MACH_AMDGCN_GFX601
Definition: ELF.h:746
@ EF_AMDGPU_MACH_AMDGCN_GFX942
Definition: ELF.h:789
@ EF_AMDGPU_MACH_R600_R630
Definition: ELF.h:716
@ EF_AMDGPU_MACH_R600_REDWOOD
Definition: ELF.h:727
@ EF_AMDGPU_MACH_R600_RV770
Definition: ELF.h:722
@ EF_AMDGPU_FEATURE_XNACK_OFF_V4
Definition: ELF.h:836
@ EF_AMDGPU_MACH_AMDGCN_GFX600
Definition: ELF.h:745
@ EF_AMDGPU_FEATURE_XNACK_V3
Definition: ELF.h:820
@ EF_AMDGPU_MACH_AMDGCN_GFX602
Definition: ELF.h:771
@ EF_AMDGPU_MACH_AMDGCN_GFX1101
Definition: ELF.h:783
@ EF_AMDGPU_MACH_AMDGCN_GFX1100
Definition: ELF.h:778
@ EF_AMDGPU_MACH_AMDGCN_GFX1033
Definition: ELF.h:770
@ EF_AMDGPU_MACH_AMDGCN_GFX801
Definition: ELF.h:753
@ EF_AMDGPU_MACH_AMDGCN_GFX705
Definition: ELF.h:772
@ EF_AMDGPU_MACH_AMDGCN_GFX1010
Definition: ELF.h:764
@ EF_AMDGPU_MACH_R600_RV670
Definition: ELF.h:718
@ EF_AMDGPU_MACH_AMDGCN_GFX701
Definition: ELF.h:748
@ EF_AMDGPU_MACH_AMDGCN_GFX10_3_GENERIC
Definition: ELF.h:796
@ EF_AMDGPU_MACH_AMDGCN_GFX1012
Definition: ELF.h:766
@ EF_AMDGPU_MACH_AMDGCN_GFX1151
Definition: ELF.h:787
@ EF_AMDGPU_MACH_AMDGCN_GFX1030
Definition: ELF.h:767
@ EF_AMDGPU_MACH_R600_CEDAR
Definition: ELF.h:724
@ EF_AMDGPU_MACH_AMDGCN_GFX1200
Definition: ELF.h:785
@ EF_AMDGPU_MACH_AMDGCN_GFX700
Definition: ELF.h:747
@ EF_AMDGPU_MACH_AMDGCN_GFX11_GENERIC
Definition: ELF.h:797
@ EF_AMDGPU_MACH_AMDGCN_GFX803
Definition: ELF.h:755
@ EF_AMDGPU_MACH_AMDGCN_GFX802
Definition: ELF.h:754
@ EF_AMDGPU_MACH_AMDGCN_GFX90C
Definition: ELF.h:763
@ EF_AMDGPU_FEATURE_XNACK_ON_V4
Definition: ELF.h:838
@ EF_AMDGPU_MACH_AMDGCN_GFX900
Definition: ELF.h:757
@ EF_AMDGPU_MACH_AMDGCN_GFX909
Definition: ELF.h:762
@ EF_AMDGPU_MACH_AMDGCN_GFX906
Definition: ELF.h:760
@ EF_AMDGPU_MACH_NONE
Definition: ELF.h:710
@ EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC
Definition: ELF.h:794
@ EF_AMDGPU_MACH_AMDGCN_GFX1103
Definition: ELF.h:781
@ EF_AMDGPU_MACH_R600_CAICOS
Definition: ELF.h:731
@ EF_AMDGPU_MACH_AMDGCN_GFX90A
Definition: ELF.h:776
@ EF_AMDGPU_MACH_AMDGCN_GFX1034
Definition: ELF.h:775
@ EF_AMDGPU_MACH_AMDGCN_GFX1013
Definition: ELF.h:779
@ EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC
Definition: ELF.h:795
@ EF_AMDGPU_MACH_AMDGCN_GFX904
Definition: ELF.h:759
@ EF_AMDGPU_MACH_R600_RS880
Definition: ELF.h:717
@ EF_AMDGPU_MACH_AMDGCN_GFX805
Definition: ELF.h:773
@ EF_AMDGPU_MACH_AMDGCN_GFX1201
Definition: ELF.h:791
@ EF_AMDGPU_MACH_AMDGCN_GFX1150
Definition: ELF.h:780
@ EF_AMDGPU_MACH_R600_SUMO
Definition: ELF.h:728
@ EF_AMDGPU_MACH_R600_BARTS
Definition: ELF.h:730
@ EF_AMDGPU_FEATURE_SRAMECC_ANY_V4
Definition: ELF.h:847
@ EF_AMDGPU_FEATURE_SRAMECC_ON_V4
Definition: ELF.h:851
@ EF_AMDGPU_MACH_AMDGCN_GFX702
Definition: ELF.h:749
@ SHT_NOTE
Definition: ELF.h:1069
@ NT_AMDGPU_METADATA
Definition: ELF.h:1849
@ STB_GLOBAL
Definition: ELF.h:1309
@ NT_AMD_HSA_ISA_NAME
Definition: ELF.h:1842
@ STT_AMDGPU_HSA_KERNEL
Definition: ELF.h:1334
@ STT_OBJECT
Definition: ELF.h:1321
const uint64_t Version
Definition: InstrProf.h:1047
@ ReallyHidden
Definition: CommandLine.h:139
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
void dumpAmdKernelCode(const amd_kernel_code_t *C, raw_ostream &OS, const char *tab)
AMD Kernel Code Object (amd_kernel_code_t).
Instruction set architecture version.
Definition: TargetParser.h:125
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85