70 enum KindTy { Token, Immediate, Register, Expression } Kind;
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
76 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
87 bool hasFPModifiers()
const {
return Abs || Neg; }
88 bool hasIntModifiers()
const {
return Sext; }
89 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
91 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
93 int64_t getFPModifiersOperand()
const {
100 int64_t getIntModifiersOperand()
const {
106 int64_t getModifiersOperand()
const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
116 friend raw_ostream &
operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
191 ImmTyMatrixAScaleFmt,
192 ImmTyMatrixBScaleFmt,
225 mutable int MCOpIdx = -1;
228 bool isToken()
const override {
return Kind == Token; }
230 bool isSymbolRefExpr()
const {
234 bool isImm()
const override {
return Kind == Immediate; }
236 bool isInlinableImm(MVT type)
const;
237 bool isLiteralImm(MVT type)
const;
239 bool isRegKind()
const {
return Kind == Register; }
241 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
243 bool isRegOrInline(
unsigned RCID, MVT type)
const {
244 return isRegClass(RCID) || isInlinableImm(type);
247 bool isRegOrInlineTarget(
unsigned TargetRCIdx, MVT type)
const {
248 return isRegClassTarget(TargetRCIdx) || isInlinableImm(type);
252 return isRegOrInline(RCID, type) || isLiteralImm(type);
255 bool isRegOrImmWithInputModsTarget(
unsigned TargetRCIdx, MVT type)
const {
256 return isRegOrInlineTarget(TargetRCIdx, type) || isLiteralImm(type);
259 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
261 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
264 bool isRegOrImmWithInt32InputMods()
const {
268 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
269 return isRegOrInline(
270 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
273 bool isRegOrInlineImmWithInt32InputMods()
const {
274 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
277 bool isRegOrImmWithInt64InputMods()
const {
278 return isRegOrImmWithInputModsTarget(AMDGPU::VS_64_AlignTarget, MVT::i64);
281 bool isRegOrImmWithFP16InputMods()
const {
285 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
287 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
290 bool isRegOrImmWithFPT16_LO16InputMods()
const {
294 bool isRegOrImmWithFP32InputMods()
const {
298 bool isRegOrImmWithFP64InputMods()
const {
299 return isRegOrImmWithInputModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
302 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
303 return isRegOrInline(
304 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
307 bool isRegOrInlineImmWithFP32InputMods()
const {
308 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
311 bool isRegOrInlineImmWithFP64InputMods()
const {
312 return isRegOrInlineTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
315 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
317 bool isVRegWithFP32InputMods()
const {
318 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
321 bool isVRegWithFP64InputMods()
const {
322 return isRegClassTarget(AMDGPU::VReg_64_AlignTarget);
325 bool isPackedFP16InputMods()
const {
329 bool isPackedVGPRFP32InputMods()
const {
333 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
335 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
337 bool isRsrcReg32()
const {
return isRegClass(AMDGPU::RsrcReg32RegClassID); }
339 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
341 bool isAV_LdSt_32_Align2_RegOp()
const {
342 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
343 isRegClass(AMDGPU::AGPR_32RegClassID);
346 bool isVRegWithInputMods()
const;
347 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
348 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
349 bool isT16_LO16VRegWithInputMods()
const;
351 bool isSDWAOperand(MVT type)
const;
352 bool isSDWAFP16Operand()
const;
353 bool isSDWAFP32Operand()
const;
354 bool isSDWAInt16Operand()
const;
355 bool isSDWAInt32Operand()
const;
357 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
359 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
361 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
363 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
365 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
366 bool isDim()
const {
return isImmTy(ImmTyDim); }
367 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
368 bool isOff()
const {
return isImmTy(ImmTyOff); }
369 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
370 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
371 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
372 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
373 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
374 bool isFlatOffset()
const {
375 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
377 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
378 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
379 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
380 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
381 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
382 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
383 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
384 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
385 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
386 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
387 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
388 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
389 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
390 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
391 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
392 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
393 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
394 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
395 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
396 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
397 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
398 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
399 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
400 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
401 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
402 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
403 bool isDone()
const {
return isImmTy(ImmTyDone); }
404 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
406 bool isRegOrImm()
const {
return isReg() || isImm(); }
408 bool isRegClass(
unsigned RCID)
const;
411 bool isRegClassTarget(
unsigned TargetRCIdx)
const;
415 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
416 return isRegOrInline(RCID, type) && !hasModifiers();
419 bool isRegOrInlineNoModsTarget(
unsigned TargetRCIdx, MVT type)
const {
420 return isRegOrInlineTarget(TargetRCIdx, type) && !hasModifiers();
423 bool isSCSrcB16()
const {
424 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
427 bool isSCSrc_b32()
const {
428 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
431 bool isSCSrc_b64()
const {
432 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
435 bool isBoolReg()
const;
437 bool isSSrc_b32()
const {
438 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
441 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
443 bool isSSrc_b64()
const {
446 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
447 (((
const MCTargetAsmParser *)AsmParser)
448 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
452 bool isSSrc_f32()
const {
453 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
456 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
458 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
460 bool isSSrc_NoInline_f16()
const {
return isSSrc_f16(); }
462 bool isSSrcOrLds_b32()
const {
463 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
464 isLiteralImm(MVT::i32) || isExpr();
467 bool isVCSrc_b32()
const {
468 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
471 bool isVCSrc_b32_Lo256()
const {
472 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
475 bool isVCSrc_b64_Lo256()
const {
476 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
479 bool isVCSrc_b64()
const {
480 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::i64);
483 bool isVCSrcT_b16()
const {
484 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
487 bool isVCSrcTB16_Lo128()
const {
488 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
491 bool isVCSrcFake16B16_Lo128()
const {
492 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
495 bool isVCSrc_b16()
const {
496 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
499 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
501 bool isVCSrc_f32()
const {
502 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
505 bool isVCSrc_f64()
const {
506 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
509 bool isVCSrcTBF16()
const {
510 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
513 bool isVCSrcT_f16()
const {
514 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
517 bool isVCSrcT_bf16()
const {
518 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
521 bool isVCSrcTF16_LO16()
const {
522 return isRegOrInlineNoMods(AMDGPU::VS_16_LO16RegClassID, MVT::f16);
525 bool isVCSrcTBF16_Lo128()
const {
526 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
529 bool isVCSrcTF16_Lo128()
const {
530 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
533 bool isVCSrcFake16BF16_Lo128()
const {
534 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
537 bool isVCSrcFake16F16_Lo128()
const {
538 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
541 bool isVCSrc_bf16()
const {
542 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
545 bool isVCSrc_f16()
const {
546 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
549 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
551 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
553 bool isVSrc_b32()
const {
554 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
557 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
559 bool isVSrc_v2b64()
const {
560 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
561 isLiteralImm(MVT::i64);
564 bool isVSrc_v2f64()
const {
565 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
566 isLiteralImm(MVT::f64);
569 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
571 bool isVSrcT_b16_Lo128()
const {
572 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
575 bool isVSrcFake16_b16_Lo128()
const {
576 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
579 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
581 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
583 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
585 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
587 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
589 bool isVSrc_f32()
const {
590 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
593 bool isVSrc_f64()
const {
594 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64) ||
595 isLiteralImm(MVT::f64);
598 bool isVSrcT_bf16()
const {
599 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
602 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
604 bool isVSrcT_f16_LO16()
const {
605 return isVCSrcTF16_LO16() || isLiteralImm(MVT::f16);
608 bool isVSrcT_bf16_Lo128()
const {
609 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
612 bool isVSrcT_f16_Lo128()
const {
613 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
616 bool isVSrcFake16_bf16_Lo128()
const {
617 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
620 bool isVSrcFake16_f16_Lo128()
const {
621 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
624 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
626 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
628 bool isVSrc_v2bf16()
const {
629 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
632 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
634 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
636 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
638 bool isVISrc_64_bf16()
const {
639 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::bf16);
642 bool isVISrc_64_f16()
const {
643 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::f16);
646 bool isVISrc_64_b32()
const {
647 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::i32);
650 bool isVISrc_64_f64()
const {
651 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::f64);
654 bool isVISrc_256_b32()
const {
655 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::i32);
658 bool isVISrc_256_f32()
const {
659 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::f32);
662 bool isVISrc_256_f64()
const {
663 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::f64);
666 bool isVISrc_512_f64()
const {
667 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::f64);
670 bool isVISrc_128_b32()
const {
671 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::i32);
674 bool isVISrc_128_f32()
const {
675 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::f32);
678 bool isVISrc_512_b32()
const {
679 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::i32);
682 bool isVISrc_512_f32()
const {
683 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::f32);
686 bool isVISrc_1024_b32()
const {
687 return isRegOrInlineNoModsTarget(AMDGPU::VReg_1024_AlignTarget, MVT::i32);
690 bool isVISrc_1024_f32()
const {
691 return isRegOrInlineNoModsTarget(AMDGPU::VReg_1024_AlignTarget, MVT::f32);
694 bool isAISrc_64_f64()
const {
695 return isRegOrInlineNoModsTarget(AMDGPU::AReg_64_AlignTarget, MVT::f64);
698 bool isAISrc_128_b32()
const {
699 return isRegOrInlineNoModsTarget(AMDGPU::AReg_128_AlignTarget, MVT::i32);
702 bool isAISrc_128_f32()
const {
703 return isRegOrInlineNoModsTarget(AMDGPU::AReg_128_AlignTarget, MVT::f32);
706 bool isVISrc_128_bf16()
const {
707 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::bf16);
710 bool isVISrc_128_f16()
const {
711 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::f16);
714 bool isAISrc_256_f64()
const {
715 return isRegOrInlineNoModsTarget(AMDGPU::AReg_256_AlignTarget, MVT::f64);
718 bool isAISrc_512_b32()
const {
719 return isRegOrInlineNoModsTarget(AMDGPU::AReg_512_AlignTarget, MVT::i32);
722 bool isAISrc_512_f32()
const {
723 return isRegOrInlineNoModsTarget(AMDGPU::AReg_512_AlignTarget, MVT::f32);
726 bool isAISrc_1024_b32()
const {
727 return isRegOrInlineNoModsTarget(AMDGPU::AReg_1024_AlignTarget, MVT::i32);
730 bool isAISrc_1024_f32()
const {
731 return isRegOrInlineNoModsTarget(AMDGPU::AReg_1024_AlignTarget, MVT::f32);
734 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
736 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
738 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
740 bool isMem()
const override {
return false; }
742 bool isExpr()
const {
return Kind == Expression; }
744 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
746 bool isSWaitCnt()
const;
747 bool isDepCtr()
const;
748 bool isSDelayALU()
const;
749 bool isHwreg()
const;
750 bool isSendMsg()
const;
751 bool isWaitEvent()
const;
752 bool isSplitBarrier()
const;
753 bool isSwizzle()
const;
754 bool isSMRDOffset8()
const;
755 bool isSMEMOffset()
const;
756 bool isSMRDLiteralOffset()
const;
758 bool isDPPCtrl()
const;
760 bool isGPRIdxMode()
const;
761 bool isS16Imm()
const;
762 bool isU16Imm()
const;
763 bool isEndpgm()
const;
765 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
766 return [
this,
P]() {
return P(*
this); };
771 return StringRef(Tok.Data, Tok.Length);
779 void setImm(int64_t Val) {
784 ImmTy getImmTy()
const {
789 MCRegister
getReg()
const override {
794 SMLoc getStartLoc()
const override {
return StartLoc; }
796 SMLoc getEndLoc()
const override {
return EndLoc; }
798 int getMCOpIdx()
const {
return MCOpIdx; }
800 Modifiers getModifiers()
const {
801 assert(isRegKind() || isImmTy(ImmTyNone));
802 return isRegKind() ?
Reg.Mods :
Imm.Mods;
805 void setModifiers(Modifiers Mods) {
806 assert(isRegKind() || isImmTy(ImmTyNone));
813 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
815 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
817 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
819 bool isForcedLit()
const {
820 return isImmLiteral() && getModifiers().isForcedLit();
823 bool isForcedLit64()
const {
824 return isImmLiteral() && getModifiers().isForcedLit64();
829 void addImmOperands(MCInst &Inst,
unsigned N,
830 bool ApplyModifiers =
true)
const;
832 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
833 bool ApplyModifiers)
const;
835 void addRegOperands(MCInst &Inst,
unsigned N)
const;
837 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
839 addRegOperands(Inst,
N);
841 addImmOperands(Inst,
N);
844 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
845 Modifiers Mods = getModifiers();
848 addRegOperands(Inst,
N);
850 addImmOperands(Inst,
N,
false);
854 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
855 assert(!hasIntModifiers());
856 addRegOrImmWithInputModsOperands(Inst,
N);
859 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
860 Modifiers Mods = getModifiers();
863 addRegOperands(Inst,
N);
866 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
867 assert(!hasIntModifiers());
868 addRegWithInputModsOperands(Inst,
N);
871 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
874 case ImmTyNone: OS <<
"None";
break;
875 case ImmTyGDS: OS <<
"GDS";
break;
876 case ImmTyLDS: OS <<
"LDS";
break;
877 case ImmTyOffen: OS <<
"Offen";
break;
878 case ImmTyIdxen: OS <<
"Idxen";
break;
879 case ImmTyAddr64: OS <<
"Addr64";
break;
880 case ImmTyOffset: OS <<
"Offset";
break;
881 case ImmTyInstOffset: OS <<
"InstOffset";
break;
882 case ImmTyOffset0: OS <<
"Offset0";
break;
883 case ImmTyOffset1: OS <<
"Offset1";
break;
884 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
885 case ImmTyCPol: OS <<
"CPol";
break;
886 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
887 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
888 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
889 case ImmTyTFE: OS <<
"TFE";
break;
890 case ImmTyIsAsync: OS <<
"IsAsync";
break;
891 case ImmTyD16: OS <<
"D16";
break;
892 case ImmTyFORMAT: OS <<
"FORMAT";
break;
893 case ImmTyClamp: OS <<
"Clamp";
break;
894 case ImmTyOModSI: OS <<
"OModSI";
break;
895 case ImmTyDPP8: OS <<
"DPP8";
break;
896 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
897 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
898 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
899 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
900 case ImmTyDppFI: OS <<
"DppFI";
break;
901 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
902 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
903 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
904 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
905 case ImmTyDMask: OS <<
"DMask";
break;
906 case ImmTyDim: OS <<
"Dim";
break;
907 case ImmTyUNorm: OS <<
"UNorm";
break;
908 case ImmTyDA: OS <<
"DA";
break;
909 case ImmTyR128A16: OS <<
"R128A16";
break;
910 case ImmTyA16: OS <<
"A16";
break;
911 case ImmTyLWE: OS <<
"LWE";
break;
912 case ImmTyOff: OS <<
"Off";
break;
913 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
914 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
915 case ImmTyExpVM: OS <<
"ExpVM";
break;
916 case ImmTyDone: OS <<
"Done";
break;
917 case ImmTyRowEn: OS <<
"RowEn";
break;
918 case ImmTyHwreg: OS <<
"Hwreg";
break;
919 case ImmTySendMsg: OS <<
"SendMsg";
break;
920 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
921 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
922 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
923 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
924 case ImmTyOpSel: OS <<
"OpSel";
break;
925 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
926 case ImmTyNegLo: OS <<
"NegLo";
break;
927 case ImmTyNegHi: OS <<
"NegHi";
break;
928 case ImmTySwizzle: OS <<
"Swizzle";
break;
929 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
930 case ImmTyHigh: OS <<
"High";
break;
931 case ImmTyBLGP: OS <<
"BLGP";
break;
932 case ImmTyCBSZ: OS <<
"CBSZ";
break;
933 case ImmTyABID: OS <<
"ABID";
break;
934 case ImmTyEndpgm: OS <<
"Endpgm";
break;
935 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
936 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
937 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
938 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
939 case ImmTyBitOp3: OS <<
"BitOp3";
break;
940 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
941 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
942 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
943 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
944 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
945 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
946 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
947 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
948 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
949 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
954 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
958 <<
" mods: " <<
Reg.Mods <<
'>';
962 if (getImmTy() != ImmTyNone) {
964 printImmTy(OS, getImmTy());
966 OS <<
" mods: " <<
Imm.Mods <<
'>';
979 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
980 int64_t Val, SMLoc Loc,
981 ImmTy
Type = ImmTyNone,
982 bool IsFPImm =
false) {
983 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
985 Op->Imm.IsFPImm = IsFPImm;
987 Op->Imm.Mods = Modifiers();
993 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
994 StringRef Str, SMLoc Loc,
995 bool HasExplicitEncodingSize =
true) {
996 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
997 Res->Tok.Data = Str.data();
998 Res->Tok.Length = Str.size();
1004 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1005 MCRegister
Reg, SMLoc S, SMLoc
E) {
1006 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1007 Op->Reg.RegNo =
Reg;
1008 Op->Reg.Mods = Modifiers();
1014 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1015 const class MCExpr *Expr, SMLoc S) {
1016 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1025 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1034#define GET_REGISTER_MATCHER
1035#include "AMDGPUGenAsmMatcher.inc"
1036#undef GET_REGISTER_MATCHER
1037#undef GET_SUBTARGET_FEATURE_NAME
1042class KernelScopeInfo {
1043 int SgprIndexUnusedMin = -1;
1044 int VgprIndexUnusedMin = -1;
1045 int AgprIndexUnusedMin = -1;
1049 void usesSgprAt(
int i) {
1050 if (i >= SgprIndexUnusedMin) {
1051 SgprIndexUnusedMin = ++i;
1054 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1060 void usesVgprAt(
int i) {
1061 if (i >= VgprIndexUnusedMin) {
1062 VgprIndexUnusedMin = ++i;
1065 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1067 VgprIndexUnusedMin);
1073 void usesAgprAt(
int i) {
1078 if (i >= AgprIndexUnusedMin) {
1079 AgprIndexUnusedMin = ++i;
1082 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1087 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1089 VgprIndexUnusedMin);
1096 KernelScopeInfo() =
default;
1100 MSTI = Ctx->getSubtargetInfo();
1102 usesSgprAt(SgprIndexUnusedMin = -1);
1103 usesVgprAt(VgprIndexUnusedMin = -1);
1105 usesAgprAt(AgprIndexUnusedMin = -1);
1109 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1110 unsigned RegWidth) {
1113 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1116 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1119 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1128 MCAsmParser &Parser;
1130 unsigned ForcedEncodingSize = 0;
1131 bool ForcedDPP =
false;
1132 bool ForcedSDWA =
false;
1133 KernelScopeInfo KernelScope;
1134 const unsigned HwMode;
1136 const AMDGPU::IsaVersion ISA;
1141#define GET_ASSEMBLER_HEADER
1142#include "AMDGPUGenAsmMatcher.inc"
1147 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1149 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1153 std::optional<AMDGPU::InfoSectionData> InfoData;
1162 SmallVector<unsigned> OpcodeStream;
1164 OpcodeStreamSymbols;
1165 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1168 void checkKernelPrologues();
1171 void createConstantSymbol(StringRef Id, int64_t Val);
1173 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1174 bool OutOfRangeError(SMRange
Range);
1190 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1191 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1192 std::optional<bool> EnableWavefrontSize32,
1193 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1194 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1195 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1196 bool ParseDirectiveAMDGCNTarget();
1197 bool ParseDirectiveAMDHSACodeObjectVersion();
1198 bool ParseDirectiveAMDHSAKernel();
1199 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1200 bool ParseDirectiveAMDKernelCodeT();
1202 bool subtargetHasRegister(
const MCRegisterInfo &
MRI, MCRegister
Reg);
1203 bool ParseDirectiveAMDGPUHsaKernel();
1205 bool ParseDirectiveISAVersion();
1206 bool ParseDirectiveHSAMetadata();
1207 bool ParseDirectivePALMetadataBegin();
1208 bool ParseDirectivePALMetadata();
1209 bool ParseDirectiveAMDGPULDS();
1210 bool ParseDirectiveAMDGPUInfo();
1214 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1215 const char *AssemblerDirectiveEnd,
1216 std::string &CollectString);
1218 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1219 RegisterKind RegKind, MCRegister Reg1,
1220 RegisterKind RegKind1, SMLoc Loc);
1221 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1222 unsigned &RegNum,
unsigned &RegWidth,
1223 bool RestoreOnFailure =
false);
1224 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1225 unsigned &RegNum,
unsigned &RegWidth,
1226 SmallVectorImpl<AsmToken> &Tokens);
1227 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1229 SmallVectorImpl<AsmToken> &Tokens);
1230 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1232 SmallVectorImpl<AsmToken> &Tokens);
1233 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1235 SmallVectorImpl<AsmToken> &Tokens);
1236 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1237 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1238 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1241 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1242 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1243 void initializeGprCountSymbol(RegisterKind RegKind);
1244 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1250 OperandMode_Default,
1254 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1256 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1257 const MCInstrInfo &MII)
1258 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1259 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1264 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1266 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1267 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1268 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1269 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1271 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1272 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1273 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1275 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1276 initializeGprCountSymbol(IS_VGPR);
1277 initializeGprCountSymbol(IS_SGPR);
1282 createConstantSymbol(Symbol, Code);
1284 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1285 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1286 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1326 bool hasBVHRayTracingInsts()
const {
1327 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1330 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1332 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1334 bool hasInv2PiInlineImm()
const {
1335 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1338 bool has64BitLiterals()
const {
1339 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1342 bool hasFlatOffsets()
const {
1343 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1346 bool hasTrue16Insts()
const {
1347 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1351 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1354 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1356 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1358 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1360 bool hasPartialNSAEncoding()
const {
1361 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1364 bool hasGloballyAddressableScratch()
const {
1365 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1368 bool hasCacheFillSize()
const {
1369 return getFeatureBits()[AMDGPU::FeatureCacheFillSize];
1382 AMDGPUTargetStreamer &getTargetStreamer() {
1383 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1384 return static_cast<AMDGPUTargetStreamer &
>(TS);
1390 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1393 const MCRegisterInfo *getMRI()
const {
1397 const MCInstrInfo *getMII()
const {
return &MII; }
1401 int16_t getTargetRegClass(
unsigned TargetRCIdx)
const {
1407 const FeatureBitset &getFeatureBits()
const {
1408 return getSTI().getFeatureBits();
1411 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1412 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1413 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1415 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1416 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1417 bool isForcedDPP()
const {
return ForcedDPP; }
1418 bool isForcedSDWA()
const {
return ForcedSDWA; }
1419 ArrayRef<unsigned> getMatchedVariants()
const;
1420 StringRef getMatchedVariantName()
const;
1422 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1423 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1424 bool RestoreOnFailure);
1425 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1426 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1427 SMLoc &EndLoc)
override;
1428 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1429 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1430 unsigned Kind)
override;
1431 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1434 bool MatchingInlineAsm)
override;
1435 bool ParseDirective(AsmToken DirectiveID)
override;
1436 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1437 void onEndOfFile()
override;
1439 OperandMode
Mode = OperandMode_Default);
1440 StringRef parseMnemonicSuffix(StringRef Name);
1441 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1447 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1451 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1452 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1454 ParseStatus parseOperandArrayWithPrefix(
1456 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1457 bool (*ConvertResult)(int64_t &) =
nullptr);
1461 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1462 bool IgnoreNegative =
false);
1463 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1468 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1472 ArrayRef<const char *> Ids,
1476 ArrayRef<const char *> Ids,
1477 AMDGPUOperand::ImmTy
Type);
1480 bool isOperandModifier(
const AsmToken &Token,
1481 const AsmToken &NextToken)
const;
1482 bool isRegOrOperandModifier(
const AsmToken &Token,
1483 const AsmToken &NextToken)
const;
1484 bool isNamedOperandModifier(
const AsmToken &Token,
1485 const AsmToken &NextToken)
const;
1486 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1487 const AsmToken &NextToken)
const;
1488 bool parseSP3NegModifier();
1495 bool AllowImm =
true);
1497 bool AllowImm =
true);
1503 AMDGPUOperand::ImmTy ImmTy);
1508 AMDGPUOperand::ImmTy
Type);
1512 AMDGPUOperand::ImmTy
Type);
1516 AMDGPUOperand::ImmTy
Type);
1520 ParseStatus parseDfmtNfmt(int64_t &
Format);
1521 ParseStatus parseUfmt(int64_t &
Format);
1522 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1524 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1527 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1528 ParseStatus parseNumericFormat(int64_t &
Format);
1532 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1533 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1538 bool parseCnt(int64_t &IntVal);
1541 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1542 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1545 bool parseDelay(int64_t &Delay);
1551 struct OperandInfoTy {
1554 bool IsSymbolic =
false;
1555 bool IsDefined =
false;
1557 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1560 struct StructuredOpField : OperandInfoTy {
1564 bool IsDefined =
false;
1566 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1567 unsigned Width, int64_t
Default)
1568 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1569 virtual ~StructuredOpField() =
default;
1571 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1572 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1576 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1578 return Error(Parser,
"not supported on this GPU");
1580 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1588 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1589 OperandInfoTy &Stream);
1590 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1591 const OperandInfoTy &Stream);
1593 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1594 OperandInfoTy &Width);
1599 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1606 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1608 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1612 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1617 bool validateBF16InlineConst(
const MCInst &Inst,
1620 bool validateConstantBusLimitations(
const MCInst &Inst,
1622 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1625 bool tryVOPD(
const MCInst &Inst);
1626 bool tryVOPD3(
const MCInst &Inst);
1627 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1629 bool validateIntClampSupported(
const MCInst &Inst);
1630 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1631 bool validateMIMGGatherDMask(
const MCInst &Inst);
1633 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1634 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1635 bool validateMIMGD16(
const MCInst &Inst);
1637 bool validateTensorR128(
const MCInst &Inst);
1638 bool validateMIMGMSAA(
const MCInst &Inst);
1639 bool validateOpSel(
const MCInst &Inst);
1640 bool validateTrue16OpSel(
const MCInst &Inst);
1641 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1643 bool validateVccOperand(MCRegister
Reg)
const;
1648 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1649 bool validateVGPRAlign(
const MCInst &Inst)
const;
1653 bool validateDivScale(
const MCInst &Inst);
1658 const unsigned CPol);
1664 bool validateClusterBarrierIsFirst(
const MCInst &Inst,
1667 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1668 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1669 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1670 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1672 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1673 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1674 ArrayRef<unsigned> Variants);
1675 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1677 bool isId(
const StringRef Id)
const;
1678 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1680 StringRef getId()
const;
1681 bool trySkipId(
const StringRef Id);
1682 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1686 bool parseString(StringRef &Val,
1687 const StringRef ErrMsg =
"expected a string");
1688 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1694 StringRef getTokenStr()
const;
1695 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1697 SMLoc getLoc()
const;
1701 void onBeginOfFile()
override;
1706 void emitTargetDirective();
1707 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1719 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1720 const unsigned MaxVal,
const Twine &ErrMsg,
1722 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1723 const unsigned MinVal,
const unsigned MaxVal,
1724 const StringRef ErrMsg);
1726 bool parseSwizzleOffset(int64_t &
Imm);
1727 bool parseSwizzleMacro(int64_t &
Imm);
1728 bool parseSwizzleQuadPerm(int64_t &
Imm);
1729 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1730 bool parseSwizzleBroadcast(int64_t &
Imm);
1731 bool parseSwizzleSwap(int64_t &
Imm);
1732 bool parseSwizzleReverse(int64_t &
Imm);
1733 bool parseSwizzleFFT(int64_t &
Imm);
1734 bool parseSwizzleRotate(int64_t &
Imm);
1737 int64_t parseGPRIdxMacro();
1740 cvtMubufImpl(Inst,
Operands,
false);
1743 cvtMubufImpl(Inst,
Operands,
true);
1749 OptionalImmIndexMap &OptionalIdx);
1758 OptionalImmIndexMap &OptionalIdx);
1760 OptionalImmIndexMap &OptionalIdx);
1764 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1766 bool parseDimId(
unsigned &Encoding);
1768 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1772 int64_t parseDPPCtrlSel(StringRef Ctrl);
1773 int64_t parseDPPCtrlPerm();
1779 bool IsDPP8 =
false);
1785 AMDGPUOperand::ImmTy
Type);
1793 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1796 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1797 bool SkipSrcVcc =
false);
1907bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
1917 if (!isImmTy(ImmTyNone)) {
1922 if (getModifiers().
Lit != LitModifier::None)
1932 if (type == MVT::f64 || type == MVT::i64) {
1934 AsmParser->hasInv2PiInlineImm());
1937 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
1956 APFloat::rmNearestTiesToEven, &Lost);
1963 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1965 AsmParser->hasInv2PiInlineImm());
1970 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
1971 AsmParser->hasInv2PiInlineImm());
1975 if (type == MVT::f64 || type == MVT::i64) {
1977 AsmParser->hasInv2PiInlineImm());
1986 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
1987 AsmParser->hasInv2PiInlineImm());
1991 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
1992 AsmParser->hasInv2PiInlineImm());
1995bool AMDGPUOperand::isLiteralImm(MVT type)
const {
1997 if (!isImmTy(ImmTyNone)) {
2002 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2007 if (type == MVT::f64 && hasFPModifiers()) {
2027 if (type == MVT::f64) {
2032 if (type == MVT::i64) {
2045 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2046 : (type == MVT::v2i16) ? MVT::f32
2047 : (type == MVT::v2f32) ? MVT::f32
2050 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2054bool AMDGPUOperand::isRegClassTarget(
unsigned TargetRCIdx)
const {
2057 int16_t RCID = AsmParser->getTargetRegClass(TargetRCIdx);
2058 return RCID >= 0 && isRegClass(RCID);
2061bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2062 return isRegKind() &&
2063 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2066bool AMDGPUOperand::isVRegWithInputMods()
const {
2067 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2069 (AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP] &&
2070 isRegClassTarget(AMDGPU::VReg_64_AlignTarget));
2073template <
bool IsFake16>
2074bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2075 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2076 : AMDGPU::VGPR_16_Lo128RegClassID);
2079template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2080 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2081 : AMDGPU::VGPR_16RegClassID);
2084bool AMDGPUOperand::isT16_LO16VRegWithInputMods()
const {
2085 return isRegClass(AMDGPU::VGPR_16_LO16RegClassID);
2088bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2089 if (AsmParser->isVI())
2091 if (AsmParser->isGFX9Plus())
2092 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2096bool AMDGPUOperand::isSDWAFP16Operand()
const {
2097 return isSDWAOperand(MVT::f16);
2100bool AMDGPUOperand::isSDWAFP32Operand()
const {
2101 return isSDWAOperand(MVT::f32);
2104bool AMDGPUOperand::isSDWAInt16Operand()
const {
2105 return isSDWAOperand(MVT::i16);
2108bool AMDGPUOperand::isSDWAInt32Operand()
const {
2109 return isSDWAOperand(MVT::i32);
2112bool AMDGPUOperand::isBoolReg()
const {
2113 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2114 (AsmParser->isWave32() && isSCSrc_b32()));
2118 unsigned Size)
const {
2119 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2134void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2135 bool ApplyModifiers)
const {
2145 addLiteralImmOperand(Inst,
Imm.Val,
2146 ApplyModifiers & isImmTy(ImmTyNone) &&
2147 Imm.Mods.hasFPModifiers());
2149 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2154void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2155 bool ApplyModifiers)
const {
2156 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2161 if (ApplyModifiers) {
2163 const unsigned Size =
2165 Val = applyInputFPModifiers(Val,
Size);
2169 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2171 bool CanUse64BitLiterals =
2174 MCContext &Ctx = AsmParser->getContext();
2185 if (
Lit == LitModifier::None &&
2187 AsmParser->hasInv2PiInlineImm())) {
2195 bool HasMandatoryLiteral =
2198 if (
Literal.getLoBits(32) != 0 &&
2199 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2200 !HasMandatoryLiteral) {
2201 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2203 "Can't encode literal as exact 64-bit floating-point operand. "
2204 "Low 32-bits will be set to zero");
2205 Val &= 0xffffffff00000000u;
2211 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2217 Lit = LitModifier::Lit64;
2218 }
else if (
Lit == LitModifier::Lit) {
2232 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2234 Lit = LitModifier::Lit64;
2241 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2242 Literal == 0x3fc45f306725feed) {
2282 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2289 if (
Lit != LitModifier::None) {
2320 if (
Lit == LitModifier::None &&
2330 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2338 if (
Lit == LitModifier::None &&
2346 if (!AsmParser->has64BitLiterals()) {
2347 Val =
static_cast<uint64_t>(Val) << 32;
2354 if (
Lit == LitModifier::Lit ||
2356 Val =
static_cast<uint64_t>(Val) << 32;
2360 if (
Lit == LitModifier::Lit)
2387 if (
Lit != LitModifier::None) {
2395void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2401bool AMDGPUOperand::isInlineValue()
const {
2409void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2420 if (Is == IS_VGPR) {
2425 return AMDGPU::VGPR_32RegClassID;
2427 return AMDGPU::VReg_64RegClassID;
2429 return AMDGPU::VReg_96RegClassID;
2431 return AMDGPU::VReg_128RegClassID;
2433 return AMDGPU::VReg_160RegClassID;
2435 return AMDGPU::VReg_192RegClassID;
2437 return AMDGPU::VReg_224RegClassID;
2439 return AMDGPU::VReg_256RegClassID;
2441 return AMDGPU::VReg_288RegClassID;
2443 return AMDGPU::VReg_320RegClassID;
2445 return AMDGPU::VReg_352RegClassID;
2447 return AMDGPU::VReg_384RegClassID;
2449 return AMDGPU::VReg_512RegClassID;
2451 return AMDGPU::VReg_1024RegClassID;
2453 }
else if (Is == IS_TTMP) {
2458 return AMDGPU::TTMP_32RegClassID;
2460 return AMDGPU::TTMP_64RegClassID;
2462 return AMDGPU::TTMP_128RegClassID;
2464 return AMDGPU::TTMP_256RegClassID;
2466 return AMDGPU::TTMP_512RegClassID;
2468 }
else if (Is == IS_SGPR) {
2473 return AMDGPU::SGPR_32RegClassID;
2475 return AMDGPU::SGPR_64RegClassID;
2477 return AMDGPU::SGPR_96RegClassID;
2479 return AMDGPU::SGPR_128RegClassID;
2481 return AMDGPU::SGPR_160RegClassID;
2483 return AMDGPU::SGPR_192RegClassID;
2485 return AMDGPU::SGPR_224RegClassID;
2487 return AMDGPU::SGPR_256RegClassID;
2489 return AMDGPU::SGPR_288RegClassID;
2491 return AMDGPU::SGPR_320RegClassID;
2493 return AMDGPU::SGPR_352RegClassID;
2495 return AMDGPU::SGPR_384RegClassID;
2497 return AMDGPU::SGPR_512RegClassID;
2499 }
else if (Is == IS_AGPR) {
2504 return AMDGPU::AGPR_32RegClassID;
2506 return AMDGPU::AReg_64RegClassID;
2508 return AMDGPU::AReg_96RegClassID;
2510 return AMDGPU::AReg_128RegClassID;
2512 return AMDGPU::AReg_160RegClassID;
2514 return AMDGPU::AReg_192RegClassID;
2516 return AMDGPU::AReg_224RegClassID;
2518 return AMDGPU::AReg_256RegClassID;
2520 return AMDGPU::AReg_288RegClassID;
2522 return AMDGPU::AReg_320RegClassID;
2524 return AMDGPU::AReg_352RegClassID;
2526 return AMDGPU::AReg_384RegClassID;
2528 return AMDGPU::AReg_512RegClassID;
2530 return AMDGPU::AReg_1024RegClassID;
2538 .
Case(
"exec", AMDGPU::EXEC)
2539 .
Case(
"vcc", AMDGPU::VCC)
2540 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2541 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2542 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2543 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2544 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2545 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2546 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2547 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2548 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2549 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2550 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2551 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2552 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2553 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2554 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2555 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2556 .
Case(
"m0", AMDGPU::M0)
2557 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2558 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2559 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2560 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2561 .
Case(
"scc", AMDGPU::SRC_SCC)
2562 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2563 .
Case(
"tba", AMDGPU::TBA)
2564 .
Case(
"tma", AMDGPU::TMA)
2565 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2566 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2567 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2568 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2569 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2570 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2571 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2572 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2573 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2574 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2575 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2576 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2577 .
Case(
"pc", AMDGPU::PC_REG)
2578 .
Case(
"null", AMDGPU::SGPR_NULL)
2582bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2583 SMLoc &EndLoc,
bool RestoreOnFailure) {
2584 auto R = parseRegister();
2588 RegNo =
R->getReg();
2589 StartLoc =
R->getStartLoc();
2590 EndLoc =
R->getEndLoc();
2594bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2596 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2599ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2601 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2602 bool PendingErrors = getParser().hasPendingError();
2603 getParser().clearPendingErrors();
2611bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2612 RegisterKind RegKind,
2614 RegisterKind RegKind1, SMLoc Loc) {
2616 if (RegKind == IS_SGPR) {
2617 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2618 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2619 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2625 if (RegKind != RegKind1) {
2626 Error(Loc,
"registers in a list must be of the same kind");
2632 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2637 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2638 Reg = AMDGPU::FLAT_SCR;
2642 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2643 Reg = AMDGPU::XNACK_MASK;
2647 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2652 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2657 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2662 Error(Loc,
"register does not fit in the list");
2668 if (Reg1 !=
Reg + RegWidth / 32) {
2669 Error(Loc,
"registers in a list must have consecutive indices");
2685 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2686 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2690 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2696 if (Str.starts_with(
Reg.Name))
2702 return !Str.getAsInteger(10, Num);
2705bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2706 const AsmToken &NextToken)
const {
2721 StringRef RegSuffix = Str.substr(
RegName.size());
2722 if (!RegSuffix.
empty()) {
2739bool AMDGPUAsmParser::isRegister() {
2740 return isRegister(
getToken(), peekToken());
2743MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2744 unsigned SubReg,
unsigned RegWidth,
2748 unsigned AlignSize = 1;
2749 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2755 if (RegNum % AlignSize != 0) {
2756 Error(Loc,
"invalid register alignment");
2757 return MCRegister();
2760 unsigned RegIdx = RegNum / AlignSize;
2763 Error(Loc,
"invalid or unsupported register size");
2764 return MCRegister();
2768 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2769 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2770 Error(Loc,
"register index is out of range");
2771 return AMDGPU::NoRegister;
2774 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2775 Error(Loc,
"register index is out of range");
2776 return MCRegister();
2785 Error(Loc,
"invalid subregister");
2791bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2793 int64_t RegLo, RegHi;
2797 SMLoc FirstIdxLoc = getLoc();
2804 SecondIdxLoc = getLoc();
2815 Error(FirstIdxLoc,
"invalid register index");
2820 Error(SecondIdxLoc,
"invalid register index");
2824 if (RegLo > RegHi) {
2825 Error(FirstIdxLoc,
"first register index should not exceed second index");
2829 if (RegHi == RegLo) {
2830 StringRef RegSuffix = getTokenStr();
2831 if (RegSuffix ==
".l") {
2832 SubReg = AMDGPU::lo16;
2834 }
else if (RegSuffix ==
".h") {
2835 SubReg = AMDGPU::hi16;
2840 Num =
static_cast<unsigned>(RegLo);
2841 RegWidth = 32 * ((RegHi - RegLo) + 1);
2846MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
2849 SmallVectorImpl<AsmToken> &Tokens) {
2855 RegKind = IS_SPECIAL;
2862MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
2865 SmallVectorImpl<AsmToken> &Tokens) {
2867 StringRef
RegName = getTokenStr();
2868 auto Loc = getLoc();
2872 Error(Loc,
"invalid register name");
2873 return MCRegister();
2881 unsigned SubReg = NoSubRegister;
2882 bool IsRange =
false;
2883 if (!RegSuffix.
empty()) {
2885 SubReg = AMDGPU::lo16;
2887 SubReg = AMDGPU::hi16;
2891 Error(Loc,
"invalid register index");
2892 return MCRegister();
2898 if (!ParseRegRange(RegNum, RegWidth, SubReg))
2899 return MCRegister();
2903 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
2904 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
2905 if (RegKind == IS_SGPR && IsRange
2906 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
2907 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
2908 Error(Loc,
"register index is out of range");
2909 return MCRegister();
2915MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
2916 unsigned &RegNum,
unsigned &RegWidth,
2917 SmallVectorImpl<AsmToken> &Tokens) {
2919 auto ListLoc = getLoc();
2922 "expected a register or a list of registers")) {
2923 return MCRegister();
2928 auto Loc = getLoc();
2929 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
2930 return MCRegister();
2931 if (RegWidth != 32) {
2932 Error(Loc,
"expected a single 32-bit register");
2933 return MCRegister();
2937 RegisterKind NextRegKind;
2939 unsigned NextRegNum, NextRegWidth;
2942 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
2944 return MCRegister();
2946 if (NextRegWidth != 32) {
2947 Error(Loc,
"expected a single 32-bit register");
2948 return MCRegister();
2950 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
2952 return MCRegister();
2956 "expected a comma or a closing square bracket")) {
2957 return MCRegister();
2961 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
2966bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
2967 MCRegister &
Reg,
unsigned &RegNum,
2969 SmallVectorImpl<AsmToken> &Tokens) {
2970 auto Loc = getLoc();
2974 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
2976 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
2978 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
2983 assert(Parser.hasPendingError());
2987 if (!subtargetHasRegister(*
TRI,
Reg)) {
2988 if (
Reg == AMDGPU::SGPR_NULL) {
2989 Error(Loc,
"'null' operand is not supported on this GPU");
2992 " register not available on this GPU");
3000bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3001 MCRegister &
Reg,
unsigned &RegNum,
3003 bool RestoreOnFailure ) {
3007 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3008 if (RestoreOnFailure) {
3009 while (!Tokens.
empty()) {
3018std::optional<StringRef>
3019AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3022 return StringRef(
".amdgcn.next_free_vgpr");
3024 return StringRef(
".amdgcn.next_free_sgpr");
3026 return std::nullopt;
3030void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3031 auto SymbolName = getGprCountSymbolName(RegKind);
3032 assert(SymbolName &&
"initializing invalid register kind");
3038bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3039 unsigned DwordRegIndex,
3040 unsigned RegWidth) {
3045 auto SymbolName = getGprCountSymbolName(RegKind);
3050 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3054 return !
Error(getLoc(),
3055 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3059 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3061 if (OldCount <= NewMax)
3067std::unique_ptr<AMDGPUOperand>
3068AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3070 SMLoc StartLoc = Tok.getLoc();
3071 SMLoc EndLoc = Tok.getEndLoc();
3072 RegisterKind RegKind;
3074 unsigned RegNum, RegWidth;
3076 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3080 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3083 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3084 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3091 if (isRegister() || isModifier())
3094 if (
Lit == LitModifier::None) {
3095 if (trySkipId(
"lit"))
3096 Lit = LitModifier::Lit;
3097 else if (trySkipId(
"lit64"))
3098 Lit = LitModifier::Lit64;
3100 if (
Lit != LitModifier::None) {
3103 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3112 const auto &NextTok = peekToken();
3115 bool Negate =
false;
3123 AMDGPUOperand::Modifiers Mods;
3131 StringRef Num = getTokenStr();
3134 APFloat RealVal(APFloat::IEEEdouble());
3135 auto roundMode = APFloat::rmNearestTiesToEven;
3136 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3139 RealVal.changeSign();
3142 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3143 S, AMDGPUOperand::ImmTyNone,
true));
3144 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3145 Op.setModifiers(Mods);
3154 if (HasSP3AbsModifier) {
3163 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3166 if (Parser.parseExpression(Expr))
3170 if (Expr->evaluateAsAbsolute(IntVal)) {
3172 return Error(S,
"literal value out of range");
3173 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3174 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3175 Op.setModifiers(Mods);
3177 if (
Lit != LitModifier::None)
3179 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3192 if (
auto R = parseRegister()) {
3202 ParseStatus Res = parseReg(
Operands);
3210bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3211 const AsmToken &NextToken)
const {
3214 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3219bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3220 const AsmToken &NextToken)
const {
3224bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3225 const AsmToken &NextToken)
const {
3226 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3229bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3230 const AsmToken &NextToken)
const {
3231 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3248bool AMDGPUAsmParser::isModifier() {
3251 AsmToken NextToken[2];
3252 peekTokens(NextToken);
3256 bool IsOpcodeModifier = isOpcodeModifierWithVal(Tok, NextToken[0]) &&
3259 return isOperandModifier(Tok, NextToken[0]) ||
3261 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3287bool AMDGPUAsmParser::parseSP3NegModifier() {
3289 AsmToken NextToken[2];
3290 peekTokens(NextToken);
3293 (isRegister(NextToken[0], NextToken[1]) ||
3311 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3313 SP3Neg = parseSP3NegModifier();
3316 Neg = trySkipId(
"neg");
3318 return Error(Loc,
"expected register or immediate");
3322 Abs = trySkipId(
"abs");
3327 if (trySkipId(
"lit")) {
3328 Lit = LitModifier::Lit;
3331 }
else if (trySkipId(
"lit64")) {
3332 Lit = LitModifier::Lit64;
3335 if (!has64BitLiterals())
3336 return Error(Loc,
"lit64 is not supported on this GPU");
3342 return Error(Loc,
"expected register or immediate");
3351 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3355 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3356 Error(Loc,
"expected immediate with lit modifier");
3358 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3364 if (
Lit != LitModifier::None &&
3368 AMDGPUOperand::Modifiers Mods;
3369 Mods.Abs = Abs || SP3Abs;
3370 Mods.Neg = Neg || SP3Neg;
3373 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3374 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3376 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3377 Op.setModifiers(Mods);
3385 bool Sext = trySkipId(
"sext");
3386 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3401 AMDGPUOperand::Modifiers Mods;
3404 if (Mods.hasIntModifiers()) {
3405 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3407 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3408 Op.setModifiers(Mods);
3415 return parseRegOrImmWithFPInputMods(
Operands,
false);
3419 return parseRegOrImmWithIntInputMods(
Operands,
false);
3426 if (!trySkipId(
"rsrcidx"))
3432 SMLoc RegLoc = getLoc();
3433 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3441 if (!
Reg->isRsrcReg32())
3442 return Error(RegLoc,
"rsrcidx operand must be a 32-bit SGPR or VGPR");
3452 auto Loc = getLoc();
3453 if (trySkipId(
"off")) {
3455 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3462 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3471unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3476 return Match_InvalidOperand;
3478 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3479 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3482 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3484 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3485 return Match_InvalidOperand;
3493 if (tryAnotherVOPDEncoding(Inst))
3494 return Match_InvalidOperand;
3496 return Match_Success;
3500 static const unsigned Variants[] = {
3509ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3510 if (isForcedDPP() && isForcedVOP3()) {
3514 if (getForcedEncodingSize() == 32) {
3519 if (isForcedVOP3()) {
3524 if (isForcedSDWA()) {
3530 if (isForcedDPP()) {
3538StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3539 if (isForcedDPP() && isForcedVOP3())
3542 if (getForcedEncodingSize() == 32)
3558AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3562 case AMDGPU::FLAT_SCR:
3564 case AMDGPU::VCC_LO:
3565 case AMDGPU::VCC_HI:
3572 return MCRegister();
3579bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3580 unsigned OpIdx)
const {
3588 const MCOperand &MO = Inst.
getOperand(OpIdx);
3638unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3644 case AMDGPU::V_LSHLREV_B64_e64:
3645 case AMDGPU::V_LSHLREV_B64_gfx10:
3646 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3647 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3648 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3649 case AMDGPU::V_LSHRREV_B64_e64:
3650 case AMDGPU::V_LSHRREV_B64_gfx10:
3651 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3652 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3653 case AMDGPU::V_ASHRREV_I64_e64:
3654 case AMDGPU::V_ASHRREV_I64_gfx10:
3655 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3656 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3657 case AMDGPU::V_LSHL_B64_e64:
3658 case AMDGPU::V_LSHR_B64_e64:
3659 case AMDGPU::V_ASHR_I64_e64:
3672 bool AddMandatoryLiterals =
false) {
3675 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3679 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3681 return {getNamedOperandIdx(Opcode, OpName::src0X),
3682 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3683 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3684 getNamedOperandIdx(Opcode, OpName::src0Y),
3685 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3686 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3691 return {getNamedOperandIdx(Opcode, OpName::src0),
3692 getNamedOperandIdx(Opcode, OpName::src1),
3693 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3696bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3697 const MCOperand &MO = Inst.
getOperand(OpIdx);
3699 return !isInlineConstant(Inst, OpIdx);
3706 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3717 const unsigned Opcode = Inst.
getOpcode();
3718 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3721 if (!LaneSelOp.
isReg())
3724 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3727bool AMDGPUAsmParser::validateConstantBusLimitations(
3729 const unsigned Opcode = Inst.
getOpcode();
3730 const MCInstrDesc &
Desc = MII.
get(Opcode);
3731 MCRegister LastSGPR;
3732 unsigned ConstantBusUseCount = 0;
3733 unsigned NumLiterals = 0;
3734 unsigned LiteralSize;
3750 SmallDenseSet<MCRegister> SGPRsUsed;
3751 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3753 SGPRsUsed.
insert(SGPRUsed);
3754 ++ConstantBusUseCount;
3759 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3761 for (
int OpIdx : OpIndices) {
3765 const MCOperand &MO = Inst.
getOperand(OpIdx);
3766 if (usesConstantBus(Inst, OpIdx)) {
3775 if (SGPRsUsed.
insert(LastSGPR).second) {
3776 ++ConstantBusUseCount;
3796 if (NumLiterals == 0) {
3799 }
else if (LiteralSize !=
Size) {
3805 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3807 "invalid operand (violates constant bus restrictions)");
3814std::optional<unsigned>
3815AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3817 const unsigned Opcode = Inst.
getOpcode();
3823 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3824 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3833 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3834 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3835 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3836 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3837 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3838 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3842 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3843 int I = getNamedOperandIdx(Opcode, OpName);
3847 int64_t
Imm =
Op.getImm();
3853 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
3854 OpName::vsrc2Y, OpName::imm}) {
3855 int I = getNamedOperandIdx(Opcode, OpName);
3865 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
3866 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
3868 return InvalidCompOprIdx;
3871bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
3878 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
3879 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
3880 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
3882 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
3886 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
3887 if (!InvalidCompOprIdx.has_value())
3890 auto CompOprIdx = *InvalidCompOprIdx;
3893 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
3894 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
3897 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
3898 if (CompOprIdx == VOPD::Component::DST) {
3900 Error(Loc,
"dst registers must be distinct");
3902 Error(Loc,
"one dst register must be even and the other odd");
3904 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
3905 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
3906 " operands must use different VGPR banks");
3914bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
3916 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
3917 if (!InvalidCompOprIdx.has_value())
3921 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
3922 if (InvalidCompOprIdx.has_value()) {
3927 if (*InvalidCompOprIdx == VOPD::Component::DST)
3940bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
3941 const unsigned Opcode = Inst.
getOpcode();
3956 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
3957 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
3958 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
3959 int I = getNamedOperandIdx(Opcode, OpName);
3966 return !tryVOPD3(Inst);
3971bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
3976 return tryVOPD(Inst);
3977 return tryVOPD3(Inst);
3980bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
3985 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
3993bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4001 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4002 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4003 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4008 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4009 hasBVHRayTracingInsts())
4012 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4013 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4018 bool IsPackedD16 =
false;
4021 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4022 IsPackedD16 = D16Idx >= 0;
4024 DataSize = (DataSize + 1) / 2;
4027 if ((VDataSize / 4) == DataSize + TFESize)
4032 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4034 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4036 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4040bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4049 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4051 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4052 AMDGPU::OpName RSrcOpName =
4054 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4055 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4056 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4060 assert(SrsrcIdx > VAddr0Idx);
4063 if (BaseOpcode->
BVH) {
4064 if (IsA16 == BaseOpcode->
A16)
4066 Error(IDLoc,
"image address size does not match a16");
4072 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4073 unsigned ActualAddrSize =
4074 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4076 unsigned ExpectedAddrSize =
4080 if (hasPartialNSAEncoding() &&
4082 int VAddrLastIdx = SrsrcIdx - 1;
4083 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4085 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4088 if (ExpectedAddrSize > 12)
4089 ExpectedAddrSize = 16;
4094 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4098 if (ActualAddrSize == ExpectedAddrSize)
4101 Error(IDLoc,
"image address size does not match dim and a16");
4105bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4112 if (!
Desc.mayLoad() || !
Desc.mayStore())
4115 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4122 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4125bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4132 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4140 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4143bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4157 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4158 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4165bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4172 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4175 if (!BaseOpcode->
MSAA)
4178 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4184 return DimInfo->
MSAA;
4189 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4190 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4191 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4201bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4209 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4220 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4224bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4229 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4232 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4243 "source operand must be either a VGPR or an inline constant");
4250bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4255 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4258 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4262 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4264 "inline constants are not allowed for this operand");
4271bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4279 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4280 if (BlgpIdx != -1) {
4281 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4282 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4292 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4294 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4299 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4301 "wrong register tuple size for blgp value " + Twine(BLGP));
4309 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4319 if (Src2Reg == DstReg)
4324 .getSizeInBits() <= 128)
4327 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4329 "source 2 operand must not partially overlap with dst");
4336bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4340 case V_DIV_SCALE_F32_gfx6_gfx7:
4341 case V_DIV_SCALE_F32_vi:
4342 case V_DIV_SCALE_F32_gfx10:
4343 case V_DIV_SCALE_F64_gfx6_gfx7:
4344 case V_DIV_SCALE_F64_vi:
4345 case V_DIV_SCALE_F64_gfx10:
4352 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4353 AMDGPU::OpName::src2_modifiers}) {
4364bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4371 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4380bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4386 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4393 case AMDGPU::V_SUBREV_F32_e32:
4394 case AMDGPU::V_SUBREV_F32_e64:
4395 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4396 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4397 case AMDGPU::V_SUBREV_F32_e32_vi:
4398 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4399 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4400 case AMDGPU::V_SUBREV_F32_e64_vi:
4402 case AMDGPU::V_SUBREV_CO_U32_e32:
4403 case AMDGPU::V_SUBREV_CO_U32_e64:
4404 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4405 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4407 case AMDGPU::V_SUBBREV_U32_e32:
4408 case AMDGPU::V_SUBBREV_U32_e64:
4409 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4410 case AMDGPU::V_SUBBREV_U32_e32_vi:
4411 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4412 case AMDGPU::V_SUBBREV_U32_e64_vi:
4414 case AMDGPU::V_SUBREV_U32_e32:
4415 case AMDGPU::V_SUBREV_U32_e64:
4416 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4417 case AMDGPU::V_SUBREV_U32_e32_vi:
4418 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4419 case AMDGPU::V_SUBREV_U32_e64_vi:
4421 case AMDGPU::V_SUBREV_F16_e32:
4422 case AMDGPU::V_SUBREV_F16_e64:
4423 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4424 case AMDGPU::V_SUBREV_F16_e32_vi:
4425 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4426 case AMDGPU::V_SUBREV_F16_e64_vi:
4428 case AMDGPU::V_SUBREV_U16_e32:
4429 case AMDGPU::V_SUBREV_U16_e64:
4430 case AMDGPU::V_SUBREV_U16_e32_vi:
4431 case AMDGPU::V_SUBREV_U16_e64_vi:
4433 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4434 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4435 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4437 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4438 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4440 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4441 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4443 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4444 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4446 case AMDGPU::V_LSHRREV_B32_e32:
4447 case AMDGPU::V_LSHRREV_B32_e64:
4448 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4449 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4450 case AMDGPU::V_LSHRREV_B32_e32_vi:
4451 case AMDGPU::V_LSHRREV_B32_e64_vi:
4452 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4453 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4455 case AMDGPU::V_ASHRREV_I32_e32:
4456 case AMDGPU::V_ASHRREV_I32_e64:
4457 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4458 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4459 case AMDGPU::V_ASHRREV_I32_e32_vi:
4460 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4461 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4462 case AMDGPU::V_ASHRREV_I32_e64_vi:
4464 case AMDGPU::V_LSHLREV_B32_e32:
4465 case AMDGPU::V_LSHLREV_B32_e64:
4466 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4467 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4468 case AMDGPU::V_LSHLREV_B32_e32_vi:
4469 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4470 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4471 case AMDGPU::V_LSHLREV_B32_e64_vi:
4473 case AMDGPU::V_LSHLREV_B16_e32:
4474 case AMDGPU::V_LSHLREV_B16_e64:
4475 case AMDGPU::V_LSHLREV_B16_e32_vi:
4476 case AMDGPU::V_LSHLREV_B16_e64_vi:
4477 case AMDGPU::V_LSHLREV_B16_gfx10:
4479 case AMDGPU::V_LSHRREV_B16_e32:
4480 case AMDGPU::V_LSHRREV_B16_e64:
4481 case AMDGPU::V_LSHRREV_B16_e32_vi:
4482 case AMDGPU::V_LSHRREV_B16_e64_vi:
4483 case AMDGPU::V_LSHRREV_B16_gfx10:
4485 case AMDGPU::V_ASHRREV_I16_e32:
4486 case AMDGPU::V_ASHRREV_I16_e64:
4487 case AMDGPU::V_ASHRREV_I16_e32_vi:
4488 case AMDGPU::V_ASHRREV_I16_e64_vi:
4489 case AMDGPU::V_ASHRREV_I16_gfx10:
4491 case AMDGPU::V_LSHLREV_B64_e64:
4492 case AMDGPU::V_LSHLREV_B64_gfx10:
4493 case AMDGPU::V_LSHLREV_B64_vi:
4495 case AMDGPU::V_LSHRREV_B64_e64:
4496 case AMDGPU::V_LSHRREV_B64_gfx10:
4497 case AMDGPU::V_LSHRREV_B64_vi:
4499 case AMDGPU::V_ASHRREV_I64_e64:
4500 case AMDGPU::V_ASHRREV_I64_gfx10:
4501 case AMDGPU::V_ASHRREV_I64_vi:
4503 case AMDGPU::V_PK_LSHLREV_B16:
4504 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4505 case AMDGPU::V_PK_LSHLREV_B16_vi:
4507 case AMDGPU::V_PK_LSHRREV_B16:
4508 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4509 case AMDGPU::V_PK_LSHRREV_B16_vi:
4510 case AMDGPU::V_PK_ASHRREV_I16:
4511 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4512 case AMDGPU::V_PK_ASHRREV_I16_vi:
4519bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4521 const unsigned Opcode = Inst.
getOpcode();
4530 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4531 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4535 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4539 "lds_direct is not supported on this GPU");
4545 "lds_direct cannot be used with this instruction");
4549 if (SrcName != OpName::src0) {
4551 "lds_direct may be used as src0 only");
4561 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4562 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4563 if (
Op.isFlatOffset())
4564 return Op.getStartLoc();
4569bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4572 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4577 return validateFlatOffset(Inst,
Operands);
4580 return validateSMEMOffset(Inst,
Operands);
4585 const unsigned OffsetSize = 24;
4586 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4588 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4589 "-bit unsigned offset for buffer ops");
4593 const unsigned OffsetSize = 16;
4594 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4596 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4603bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4609 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4613 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4615 "flat offset modifier is not supported on this GPU");
4622 bool AllowNegative =
4624 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4626 Twine(
"expected a ") +
4627 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4628 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4637 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4638 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4639 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4640 return Op.getStartLoc();
4645bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4654 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4670 ?
"expected a 23-bit unsigned offset for buffer ops"
4671 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4672 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4673 :
"expected a 21-bit signed offset");
4683bool AMDGPUAsmParser::validateBF16InlineConst(
const MCInst &Inst,
4685 if (!getFeatureBits()[AMDGPU::FeatureBF16InlineConstFromUpperFP32])
4700 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4704 const MCOperandInfo &Src0Info =
Desc.operands()[Src0Idx];
4709 if (!
Src0.isImm() ||
4711 hasInv2PiInlineImm()))
4716 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
4717 if (ModsIdx != -1 &&
4723 "bf16 inline constant is read from the high half of the fp32 inline "
4724 "constant on this GPU; use the e64 encoding with op_sel:[1,0]");
4728bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4731 const MCInstrDesc &
Desc = MII.
get(Opcode);
4735 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4736 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4738 const int OpIndices[] = {Src0Idx, Src1Idx};
4740 unsigned NumExprs = 0;
4741 unsigned NumLiterals = 0;
4744 for (
int OpIdx : OpIndices) {
4748 const MCOperand &MO = Inst.
getOperand(OpIdx);
4752 std::optional<int64_t>
Imm;
4755 }
else if (MO.
isExpr()) {
4764 if (!
Imm.has_value()) {
4766 }
else if (!isInlineConstant(Inst, OpIdx)) {
4770 if (NumLiterals == 0 || LiteralValue !=
Value) {
4778 if (NumLiterals + NumExprs <= 1)
4782 "only one unique literal operand is allowed");
4786bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4789 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4797 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4798 if (OpSelIdx != -1) {
4802 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4803 if (OpSelHiIdx != -1) {
4812 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4819 if (getFeatureBits()[AMDGPU::FeatureGFX950Insts] &&
4820 (
Opc == AMDGPU::V_PK_ADD_F32_vi ||
Opc == AMDGPU::V_PK_MUL_F32_vi ||
4821 Opc == AMDGPU::V_PK_FMA_F32_vi)) {
4822 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4826 if (
Opc == AMDGPU::V_PK_FMA_F32_vi &&
4827 (((OpSel >> 1) & 1) || ((OpSel >> 2) & 1)))
4830 else if ((OpSel & 3) == 2)
4838 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4839 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4840 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4841 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4850 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4852 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4862 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4863 if (Src2Idx != -1) {
4874bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4875 if (!hasTrue16Insts())
4877 const MCRegisterInfo *
MRI = getMRI();
4879 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4885 if (OpSelOpValue == 0)
4887 unsigned OpCount = 0;
4888 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4889 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4890 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4897 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4898 if (OpSelOpIsHi != VGPRSuffixIsHi)
4907bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4908 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
4920 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
4931 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
4932 AMDGPU::OpName::src1_modifiers,
4933 AMDGPU::OpName::src2_modifiers};
4935 for (
unsigned i = 0; i < 3; ++i) {
4945bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
4948 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
4949 if (DppCtrlIdx >= 0) {
4953 getSTI().
hasFeature(AMDGPU::FeatureDPALU_DPP) &&
4957 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
4958 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
4959 :
"DP ALU dpp only supports row_newbcast");
4964 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
4965 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
4968 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4974 "invalid operand for instruction");
4979 "src1 immediate operand invalid for instruction");
4989bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
4990 return (
Reg == AMDGPU::VCC && isWave64()) ||
4991 (
Reg == AMDGPU::VCC_LO && isWave32());
4995bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
4998 const MCInstrDesc &
Desc = MII.
get(Opcode);
4999 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5006 std::optional<unsigned> LiteralOpIdx;
5009 for (
int OpIdx : OpIndices) {
5013 const MCOperand &MO = Inst.
getOperand(OpIdx);
5019 std::optional<int64_t>
Imm;
5025 bool IsAnotherLiteral =
false;
5026 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5027 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5028 if (!
Imm.has_value()) {
5030 IsAnotherLiteral =
true;
5031 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5036 HasMandatoryLiteral);
5048 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5049 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5051 "invalid operand for instruction");
5056 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5057 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5059 "invalid operand for instruction");
5064 if (IsValid32Op && !IsForcedFP64 && !IsForcedLit64) {
5065 Value =
static_cast<uint32_t
>(
5073 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5074 !getFeatureBits()[FeatureVOP3Literal]) {
5076 "literal operands are not supported");
5080 if (LiteralOpIdx && IsAnotherLiteral) {
5082 getOperandLoc(
Operands, *LiteralOpIdx)),
5083 "only one unique literal operand is allowed");
5087 if (IsAnotherLiteral)
5088 LiteralOpIdx = OpIdx;
5097 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5111bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5117 ? AMDGPU::OpName::data0
5118 : AMDGPU::OpName::vdata;
5120 const MCRegisterInfo *
MRI = getMRI();
5126 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5130 auto FB = getFeatureBits();
5131 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5132 if (DataAreg < 0 || DstAreg < 0)
5134 return DstAreg == DataAreg;
5137 return DstAreg < 1 && DataAreg < 1;
5140bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5141 auto FB = getFeatureBits();
5142 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5146 const MCRegisterInfo *
MRI = getMRI();
5149 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5152 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5156 case AMDGPU::DS_LOAD_TR6_B96:
5157 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5161 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5162 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5166 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5167 if (VAddrIdx != -1) {
5169 MCRegister
Sub =
MRI->getSubReg(
Op.getReg(), AMDGPU::sub0);
5170 if ((
Sub - AMDGPU::VGPR0) & 1)
5175 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5176 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5181 const MCRegisterClass &VGPR32 =
MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5182 const MCRegisterClass &AGPR32 =
MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5188 MCRegister
Sub =
MRI->getSubReg(
Op.getReg(), AMDGPU::sub0);
5202 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5203 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5205 return Op.getStartLoc();
5210bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5213 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5216 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5219 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5220 auto FB = getFeatureBits();
5221 bool UsesNeg =
false;
5222 if (FB[AMDGPU::FeatureGFX940Insts]) {
5224 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5225 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5226 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5227 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5232 if (IsNeg == UsesNeg)
5235 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5236 :
"invalid modifier: neg is not supported");
5241bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5247 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5248 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5249 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5250 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5253 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5256 if (
Reg == AMDGPU::SGPR_NULL)
5259 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5263bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5268 return validateGWS(Inst,
Operands);
5273 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5278 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5279 Error(S,
"gds modifier is not supported on this GPU");
5287bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5289 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5293 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5294 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5297 const MCRegisterInfo *
MRI = getMRI();
5298 const MCRegisterClass &VGPR32 =
MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5300 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5303 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5305 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5312bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5316 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5323 if (!hasCacheFillSize()) {
5324 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5327 Error(S,
"Cache fill size is not supported on this GPU");
5328 }
else if (!validateCFSBits(Inst,
Operands)) {
5335 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5338 Error(S,
"scale_offset is not supported on this GPU");
5341 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5344 Error(S,
"nv is not supported on this GPU");
5349 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5352 Error(S,
"scale_offset is not supported for this instruction");
5356 return validateTHAndScopeBits(Inst,
Operands, CPol);
5360 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5361 Error(S,
"cache policy is not supported for SMRD instructions");
5365 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5372 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5376 "scc modifier is not supported for this instruction on this GPU");
5387 :
"instruction must use glc");
5392 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5395 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5397 :
"instruction must not use glc");
5405bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5407 const unsigned CPol) {
5412 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5419 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5424 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5432 return PrintError(
"invalid th value for SMEM instruction");
5439 return PrintError(
"scope and th combination is not valid");
5445 return PrintError(
"invalid th value for atomic instructions");
5448 return PrintError(
"invalid th value for store instructions");
5451 return PrintError(
"invalid th value for load instructions");
5457bool AMDGPUAsmParser::validateCFSBits(
const MCInst &Inst,
5459 const unsigned Opcode = Inst.
getOpcode();
5460 const MCInstrDesc &
Desc = MII.
get(Opcode);
5463 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5471 return PrintError(
"Cache fill size is only supported for VC operations");
5474bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5478 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5480 Error(Loc,
"TFE modifier has no meaning for store instructions");
5488bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5494 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5498 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5501 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5502 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5511 "wrong register tuple size for " +
5516 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5517 !validateFmt(BFmt, AMDGPU::OpName::src1))
5521 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5522 if (AScaleIdx == -1)
5526 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5530 "invalid matrix and scale format combination");
5537bool AMDGPUAsmParser::validateMonitorSleep(
const MCInst &Inst,
5540 if (
Opc != AMDGPU::S_MONITOR_SLEEP_gfx12 ||
5541 !getSTI().
hasFeature(AMDGPU::FeatureNoSleepForever))
5544 int ImmIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::simm16);
5547 "sleep forever is unsuported on the target");
5554bool AMDGPUAsmParser::validateClusterBarrierIsFirst(
5557 if (
Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx12 &&
5558 Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx13)
5561 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
5568 "s_barrier_signal_isfirst does not support user_cluster_barrier_id (-3)");
5572bool AMDGPUAsmParser::validateScaleSel(
const MCInst &Inst,
5575 int ScaleSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::scale_sel);
5576 if (ScaleSelIdx == -1)
5580 case AMDGPU::V_CVT_SCALE_PK16_F16_FP6_e64_gfx1250:
5581 case AMDGPU::V_CVT_SCALE_PK16_BF16_FP6_e64_gfx1250:
5582 case AMDGPU::V_CVT_SCALE_PK16_F16_BF6_e64_gfx1250:
5583 case AMDGPU::V_CVT_SCALE_PK16_BF16_BF6_e64_gfx1250:
5584 case AMDGPU::V_CVT_SCALE_PK16_F32_FP6_e64_gfx1250:
5585 case AMDGPU::V_CVT_SCALE_PK16_F32_BF6_e64_gfx1250:
5586 case AMDGPU::V_CVT_SCALE_PK8_F16_FP4_e64_gfx1250:
5587 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP4_e64_gfx1250:
5588 case AMDGPU::V_CVT_SCALE_PK8_F32_FP4_e64_gfx1250:
5591 case AMDGPU::V_CVT_SCALE_PK8_F16_FP8_e64_gfx1250:
5592 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP8_e64_gfx1250:
5593 case AMDGPU::V_CVT_SCALE_PK8_F16_BF8_e64_gfx1250:
5594 case AMDGPU::V_CVT_SCALE_PK8_BF16_BF8_e64_gfx1250:
5595 case AMDGPU::V_CVT_SCALE_PK8_F32_FP8_e64_gfx1250:
5596 case AMDGPU::V_CVT_SCALE_PK8_F32_BF8_e64_gfx1250:
5603 if (getSTI().
hasFeature(AMDGPU::FeatureBlock16ConversionScaleInsts))
5607 if (ScaleSel < MaxSel)
5611 "scale_sel maximum supported value is " + Twine(MaxSel - 1));
5615bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5617 if (!validateLdsDirect(Inst,
Operands))
5619 if (!validateTrue16OpSel(Inst)) {
5621 "op_sel operand conflicts with 16-bit operand suffix");
5624 if (!validateSOPLiteral(Inst,
Operands))
5626 if (!validateVOPLiteral(Inst,
Operands)) {
5629 if (!validateConstantBusLimitations(Inst,
Operands)) {
5632 if (!validateVOPD(Inst,
Operands)) {
5635 if (!validateIntClampSupported(Inst)) {
5637 "integer clamping is not supported on this GPU");
5640 if (!validateOpSel(Inst)) {
5642 "invalid op_sel operand");
5645 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5647 "invalid neg_lo operand");
5650 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5652 "invalid neg_hi operand");
5655 if (!validateDPP(Inst,
Operands)) {
5659 if (!validateMIMGD16(Inst)) {
5661 "d16 modifier is not supported on this GPU");
5664 if (!validateMIMGDim(Inst,
Operands)) {
5665 Error(IDLoc,
"missing dim operand");
5668 if (!validateTensorR128(Inst)) {
5670 "instruction must set modifier r128=0");
5673 if (!validateMIMGMSAA(Inst)) {
5675 "invalid dim; must be MSAA type");
5678 if (!validateMIMGDataSize(Inst, IDLoc)) {
5681 if (!validateMIMGAddrSize(Inst, IDLoc))
5683 if (!validateMIMGAtomicDMask(Inst)) {
5685 "invalid atomic image dmask");
5688 if (!validateMIMGGatherDMask(Inst)) {
5690 "invalid image_gather dmask: only one bit must be set");
5693 if (!validateMovrels(Inst,
Operands)) {
5696 if (!validateOffset(Inst,
Operands)) {
5699 if (!validateBF16InlineConst(Inst,
Operands)) {
5702 if (!validateMAIAccWrite(Inst,
Operands)) {
5705 if (!validateMAISrc2(Inst,
Operands)) {
5708 if (!validateMFMA(Inst,
Operands)) {
5711 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5715 if (!validateAGPRLdSt(Inst)) {
5718 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5719 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5720 :
"invalid register class: agpr loads and stores not supported on "
5724 if (!validateVGPRAlign(Inst)) {
5725 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5732 if (!validateBLGP(Inst,
Operands)) {
5736 if (!validateDivScale(Inst)) {
5737 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5740 if (!validateWaitCnt(Inst,
Operands)) {
5743 if (!validateTFE(Inst,
Operands)) {
5746 if (!validateWMMA(Inst,
Operands)) {
5749 if (!validateMonitorSleep(Inst,
Operands)) {
5752 if (!validateClusterBarrierIsFirst(Inst,
Operands)) {
5755 if (!validateScaleSel(Inst,
Operands)) {
5764 unsigned VariantID = 0);
5768 unsigned VariantID);
5770bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5775bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5776 const FeatureBitset &FBS,
5777 ArrayRef<unsigned> Variants) {
5778 for (
auto Variant : Variants) {
5786bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5788 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5791 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5796 getParser().clearPendingErrors();
5800 StringRef VariantName = getMatchedVariantName();
5801 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5802 return Error(IDLoc, Twine(VariantName,
5803 " variant of this instruction is not supported"));
5807 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5808 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5810 FeatureBitset FeaturesWS32 = getFeatureBits();
5811 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5812 .
flip(AMDGPU::FeatureWavefrontSize32);
5813 FeatureBitset AvailableFeaturesWS32 =
5814 ComputeAvailableFeatures(FeaturesWS32);
5816 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5817 return Error(IDLoc,
"instruction requires wavesize=32");
5821 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5822 return Error(IDLoc,
"instruction not supported on this GPU (" +
5823 getSTI().
getCPU() +
")" +
": " + Mnemo);
5828 return Error(IDLoc,
"invalid instruction" + Suggestion);
5834 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5835 if (
Op.isToken() && InvalidOprIdx > 1) {
5836 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5837 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5842bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5846 bool MatchingInlineAsm) {
5849 unsigned Result = Match_Success;
5854 auto atLeastAsSpecific = [](
unsigned New,
unsigned Cur) {
5855 auto rank = [](
unsigned M) {
5856 return M == Match_MnemonicFail ? 1
5857 :
M == Match_InvalidOperand ? 2
5858 :
M == Match_MissingFeature ? 3
5861 return rank(New) >= rank(Cur);
5864 for (
auto Variant : getMatchedVariants()) {
5867 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5868 if (R == Match_Success || atLeastAsSpecific(R, Result)) {
5872 if (R == Match_Success)
5876 if (Result == Match_Success) {
5877 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5880 emitTargetDirective();
5881 Out.emitInstruction(Inst, getSTI());
5888 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5895 case Match_MissingFeature:
5899 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5901 case Match_InvalidOperand: {
5902 SMLoc ErrorLoc = IDLoc;
5903 if (ErrorInfo != ~0ULL) {
5904 if (ErrorInfo >=
Operands.size()) {
5905 return Error(IDLoc,
"too few operands for instruction");
5907 AMDGPUOperand &ErrorOp = (AMDGPUOperand &)*
Operands[ErrorInfo];
5908 ErrorLoc = ErrorOp.getStartLoc();
5909 if (ErrorLoc == SMLoc())
5913 return Error(ErrorLoc,
"invalid VOPDY instruction");
5915 return Error(ErrorLoc,
"invalid operand for instruction");
5918 case Match_MnemonicFail:
5924bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5929 if (getParser().parseAbsoluteExpression(Tmp)) {
5932 Ret =
static_cast<uint32_t
>(Tmp);
5936bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5937 if (!getSTI().getTargetTriple().isAMDGCN())
5938 return TokError(
"directive only supported for amdgcn architecture");
5940 std::string TargetIDDirective;
5941 SMLoc TargetStart = getTok().getLoc();
5942 if (getParser().parseEscapedString(TargetIDDirective))
5945 std::optional<AMDGPU::TargetID> MaybeParsed =
5948 return getParser().Error(TargetStart,
5949 "malformed target id '" + TargetIDDirective +
"'");
5952 const Triple &
TT = getSTI().getTargetTriple();
5958 return getParser().Error(
5959 TargetStart,
"target id '" + TargetIDDirective +
5960 "' specifies a processor that is not valid for "
5962 TT.getArchName() +
"'");
5965 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5966 getTargetStreamer().getTargetID();
5969 const Triple &STITriple = getSTI().getTargetTriple();
5970 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5971 return getParser().Error(
5972 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5973 " is incompatible with " +
5974 Twine(CurrentTargetID->toString()));
5978 StringRef DirectiveProcessor =
5981 if (DirectiveISA != ISA) {
5982 return getParser().Error(TargetStart,
5983 ".amdgcn_target directive processor " +
5984 Twine(DirectiveProcessor) +
5985 " does not match the specified processor " +
5986 Twine(getSTI().
getCPU()));
5992 CurrentTargetID->getXnackSetting())) {
5994 ".amdgcn_target directive has conflicting xnack settings");
5998 CurrentTargetID->getSramEccSetting())) {
6000 ".amdgcn_target directive has conflicting sramecc settings");
6006 getTargetStreamer().getTargetID()->setXnackSetting(
6008 getTargetStreamer().getTargetID()->setSramEccSetting(
6014bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
6018bool AMDGPUAsmParser::calculateGPRBlocks(
6019 const FeatureBitset &Features,
const MCExpr *VCCUsed,
6020 const MCExpr *FlatScrUsed,
bool XNACKUsed,
6021 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
6022 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
6023 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
6028 const MCExpr *
NumSGPRs = NextFreeSGPR;
6029 int64_t EvaluatedSGPRs;
6031 if (
ISA.Major >= 10)
6036 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
6037 !Features.
test(FeatureSGPRInitBug) &&
6038 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6039 return OutOfRangeError(SGPRRange);
6041 const MCExpr *ExtraSGPRs =
6045 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
6046 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
6047 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6048 return OutOfRangeError(SGPRRange);
6050 if (Features.
test(FeatureSGPRInitBug))
6057 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6058 unsigned Granule) ->
const MCExpr * {
6062 const MCExpr *AlignToGPR =
6064 const MCExpr *DivGPR =
6070 VGPRBlocks = GetNumGPRBlocks(
6079bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6080 if (!getSTI().getTargetTriple().isAMDGCN())
6081 return TokError(
"directive only supported for amdgcn architecture");
6084 return TokError(
"directive only supported for amdhsa OS");
6086 StringRef KernelName;
6087 if (getParser().parseIdentifier(KernelName))
6094 AMDGPU::MCKernelDescriptor KD =
6104 const MCExpr *NextFreeVGPR = ZeroExpr;
6106 const MCExpr *NamedBarCnt = ZeroExpr;
6111 const MCExpr *NextFreeSGPR = ZeroExpr;
6114 unsigned ImpliedUserSGPRCount = 0;
6118 std::optional<unsigned> ExplicitUserSGPRCount;
6119 const MCExpr *ReserveVCC = OneExpr;
6120 const MCExpr *ReserveFlatScr = OneExpr;
6121 std::optional<bool> EnableWavefrontSize32;
6128 SMRange IDRange = getTok().getLocRange();
6129 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6132 if (ID ==
".end_amdhsa_kernel")
6135 if (!Seen.
insert(ID).second)
6136 return TokError(
".amdhsa_ directives cannot be repeated");
6138 SMLoc ValStart = getLoc();
6139 const MCExpr *ExprVal;
6140 if (getParser().parseExpression(ExprVal))
6142 SMLoc ValEnd = getLoc();
6143 SMRange ValRange = SMRange(ValStart, ValEnd);
6147 bool EvaluatableExpr;
6148 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6150 return OutOfRangeError(ValRange);
6154#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6155 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6156 return OutOfRangeError(RANGE); \
6157 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6162#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6164 return Error(IDRange.Start, "directive should have resolvable expression", \
6167 if (ID ==
".amdhsa_group_segment_fixed_size") {
6170 return OutOfRangeError(ValRange);
6172 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6175 return OutOfRangeError(ValRange);
6177 }
else if (ID ==
".amdhsa_kernarg_size") {
6179 return OutOfRangeError(ValRange);
6181 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6183 ExplicitUserSGPRCount = Val;
6184 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6188 "directive is not supported with architected flat scratch",
6191 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6194 ImpliedUserSGPRCount += 4;
6195 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6198 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6201 return OutOfRangeError(ValRange);
6205 ImpliedUserSGPRCount += Val;
6206 PreloadLength = Val;
6208 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6211 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6214 return OutOfRangeError(ValRange);
6218 PreloadOffset = Val;
6219 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6222 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6225 ImpliedUserSGPRCount += 2;
6226 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6229 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6232 ImpliedUserSGPRCount += 2;
6233 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6236 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6239 ImpliedUserSGPRCount += 2;
6240 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6243 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6246 ImpliedUserSGPRCount += 2;
6247 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6250 "directive is not supported with architected flat scratch",
6254 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6257 ImpliedUserSGPRCount += 2;
6258 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6261 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6264 ImpliedUserSGPRCount += 1;
6265 }
else if (ID ==
".amdhsa_wavefront_size32") {
6268 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6269 if (!getFeatureBits().
test(AMDGPU::FeatureSupportsWave32) ||
6270 !getFeatureBits().
test(AMDGPU::FeatureSupportsWave64)) {
6272 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6274 EnableWavefrontSize32 = Val;
6276 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6278 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6280 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6282 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6285 "directive is not supported with architected flat scratch",
6288 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6290 }
else if (ID ==
".amdhsa_enable_private_segment") {
6294 "directive is not supported without architected flat scratch",
6297 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6299 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6301 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6303 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6305 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6307 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6309 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6311 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6313 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6315 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6317 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6319 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6320 VGPRRange = ValRange;
6321 NextFreeVGPR = ExprVal;
6322 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6323 SGPRRange = ValRange;
6324 NextFreeSGPR = ExprVal;
6325 }
else if (ID ==
".amdhsa_accum_offset") {
6327 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6328 AccumOffset = ExprVal;
6329 }
else if (ID ==
".amdhsa_named_barrier_count") {
6331 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6332 NamedBarCnt = ExprVal;
6333 }
else if (ID ==
".amdhsa_reserve_vcc") {
6335 return OutOfRangeError(ValRange);
6336 ReserveVCC = ExprVal;
6337 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6339 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6342 "directive is not supported with architected flat scratch",
6345 return OutOfRangeError(ValRange);
6346 ReserveFlatScr = ExprVal;
6347 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6349 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6351 return OutOfRangeError(ValRange);
6352 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny();
6353 if (Val != XnackOn) {
6354 return getParser().Error(
6356 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6358 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6360 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6362 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6364 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6366 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6368 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6370 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6372 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6374 }
else if (ID ==
".amdhsa_dx10_clamp") {
6375 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6376 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6379 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6381 }
else if (ID ==
".amdhsa_ieee_mode") {
6382 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6383 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6386 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6388 }
else if (ID ==
".amdhsa_fp16_overflow") {
6390 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6392 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6394 }
else if (ID ==
".amdhsa_tg_split") {
6396 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6399 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6402 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6404 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6406 }
else if (ID ==
".amdhsa_memory_ordered") {
6408 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6410 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6412 }
else if (ID ==
".amdhsa_forward_progress") {
6414 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6416 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6418 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6420 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6421 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6423 SharedVGPRCount = Val;
6425 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6427 }
else if (ID ==
".amdhsa_inst_pref_size") {
6429 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6430 if (
ISA.Major == 11) {
6432 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6436 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6439 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6442 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6444 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6446 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6448 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6451 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6453 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6455 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6457 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6459 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6461 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6463 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6465 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6467 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6469 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6471 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6473 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6476 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6479#undef PARSE_BITS_ENTRY
6482 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6483 return TokError(
".amdhsa_next_free_vgpr directive is required");
6485 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6486 return TokError(
".amdhsa_next_free_sgpr directive is required");
6488 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6490 return TokError(
"too many user SGPRs enabled, found " +
6491 Twine(UserSGPRCount) +
", but only " +
6497 if (PreloadLength) {
6503 const MCExpr *VGPRBlocks;
6504 const MCExpr *SGPRBlocks;
6505 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6506 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6507 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6508 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6511 int64_t EvaluatedVGPRBlocks;
6512 bool VGPRBlocksEvaluatable =
6513 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6514 if (VGPRBlocksEvaluatable &&
6516 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6517 return OutOfRangeError(VGPRRange);
6521 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6522 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6524 int64_t EvaluatedSGPRBlocks;
6525 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6527 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6528 return OutOfRangeError(SGPRRange);
6531 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6532 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6534 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6535 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6536 "enabled user SGPRs");
6542 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6543 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6548 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6549 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6554 return TokError(
"Kernarg size should be resolvable");
6556 if (PreloadLength && kernarg_size &&
6557 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6558 return TokError(
"Kernarg preload length + offset is larger than the "
6559 "kernarg segment size");
6562 if (!Seen.
contains(
".amdhsa_accum_offset"))
6563 return TokError(
".amdhsa_accum_offset directive is required");
6564 int64_t EvaluatedAccum;
6565 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6566 uint64_t UEvaluatedAccum = EvaluatedAccum;
6567 if (AccumEvaluatable &&
6568 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6569 return TokError(
"accum_offset should be in range [4..256] in "
6572 int64_t EvaluatedNumVGPR;
6573 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6577 return TokError(
"accum_offset exceeds total VGPR allocation");
6583 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6584 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6590 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6591 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6594 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6596 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6597 return TokError(
"shared_vgpr_count directive not valid on "
6598 "wavefront size 32");
6601 if (VGPRBlocksEvaluatable &&
6602 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6604 return TokError(
"shared_vgpr_count*2 + "
6605 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6610 emitTargetDirective();
6611 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6612 NextFreeVGPR, NextFreeSGPR,
6613 ReserveVCC, ReserveFlatScr);
6617bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6619 if (ParseAsAbsoluteExpression(
Version))
6622 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6623 emitTargetDirective();
6627bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6628 AMDGPUMCKernelCodeT &
C) {
6631 if (ID ==
"max_scratch_backing_memory_byte_size") {
6632 Parser.eatToEndOfStatement();
6636 SmallString<40> ErrStr;
6637 raw_svector_ostream Err(ErrStr);
6638 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6639 return TokError(Err.
str());
6643 if (ID ==
"enable_wavefront_size32") {
6646 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6648 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6651 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6655 if (ID ==
"wavefront_size") {
6656 if (
C.wavefront_size == 5) {
6658 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6660 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6661 }
else if (
C.wavefront_size == 6) {
6663 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6670bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6671 AMDGPUMCKernelCodeT KernelCode;
6681 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6684 if (ID ==
".end_amd_kernel_code_t")
6687 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6692 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6697bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6698 StringRef KernelName;
6699 if (!parseId(KernelName,
"expected symbol name"))
6702 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6709bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6710 if (!getSTI().getTargetTriple().isAMDGCN()) {
6711 return Error(getLoc(),
6712 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6716 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6718 std::optional<AMDGPU::TargetID> MaybeParsed =
6721 return Error(getParser().getTok().getLoc(),
6722 "malformed target id '" + TargetIDDirective +
"'");
6725 const Triple &
TT = getSTI().getTargetTriple();
6731 return Error(getParser().getTok().getLoc(),
6732 "target id '" + TargetIDDirective +
6733 "' specifies a processor that is not valid for subarch '" +
6734 TT.getArchName() +
"'");
6737 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6738 getTargetStreamer().getTargetID();
6741 const Triple &STITriple = getSTI().getTargetTriple();
6742 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6743 return Error(getParser().getTok().getLoc(),
6744 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6745 " is incompatible with " +
6746 Twine(CurrentTargetID->toString()));
6750 StringRef DirectiveProcessor =
6753 if (DirectiveISA != ISA) {
6754 return Error(getParser().getTok().getLoc(),
6755 ".amd_amdgpu_isa directive processor " +
6756 Twine(DirectiveProcessor) +
6757 " does not match the specified processor " +
6758 Twine(getSTI().
getCPU()));
6761 getTargetStreamer().EmitISAVersion();
6767bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6770 std::string HSAMetadataString;
6775 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6776 return Error(getLoc(),
"invalid HSA metadata");
6783bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6784 const char *AssemblerDirectiveEnd,
6785 std::string &CollectString) {
6787 raw_string_ostream CollectStream(CollectString);
6789 getLexer().setSkipSpace(
false);
6791 bool FoundEnd =
false;
6794 CollectStream << getTokenStr();
6798 if (trySkipId(AssemblerDirectiveEnd)) {
6803 CollectStream << Parser.parseStringToEndOfStatement()
6804 <<
getContext().getAsmInfo().getSeparatorString();
6806 Parser.eatToEndOfStatement();
6809 getLexer().setSkipSpace(
true);
6812 return TokError(Twine(
"expected directive ") +
6813 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6820bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6826 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6827 if (!PALMetadata->setFromString(
String))
6828 return Error(getLoc(),
"invalid PAL metadata");
6833bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6836 Twine(
" directive is "
6837 "not available on non-amdpal OSes"))
6841 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6842 PALMetadata->setLegacy();
6845 if (ParseAsAbsoluteExpression(
Key)) {
6846 return TokError(Twine(
"invalid value in ") +
6850 return TokError(Twine(
"expected an even number of values in ") +
6853 if (ParseAsAbsoluteExpression(
Value)) {
6854 return TokError(Twine(
"invalid value in ") +
6857 PALMetadata->setRegister(
Key,
Value);
6866bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6867 if (getParser().checkForValidSection())
6871 SMLoc NameLoc = getLoc();
6872 if (getParser().parseIdentifier(Name))
6873 return TokError(
"expected identifier in directive");
6876 if (getParser().parseComma())
6882 SMLoc SizeLoc = getLoc();
6883 if (getParser().parseAbsoluteExpression(
Size))
6886 return Error(SizeLoc,
"size must be non-negative");
6887 if (
Size > LocalMemorySize)
6888 return Error(SizeLoc,
"size is too large");
6892 SMLoc AlignLoc = getLoc();
6893 if (getParser().parseAbsoluteExpression(Alignment))
6896 return Error(AlignLoc,
"alignment must be a power of two");
6901 if (Alignment >= 1u << 31)
6902 return Error(AlignLoc,
"alignment is too large");
6908 Symbol->redefineIfPossible();
6909 if (!
Symbol->isUndefined())
6910 return Error(NameLoc,
"invalid symbol redefinition");
6912 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6916bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6917 if (getParser().checkForValidSection())
6921 if (getParser().parseIdentifier(FuncName))
6922 return TokError(
"expected symbol name after .amdgpu_info");
6925 AMDGPU::InfoSectionData ParsedInfoData;
6926 AMDGPU::FuncInfo FI;
6928 bool HasScalarAttrs =
false;
6935 SMLoc IDLoc = getLoc();
6936 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6939 if (ID ==
".end_amdgpu_info")
6947 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6949 if (Dir ==
"flags") {
6951 if (getParser().parseAbsoluteExpression(Val))
6954 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6956 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6958 HasScalarAttrs =
true;
6959 }
else if (Dir ==
"num_sgpr") {
6961 if (getParser().parseAbsoluteExpression(Val))
6963 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6964 HasScalarAttrs =
true;
6965 }
else if (Dir ==
"num_vgpr") {
6967 if (getParser().parseAbsoluteExpression(Val))
6970 HasScalarAttrs =
true;
6971 }
else if (Dir ==
"num_agpr") {
6973 if (getParser().parseAbsoluteExpression(Val))
6976 HasScalarAttrs =
true;
6977 }
else if (Dir ==
"private_segment_size") {
6979 if (getParser().parseAbsoluteExpression(Val))
6982 HasScalarAttrs =
true;
6983 }
else if (Dir ==
"use") {
6985 if (getParser().parseIdentifier(ResName))
6986 return TokError(
"expected resource symbol for .amdgpu_use");
6987 ParsedInfoData.
Uses.push_back(
6988 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
6989 }
else if (Dir ==
"call") {
6991 if (getParser().parseIdentifier(DstName))
6992 return TokError(
"expected callee symbol for .amdgpu_call");
6993 ParsedInfoData.
Calls.push_back(
6994 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
6995 }
else if (Dir ==
"indirect_call") {
6997 if (getParser().parseEscapedString(TypeId))
6998 return TokError(
"expected type ID string for .amdgpu_indirect_call");
6999 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
7000 }
else if (Dir ==
"typeid") {
7002 if (getParser().parseEscapedString(TypeId))
7003 return TokError(
"expected type ID string for .amdgpu_typeid");
7004 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
7006 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
7011 ParsedInfoData.
Funcs.push_back(std::move(FI));
7013 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
7014 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
7015 Data.Funcs.push_back(std::move(Func));
7016 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
7017 Data.Uses.push_back(Use);
7018 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
7020 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
7023 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
7024 Data.TypeIds.push_back(std::move(TypeId));
7029void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
7036void AMDGPUAsmParser::checkKernelPrologues() {
7037 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
7038 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
7039 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
7040 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
7041 if (!AMDHSAKernelSymbols.
contains(Sym))
7043 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
7044 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
7048 "' does not begin with the required prologue "
7049 "sequence: s_mov_b64 followed by v_nop and "
7050 "global_prefetch_b8");
7054 OpcodeStream.
clear();
7055 OpcodeStreamSymbols.clear();
7056 AMDHSAKernelSymbols.
clear();
7059void AMDGPUAsmParser::onEndOfFile() {
7060 emitTargetDirective();
7061 checkKernelPrologues();
7063 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7066bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7067 StringRef IDVal = DirectiveID.
getString();
7070 if (IDVal ==
".amdhsa_kernel")
7071 return ParseDirectiveAMDHSAKernel();
7073 if (IDVal ==
".amdhsa_code_object_version")
7074 return ParseDirectiveAMDHSACodeObjectVersion();
7078 return ParseDirectiveHSAMetadata();
7080 if (IDVal ==
".amd_kernel_code_t")
7081 return ParseDirectiveAMDKernelCodeT();
7083 if (IDVal ==
".amdgpu_hsa_kernel")
7084 return ParseDirectiveAMDGPUHsaKernel();
7086 if (IDVal ==
".amd_amdgpu_isa")
7087 return ParseDirectiveISAVersion();
7091 Twine(
" directive is "
7092 "not available on non-amdhsa OSes"))
7097 if (IDVal ==
".amdgcn_target")
7098 return ParseDirectiveAMDGCNTarget();
7100 if (IDVal ==
".amdgpu_lds")
7101 return ParseDirectiveAMDGPULDS();
7103 if (IDVal ==
".amdgpu_info")
7104 return ParseDirectiveAMDGPUInfo();
7107 return ParseDirectivePALMetadataBegin();
7110 return ParseDirectivePALMetadata();
7115bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &
MRI,
7117 if (
MRI.regsOverlap(TTMP12_TTMP13_TTMP14_TTMP15,
Reg))
7121 if (
MRI.regsOverlap(SGPR104_SGPR105,
Reg))
7122 return hasSGPR104_SGPR105();
7125 case SRC_SHARED_BASE_LO:
7126 case SRC_SHARED_BASE:
7127 case SRC_SHARED_LIMIT_LO:
7128 case SRC_SHARED_LIMIT:
7130 case SRC_PRIVATE_BASE_LO:
7131 case SRC_PRIVATE_BASE:
7132 case SRC_PRIVATE_LIMIT_LO:
7133 case SRC_PRIVATE_LIMIT:
7135 case SRC_FLAT_SCRATCH_BASE_LO:
7136 case SRC_FLAT_SCRATCH_BASE_HI:
7137 return hasGloballyAddressableScratch();
7138 case SRC_POPS_EXITING_WAVE_ID:
7151 getTargetStreamer().getTargetID()->isXnackSupported();
7180 if (
MRI.regsOverlap(SGPR102_SGPR103,
Reg))
7181 return hasSGPR102_SGPR103();
7189 ParseStatus Res = parseVOPD(
Operands);
7194 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7206 SMLoc LBraceLoc = getLoc();
7211 auto Loc = getLoc();
7214 Error(Loc,
"expected a register");
7218 RBraceLoc = getLoc();
7223 "expected a comma or a closing square bracket"))
7227 if (
Operands.size() - Prefix > 1) {
7229 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7230 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7239StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7241 setForcedEncodingSize(0);
7242 setForcedDPP(
false);
7243 setForcedSDWA(
false);
7245 if (
Name.consume_back(
"_e64_dpp")) {
7247 setForcedEncodingSize(64);
7250 if (
Name.consume_back(
"_e64")) {
7251 setForcedEncodingSize(64);
7254 if (
Name.consume_back(
"_e32")) {
7255 setForcedEncodingSize(32);
7258 if (
Name.consume_back(
"_dpp")) {
7262 if (
Name.consume_back(
"_sdwa")) {
7263 setForcedSDWA(
true);
7271 unsigned VariantID);
7277 Name = parseMnemonicSuffix(Name);
7283 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7285 bool IsMIMG = Name.starts_with(
"image_");
7288 OperandMode
Mode = OperandMode_Default;
7290 Mode = OperandMode_NSA;
7294 checkUnsupportedInstruction(Name, NameLoc);
7295 if (!Parser.hasPendingError()) {
7298 :
"not a valid operand.";
7318ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7321 if (!trySkipId(Name))
7324 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7328ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7337ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7339 std::function<
bool(int64_t &)> ConvertResult) {
7343 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7347 if (ConvertResult && !ConvertResult(
Value)) {
7348 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7351 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7355ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7357 bool (*ConvertResult)(int64_t &)) {
7366 const unsigned MaxSize = 4;
7370 for (
int I = 0;; ++
I) {
7372 SMLoc Loc = getLoc();
7376 if (
Op != 0 &&
Op != 1)
7377 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7384 if (
I + 1 == MaxSize)
7385 return Error(getLoc(),
"expected a closing square bracket");
7391 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7395ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7397 AMDGPUOperand::ImmTy ImmTy,
7398 bool IgnoreNegative) {
7402 if (trySkipId(Name)) {
7404 }
else if (trySkipId(
"no", Name)) {
7413 return Error(S,
"r128 modifier is not supported on this GPU");
7414 if (Name ==
"a16" && !
hasA16())
7415 return Error(S,
"a16 modifier is not supported on this GPU");
7417 if (Bit == 0 && Name ==
"gds") {
7420 return Error(S,
"nogds is not allowed");
7423 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7424 ImmTy = AMDGPUOperand::ImmTyR128A16;
7426 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7430unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7431 bool &Disabling)
const {
7432 Disabling =
Id.consume_front(
"no");
7435 return StringSwitch<unsigned>(Id)
7442 return StringSwitch<unsigned>(Id)
7452 SMLoc StringLoc = getLoc();
7454 int64_t CPolVal = 0;
7475 ResScope = parseScope(
Operands, Scope);
7488 if (trySkipId(
"nv")) {
7492 }
else if (trySkipId(
"no",
"nv")) {
7499 if (trySkipId(
"scale_offset")) {
7503 }
else if (trySkipId(
"no",
"scale_offset")) {
7527 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7528 AMDGPUOperand::ImmTyCPol));
7533 SMLoc OpLoc = getLoc();
7534 unsigned Enabled = 0, Seen = 0;
7538 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7545 return Error(S,
"dlc modifier is not supported on this GPU");
7548 return Error(S,
"scc modifier is not supported on this GPU");
7551 return Error(S,
"duplicate cache policy modifier");
7563 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7572 ParseStatus Res = parseStringOrIntWithPrefix(
7573 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7587 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7591 if (
Value ==
"TH_DEFAULT")
7593 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7594 Value ==
"TH_LOAD_NT_WB") {
7595 return Error(StringLoc,
"invalid th value");
7596 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7598 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7600 }
else if (
Value.consume_front(
"TH_STORE_")) {
7603 return Error(StringLoc,
"invalid th value");
7606 if (
Value ==
"BYPASS")
7611 TH |= StringSwitch<int64_t>(
Value)
7621 .Default(0xffffffff);
7623 TH |= StringSwitch<int64_t>(
Value)
7634 .Default(0xffffffff);
7637 if (TH == 0xffffffff)
7638 return Error(StringLoc,
"invalid th value");
7647 ParseStatus Res = parseStringOrIntWithPrefix(
7648 Operands,
"cfs", {
"CFS_256B",
"CFS_128B",
"CFS_64B",
"CFS_32B"},
CFS);
7658 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7659 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7660 std::optional<unsigned> InsertAt = std::nullopt) {
7661 auto i = OptionalIdx.find(ImmT);
7662 if (i != OptionalIdx.end()) {
7663 unsigned Idx = i->second;
7664 const AMDGPUOperand &
Op =
7665 static_cast<const AMDGPUOperand &
>(*
Operands[
Idx]);
7669 Op.addImmOperands(Inst, 1);
7671 if (InsertAt.has_value())
7678ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7684 StringLoc = getLoc();
7689ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7695 SMLoc StringLoc = getLoc();
7699 Value = getTokenStr();
7703 if (
Value == Ids[IntVal])
7708 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7709 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7714ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7716 AMDGPUOperand::ImmTy
Type) {
7720 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7722 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7731bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7734 SMLoc Loc = getLoc();
7736 auto Res = parseIntWithPrefix(Pref, Val);
7742 if (Val < 0 || Val > MaxVal) {
7743 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7752 AMDGPUOperand::ImmTy ImmTy) {
7753 const char *Pref =
"index_key";
7755 SMLoc Loc = getLoc();
7756 auto Res = parseIntWithPrefix(Pref, ImmVal);
7760 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7761 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7762 (ImmVal < 0 || ImmVal > 1))
7763 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7765 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7766 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7768 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7773 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7777 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7781 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7786 AMDGPUOperand::ImmTy
Type) {
7792 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7793 AMDGPUOperand::ImmTyMatrixAFMT);
7797 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7798 AMDGPUOperand::ImmTyMatrixBFMT);
7803 AMDGPUOperand::ImmTy
Type) {
7809 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7810 AMDGPUOperand::ImmTyMatrixAScale);
7814 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7815 AMDGPUOperand::ImmTyMatrixBScale);
7820 AMDGPUOperand::ImmTy
Type) {
7826 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7827 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7831 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7832 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7837ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7838 using namespace llvm::AMDGPU::MTBUFFormat;
7844 for (
int I = 0;
I < 2; ++
I) {
7845 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7848 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7853 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7859 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7862 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7863 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7869ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7870 using namespace llvm::AMDGPU::MTBUFFormat;
7874 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7877 if (Fmt == UFMT_UNDEF)
7884bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7885 StringRef FormatStr, SMLoc Loc) {
7886 using namespace llvm::AMDGPU::MTBUFFormat;
7890 if (
Format != DFMT_UNDEF) {
7896 if (
Format != NFMT_UNDEF) {
7901 Error(Loc,
"unsupported format");
7905ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7908 using namespace llvm::AMDGPU::MTBUFFormat;
7912 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7917 SMLoc Loc = getLoc();
7918 if (!parseId(Str,
"expected a format string") ||
7919 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7921 if (Dfmt == DFMT_UNDEF)
7922 return Error(Loc,
"duplicate numeric format");
7923 if (Nfmt == NFMT_UNDEF)
7924 return Error(Loc,
"duplicate data format");
7927 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7928 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7932 if (Ufmt == UFMT_UNDEF)
7933 return Error(FormatLoc,
"unsupported format");
7942ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7945 using namespace llvm::AMDGPU::MTBUFFormat;
7948 if (Id == UFMT_UNDEF)
7952 return Error(Loc,
"unified format is not supported on this GPU");
7958ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7959 using namespace llvm::AMDGPU::MTBUFFormat;
7960 SMLoc Loc = getLoc();
7965 return Error(Loc,
"out of range format");
7970ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7971 using namespace llvm::AMDGPU::MTBUFFormat;
7977 StringRef FormatStr;
7978 SMLoc Loc = getLoc();
7979 if (!parseId(FormatStr,
"expected a format string"))
7982 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7984 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7994 return parseNumericFormat(
Format);
7998 using namespace llvm::AMDGPU::MTBUFFormat;
8002 SMLoc Loc = getLoc();
8012 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
8031 Res = parseSymbolicOrNumericFormat(
Format);
8036 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
Size - 2]);
8037 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
8044 return Error(getLoc(),
"duplicate format");
8050 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
8052 Res = parseIntWithPrefix(
"inst_offset",
Operands,
8053 AMDGPUOperand::ImmTyInstOffset);
8060 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
8062 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
8068 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
8071 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8081 OptionalImmIndexMap OptionalIdx;
8083 unsigned OperandIdx[4];
8084 unsigned EnMask = 0;
8087 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8088 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8093 OperandIdx[SrcIdx] = Inst.
size();
8094 Op.addRegOperands(Inst, 1);
8101 OperandIdx[SrcIdx] = Inst.
size();
8107 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8108 Op.addImmOperands(Inst, 1);
8112 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8116 OptionalIdx[
Op.getImmTy()] = i;
8122 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8129 for (
auto i = 0; i < SrcIdx; ++i) {
8131 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8137 AMDGPUOperand::ImmTyExpCompr);
8147 int64_t CntVal,
bool Saturate,
8153 IntVal =
encode(ISA, IntVal, CntVal);
8154 if (CntVal !=
decode(ISA, IntVal)) {
8156 IntVal =
encode(ISA, IntVal, -1);
8164bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8166 SMLoc CntLoc = getLoc();
8167 StringRef CntName = getTokenStr();
8174 SMLoc ValLoc = getLoc();
8181 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8183 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8185 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8188 Error(CntLoc,
"invalid counter name " + CntName);
8193 Error(ValLoc,
"too large value for " + CntName);
8202 Error(getLoc(),
"expected a counter name");
8216 if (!parseCnt(Waitcnt))
8224 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8228bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8229 SMLoc FieldLoc = getLoc();
8230 StringRef FieldName = getTokenStr();
8235 SMLoc ValueLoc = getLoc();
8242 if (FieldName ==
"instid0") {
8244 }
else if (FieldName ==
"instskip") {
8246 }
else if (FieldName ==
"instid1") {
8249 Error(FieldLoc,
"invalid field name " + FieldName);
8268 .Case(
"VALU_DEP_1", 1)
8269 .Case(
"VALU_DEP_2", 2)
8270 .Case(
"VALU_DEP_3", 3)
8271 .Case(
"VALU_DEP_4", 4)
8272 .Case(
"TRANS32_DEP_1", 5)
8273 .Case(
"TRANS32_DEP_2", 6)
8274 .Case(
"TRANS32_DEP_3", 7)
8275 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8276 .Case(
"SALU_CYCLE_1", 9)
8277 .Case(
"SALU_CYCLE_2", 10)
8278 .Case(
"SALU_CYCLE_3", 11)
8286 Delay |=
Value << Shift;
8296 if (!parseDelay(Delay))
8304 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8308bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8310bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8316void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8317 StringRef DepCtrName) {
8320 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8323 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8326 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8329 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8336bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8338 using namespace llvm::AMDGPU::DepCtr;
8340 SMLoc DepCtrLoc = getLoc();
8341 StringRef DepCtrName = getTokenStr();
8351 unsigned PrevOprMask = UsedOprMask;
8352 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8355 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8364 Error(getLoc(),
"expected a counter name");
8369 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8370 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8375 using namespace llvm::AMDGPU::DepCtr;
8378 SMLoc Loc = getLoc();
8381 unsigned UsedOprMask = 0;
8383 if (!parseDepCtr(DepCtr, UsedOprMask))
8391 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8395bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8401ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8403 OperandInfoTy &Width) {
8404 using namespace llvm::AMDGPU::Hwreg;
8410 HwReg.Loc = getLoc();
8413 HwReg.IsSymbolic =
true;
8415 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8423 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8433 Width.Loc = getLoc();
8442 using namespace llvm::AMDGPU::Hwreg;
8445 SMLoc Loc = getLoc();
8447 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8449 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8450 HwregOffset::Default);
8451 struct : StructuredOpField {
8452 using StructuredOpField::StructuredOpField;
8453 bool validate(AMDGPUAsmParser &Parser)
const override {
8455 return Error(Parser,
"only values from 1 to 32 are legal");
8458 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8459 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8462 Res = parseHwregFunc(HwReg,
Offset, Width);
8465 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8467 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8471 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8478 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8480 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8484bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8490bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8491 OperandInfoTy &Stream) {
8492 using namespace llvm::AMDGPU::SendMsg;
8497 Msg.IsSymbolic =
true;
8504 Op.IsDefined =
true;
8510 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8515 Stream.IsDefined =
true;
8516 Stream.Loc = getLoc();
8525bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8526 const OperandInfoTy &
Op,
8527 const OperandInfoTy &Stream) {
8528 using namespace llvm::AMDGPU::SendMsg;
8537 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8542 Error(
Msg.Loc,
"invalid message id");
8548 Error(
Op.Loc,
"message does not support operations");
8550 Error(
Msg.Loc,
"missing message operation");
8556 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8558 Error(
Op.Loc,
"invalid operation id");
8563 Error(Stream.Loc,
"message operation does not support streams");
8567 Error(Stream.Loc,
"invalid message stream id");
8574 using namespace llvm::AMDGPU::SendMsg;
8577 SMLoc Loc = getLoc();
8581 OperandInfoTy
Op(OP_NONE_);
8582 OperandInfoTy Stream(STREAM_ID_NONE_);
8583 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8588 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8590 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8596 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8600bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8603 using namespace llvm::AMDGPU::WaitEvent;
8605 SMLoc Loc = getLoc();
8608 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8610 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8612 StructuredOpField *TargetBitfield =
8613 isGFX11() ? &DontWaitExportReady : &ExportReady;
8615 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8619 if (!validateStructuredOpFields({TargetBitfield}))
8621 ImmVal = TargetBitfield->Val;
8628 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8630 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8631 AMDGPUOperand::ImmTyWaitEvent));
8635bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8648 int Slot = StringSwitch<int>(Str)
8655 return Error(S,
"invalid interpolation slot");
8658 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8669 if (!Str.starts_with(
"attr"))
8670 return Error(S,
"invalid interpolation attribute");
8672 StringRef Chan = Str.take_back(2);
8673 int AttrChan = StringSwitch<int>(Chan)
8680 return Error(S,
"invalid or missing interpolation attribute channel");
8682 Str = Str.drop_back(2).drop_front(4);
8685 if (Str.getAsInteger(10, Attr))
8686 return Error(S,
"invalid or missing interpolation attribute number");
8689 return Error(S,
"out of bounds interpolation attribute number");
8694 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8695 Operands.push_back(AMDGPUOperand::CreateImm(
8696 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8705 using namespace llvm::AMDGPU::Exp;
8715 return Error(S, (Id == ET_INVALID)
8716 ?
"invalid exp target"
8717 :
"exp target is not supported on this GPU");
8720 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8728bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8732bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8737 return getTokenKind() ==
Kind;
8740StringRef AMDGPUAsmParser::getId()
const {
8744bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8752bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8754 StringRef Tok = getTokenStr();
8763bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8765 if (isId(Id) && peekToken().is(Kind)) {
8774 if (isToken(Kind)) {
8782 const StringRef ErrMsg) {
8783 if (!trySkipToken(Kind)) {
8784 Error(getLoc(), ErrMsg);
8790bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8794 if (Parser.parseExpression(Expr))
8797 if (Expr->evaluateAsAbsolute(
Imm))
8800 if (Expected.empty()) {
8801 Error(S,
"expected absolute expression");
8804 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8813 if (Parser.parseExpression(Expr))
8817 if (Expr->evaluateAsAbsolute(IntVal)) {
8818 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8820 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8825bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8827 Val =
getToken().getStringContents();
8831 Error(getLoc(), ErrMsg);
8835bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8837 Val = getTokenStr();
8841 if (!ErrMsg.
empty())
8842 Error(getLoc(), ErrMsg);
8846AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8848AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8851 : getLexer().peekTok(ShouldSkipSpace);
8855 auto TokCount = getLexer().peekTokens(Tokens);
8862 return getLexer().getKind();
8865SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8867StringRef AMDGPUAsmParser::getTokenStr()
const {
8871void AMDGPUAsmParser::lex() { Parser.Lex(); }
8873const AMDGPUOperand &
8875 int MCOpIdx)
const {
8877 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8878 if (TargetOp.getMCOpIdx() == MCOpIdx)
8885 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8889SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8894 int MCOpIdx)
const {
8895 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8898SMLoc AMDGPUAsmParser::getOperandLoc(
8899 std::function<
bool(
const AMDGPUOperand &)>
Test,
8901 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8902 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8904 return Op.getStartLoc();
8909SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8911 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8926 StringRef
Id = getTokenStr();
8927 SMLoc IdLoc = getLoc();
8933 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8934 if (
I == Fields.
end())
8935 return Error(IdLoc,
"unknown field");
8936 if ((*I)->IsDefined)
8937 return Error(IdLoc,
"duplicate field");
8940 (*I)->Loc = getLoc();
8943 (*I)->IsDefined =
true;
8950bool AMDGPUAsmParser::validateStructuredOpFields(
8952 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8953 return F->validate(*
this);
8963 const unsigned XorMask) {
8970bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8971 const unsigned MaxVal,
8972 const Twine &ErrMsg, SMLoc &Loc) {
8988bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8989 const unsigned MinVal,
8990 const unsigned MaxVal,
8991 const StringRef ErrMsg) {
8993 for (
unsigned i = 0; i < OpNum; ++i) {
8994 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
9001bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
9002 using namespace llvm::AMDGPU::Swizzle;
9005 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
9006 "expected a 2-bit lane id")) {
9016bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
9017 using namespace llvm::AMDGPU::Swizzle;
9023 if (!parseSwizzleOperand(GroupSize, 2, 32,
9024 "group size must be in the interval [2,32]", Loc)) {
9028 Error(Loc,
"group size must be a power of two");
9031 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
9032 "lane id must be in the interval [0,group size - 1]",
9040bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
9041 using namespace llvm::AMDGPU::Swizzle;
9046 if (!parseSwizzleOperand(GroupSize, 2, 32,
9047 "group size must be in the interval [2,32]", Loc)) {
9051 Error(Loc,
"group size must be a power of two");
9059bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
9060 using namespace llvm::AMDGPU::Swizzle;
9065 if (!parseSwizzleOperand(GroupSize, 1, 16,
9066 "group size must be in the interval [1,16]", Loc)) {
9070 Error(Loc,
"group size must be a power of two");
9078bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9079 using namespace llvm::AMDGPU::Swizzle;
9086 SMLoc StrLoc = getLoc();
9087 if (!parseString(Ctl)) {
9090 if (Ctl.
size() != BITMASK_WIDTH) {
9091 Error(StrLoc,
"expected a 5-character mask");
9095 unsigned AndMask = 0;
9096 unsigned OrMask = 0;
9097 unsigned XorMask = 0;
9099 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9103 Error(StrLoc,
"invalid mask");
9124bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9125 using namespace llvm::AMDGPU::Swizzle;
9128 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9134 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9135 "FFT swizzle must be in the interval [0," +
9136 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9144bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9145 using namespace llvm::AMDGPU::Swizzle;
9148 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9155 if (!parseSwizzleOperand(
Direction, 0, 1,
9156 "direction must be 0 (left) or 1 (right)", Loc))
9160 if (!parseSwizzleOperand(
9161 RotateSize, 0, ROTATE_MAX_SIZE,
9162 "number of threads to rotate must be in the interval [0," +
9163 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9168 (RotateSize << ROTATE_SIZE_SHIFT);
9172bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9174 SMLoc OffsetLoc = getLoc();
9180 Error(OffsetLoc,
"expected a 16-bit offset");
9186bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9187 using namespace llvm::AMDGPU::Swizzle;
9191 SMLoc ModeLoc = getLoc();
9194 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9195 Ok = parseSwizzleQuadPerm(
Imm);
9196 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9197 Ok = parseSwizzleBitmaskPerm(
Imm);
9198 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9199 Ok = parseSwizzleBroadcast(
Imm);
9200 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9201 Ok = parseSwizzleSwap(
Imm);
9202 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9203 Ok = parseSwizzleReverse(
Imm);
9204 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9205 Ok = parseSwizzleFFT(
Imm);
9206 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9207 Ok = parseSwizzleRotate(
Imm);
9209 Error(ModeLoc,
"expected a swizzle mode");
9212 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9222 if (trySkipId(
"offset")) {
9226 if (trySkipId(
"swizzle")) {
9227 Ok = parseSwizzleMacro(
Imm);
9229 Ok = parseSwizzleOffset(
Imm);
9234 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9241bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9247int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9249 using namespace llvm::AMDGPU::VGPRIndexMode;
9261 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9262 if (trySkipId(IdSymbolic[ModeId])) {
9270 ?
"expected a VGPR index mode or a closing parenthesis"
9271 :
"expected a VGPR index mode");
9276 Error(S,
"duplicate VGPR index mode");
9284 "expected a comma or a closing parenthesis"))
9293 using namespace llvm::AMDGPU::VGPRIndexMode;
9299 Imm = parseGPRIdxMacro();
9303 if (getParser().parseAbsoluteExpression(
Imm))
9306 return Error(S,
"invalid immediate: only 4-bit values are legal");
9310 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9314bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9325 if (isRegister() || isModifier())
9332 assert(Opr.isImm() || Opr.isExpr());
9333 SMLoc Loc = Opr.getStartLoc();
9337 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9338 Error(Loc,
"expected an absolute expression or a label");
9339 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9340 Error(Loc,
"expected a 16-bit signed jump offset");
9360 OptionalImmIndexMap OptionalIdx;
9361 unsigned FirstOperandIdx = 1;
9362 bool IsAtomicReturn =
false;
9368 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9369 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9373 Op.addRegOperands(Inst, 1);
9377 if (IsAtomicReturn && i == FirstOperandIdx)
9378 Op.addRegOperands(Inst, 1);
9383 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9384 Op.addImmOperands(Inst, 1);
9396 OptionalIdx[
Op.getImmTy()] = i;
9400 AMDGPUOperand::ImmTyOffset);
9416bool AMDGPUOperand::isSMRDOffset8()
const {
9420bool AMDGPUOperand::isSMEMOffset()
const {
9422 return isImmLiteral();
9425bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9460bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9461 if (BoundCtrl == 0 || BoundCtrl == 1) {
9469void AMDGPUAsmParser::onBeginOfFile() {
9470 if (!getParser().getStreamer().getTargetStreamer())
9473 if (!getTargetStreamer().getTargetID())
9474 getTargetStreamer().initializeTargetID(getSTI(),
9478void AMDGPUAsmParser::emitTargetDirective() {
9479 if (!getParser().getStreamer().getTargetStreamer() ||
9484 getTargetStreamer().emitDirectiveAMDGCNTargetOnce();
9493bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9497 StringRef TokenId = getTokenStr();
9498 AGVK VK = StringSwitch<AGVK>(TokenId)
9499 .Case(
"max", AGVK::AGVK_Max)
9500 .Case(
"min", AGVK::AGVK_Min)
9501 .Case(
"or", AGVK::AGVK_Or)
9502 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9503 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9504 .Case(
"alignto", AGVK::AGVK_AlignTo)
9505 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9506 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9507 .Default(AGVK::AGVK_None);
9516 if (Exprs.
empty()) {
9518 "empty " + Twine(TokenId) +
" expression");
9521 if (CommaCount + 1 != Exprs.
size()) {
9523 "mismatch of commas in " + Twine(TokenId) +
" expression");
9527 Expected && Exprs.
size() != Expected) {
9528 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9529 Twine(Expected) +
" operands");
9536 if (getParser().parseExpression(Expr, EndLoc))
9540 if (LastTokenWasComma)
9544 "unexpected token in " + Twine(TokenId) +
" expression");
9550 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9554 StringRef
Name = getTokenStr();
9555 if (Name ==
"mul") {
9556 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9560 if (Name ==
"div") {
9561 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9572 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9577 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9578 AMDGPU::OpName::src2};
9586 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9591 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9593 if (
DstOp.isReg() &&
9594 MRI.getRegClass(AMDGPU::VGPR_16RegClassID).contains(
DstOp.
getReg())) {
9598 if ((OpSel & (1 << SrcNum)) != 0)
9604void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9611 OptionalImmIndexMap &OptionalIdx) {
9612 cvtVOP3P(Inst,
Operands, OptionalIdx);
9621 &&
Desc.NumOperands > (OpNum + 1)
9623 &&
Desc.operands()[OpNum + 1].RegClass != -1
9625 &&
Desc.getOperandConstraint(OpNum + 1,
9629void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9631 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9632 AMDGPU::OpName::src2};
9633 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9634 AMDGPU::OpName::src1_modifiers,
9635 AMDGPU::OpName::src2_modifiers};
9636 for (
int J = 0; J < 3; ++J) {
9637 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9643 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9646 if ((OpSel & (1 << J)) != 0)
9649 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9656void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9658 OptionalImmIndexMap OptionalIdx;
9663 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9664 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9668 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9670 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9671 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9672 Op.isInterpAttrChan()) {
9674 }
else if (
Op.isImmModifier()) {
9675 OptionalIdx[
Op.getImmTy()] =
I;
9683 AMDGPUOperand::ImmTyHigh);
9687 AMDGPUOperand::ImmTyClamp);
9691 AMDGPUOperand::ImmTyOModSI);
9696 AMDGPUOperand::ImmTyOpSel);
9697 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9700 cvtOpSelHelper(Inst, OpSel);
9705 OptionalImmIndexMap OptionalIdx;
9710 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9711 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9715 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9717 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9718 }
else if (
Op.isImmModifier()) {
9719 OptionalIdx[
Op.getImmTy()] =
I;
9727 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9730 AMDGPUOperand::ImmTyOpSel);
9733 AMDGPUOperand::ImmTyWaitEXP);
9739 cvtOpSelHelper(Inst, OpSel);
9742void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9744 OptionalImmIndexMap OptionalIdx;
9747 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9751 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9752 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9755 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9760 if (NumOperands == CbszOpIdx) {
9765 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9766 }
else if (
Op.isImmModifier()) {
9767 OptionalIdx[
Op.getImmTy()] =
I;
9769 Op.addRegOrImmOperands(Inst, 1);
9774 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9775 if (CbszIdx != OptionalIdx.end()) {
9776 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9780 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9781 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9782 if (BlgpIdx != OptionalIdx.end()) {
9783 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9794 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9795 if (OpselIdx != OptionalIdx.end()) {
9796 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9800 unsigned OpSelHi = 0;
9801 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9802 if (OpselHiIdx != OptionalIdx.end()) {
9803 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9806 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9807 AMDGPU::OpName::src1_modifiers};
9809 for (
unsigned J = 0; J < 2; ++J) {
9810 unsigned ModVal = 0;
9811 if (OpSel & (1 << J))
9813 if (OpSelHi & (1 << J))
9816 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9822 OptionalImmIndexMap &OptionalIdx) {
9827 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9828 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9832 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9834 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9835 }
else if (
Op.isImmModifier()) {
9836 OptionalIdx[
Op.getImmTy()] =
I;
9838 Op.addRegOrImmOperands(Inst, 1);
9844 AMDGPUOperand::ImmTyScaleSel);
9848 AMDGPUOperand::ImmTyClamp);
9854 AMDGPUOperand::ImmTyByteSel);
9859 AMDGPUOperand::ImmTyOModSI);
9866 auto *it = Inst.
begin();
9868 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9877 OptionalImmIndexMap OptionalIdx;
9878 cvtVOP3(Inst,
Operands, OptionalIdx);
9882 OptionalImmIndexMap &OptIdx) {
9887 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9888 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9889 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9890 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9891 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9892 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9893 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9894 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9895 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9896 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9905 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9906 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9909 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9910 if (BitOp3Idx != -1) {
9917 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9918 if (OpSelIdx != -1) {
9922 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9923 if (OpSelHiIdx != -1) {
9924 int DefaultVal =
IsPacked ? -1 : 0;
9930 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9931 if (MatrixAFMTIdx != -1) {
9933 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9937 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9938 if (MatrixBFMTIdx != -1) {
9940 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9943 int MatrixAScaleIdx =
9944 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9945 if (MatrixAScaleIdx != -1) {
9947 AMDGPUOperand::ImmTyMatrixAScale, 0);
9950 int MatrixBScaleIdx =
9951 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9952 if (MatrixBScaleIdx != -1) {
9954 AMDGPUOperand::ImmTyMatrixBScale, 0);
9957 int MatrixAScaleFmtIdx =
9958 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9959 if (MatrixAScaleFmtIdx != -1) {
9961 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9964 int MatrixBScaleFmtIdx =
9965 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9966 if (MatrixBScaleFmtIdx != -1) {
9968 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9973 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9977 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9979 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9983 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9987 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9988 AMDGPU::OpName::src2};
9989 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9990 AMDGPU::OpName::src1_modifiers,
9991 AMDGPU::OpName::src2_modifiers};
9994 unsigned OpSelHi = 0;
10001 if (OpSelHiIdx != -1)
10004 if (NegLoIdx != -1)
10007 if (NegHiIdx != -1)
10010 for (
int J = 0; J < 3; ++J) {
10011 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
10015 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
10025 uint32_t ModVal = 0;
10027 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
10028 if (SrcOp.
isReg() && getMRI()
10032 if (VGPRSuffixIsHi)
10035 if ((OpSel & (1 << J)) != 0)
10039 if ((OpSelHi & (1 << J)) != 0)
10042 if ((NegLo & (1 << J)) != 0)
10045 if ((NegHi & (1 << J)) != 0)
10053 OptionalImmIndexMap OptIdx;
10059 unsigned i,
unsigned Opc,
10060 AMDGPU::OpName
OpName) {
10061 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
10062 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
10064 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
10070 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10073 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10074 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
10076 OptionalImmIndexMap OptIdx;
10077 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10078 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10079 OptIdx[
Op.getImmTy()] = i;
10084 AMDGPUOperand::ImmTyIndexKey8bit);
10088 AMDGPUOperand::ImmTyIndexKey16bit);
10092 AMDGPUOperand::ImmTyIndexKey32bit);
10109 SMLoc S = getLoc();
10112 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10113 SMLoc OpYLoc = getLoc();
10116 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10119 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10128 auto addOp = [&](uint16_t ParsedOprIdx) {
10129 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10131 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10135 Op.addRegOperands(Inst, 1);
10138 if (
Op.isImm() ||
Op.isExpr()) {
10139 Op.addImmOperands(Inst, 1);
10151 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10155 const auto &CInfo = InstInfo[CompIdx];
10156 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10157 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10158 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10159 if (CInfo.hasSrc2Acc())
10160 addOp(CInfo.getIndexOfDstInParsedOperands());
10164 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10165 if (BitOp3Idx != -1) {
10166 OptionalImmIndexMap OptIdx;
10167 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10169 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10179bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10181bool AMDGPUOperand::isDPPCtrl()
const {
10182 using namespace AMDGPU::DPP;
10184 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10187 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10188 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10189 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10190 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10191 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10192 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10193 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10194 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10195 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10196 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10197 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10198 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10207bool AMDGPUOperand::isBLGP()
const {
10211bool AMDGPUOperand::isS16Imm()
const {
10215bool AMDGPUOperand::isU16Imm()
const {
10223bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10228 SMLoc Loc =
getToken().getEndLoc();
10229 Token = std::string(getTokenStr());
10231 if (getLoc() != Loc)
10236 if (!parseId(Suffix))
10240 StringRef DimId = Token;
10255 SMLoc S = getLoc();
10261 SMLoc Loc = getLoc();
10262 if (!parseDimId(Encoding))
10263 return Error(Loc,
"invalid dim value");
10266 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10275 SMLoc S = getLoc();
10284 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10287 for (
size_t i = 0; i < 8; ++i) {
10291 SMLoc Loc = getLoc();
10292 if (getParser().parseAbsoluteExpression(Sels[i]))
10294 if (0 > Sels[i] || 7 < Sels[i])
10295 return Error(Loc,
"expected a 3-bit value");
10298 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10302 for (
size_t i = 0; i < 8; ++i)
10303 DPP8 |= (Sels[i] << (i * 3));
10306 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10310bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10312 if (Ctrl ==
"row_newbcast")
10315 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10318 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10319 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10322 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10323 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10327int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10330 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10334 for (
int i = 0; i < 4; ++i) {
10339 SMLoc Loc = getLoc();
10340 if (getParser().parseAbsoluteExpression(Temp))
10342 if (Temp < 0 || Temp > 3) {
10343 Error(Loc,
"expected a 2-bit value");
10347 Val += (Temp << i * 2);
10350 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10356int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10357 using namespace AMDGPU::DPP;
10362 SMLoc Loc = getLoc();
10364 if (getParser().parseAbsoluteExpression(Val))
10367 struct DppCtrlCheck {
10373 DppCtrlCheck
Check =
10374 StringSwitch<DppCtrlCheck>(Ctrl)
10375 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10376 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10377 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10378 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10379 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10380 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10381 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10382 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10383 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10384 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10388 if (
Check.Ctrl == -1) {
10389 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10397 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10405 using namespace AMDGPU::DPP;
10408 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10411 SMLoc S = getLoc();
10417 if (Ctrl ==
"row_mirror") {
10418 Val = DppCtrl::ROW_MIRROR;
10419 }
else if (Ctrl ==
"row_half_mirror") {
10420 Val = DppCtrl::ROW_HALF_MIRROR;
10423 if (Ctrl ==
"quad_perm") {
10424 Val = parseDPPCtrlPerm();
10426 Val = parseDPPCtrlSel(Ctrl);
10435 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10441 OptionalImmIndexMap OptionalIdx;
10448 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10450 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10451 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10455 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10456 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10460 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10461 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10462 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10463 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10464 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10465 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10466 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10467 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10468 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10474 if (OldIdx == NumOperands) {
10476 constexpr int DST_IDX = 0;
10478 }
else if (Src2ModIdx == NumOperands) {
10488 if (IsVOP3CvtSrDpp) {
10497 if (TiedTo != -1) {
10502 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10504 if (IsDPP8 &&
Op.isDppFI()) {
10507 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10508 }
else if (
Op.isReg()) {
10509 Op.addRegOperands(Inst, 1);
10510 }
else if (
Op.isImm() &&
10512 Op.addImmOperands(Inst, 1);
10513 }
else if (
Op.isImm()) {
10514 OptionalIdx[
Op.getImmTy()] =
I;
10522 AMDGPUOperand::ImmTyClamp);
10528 AMDGPUOperand::ImmTyByteSel);
10533 AMDGPUOperand::ImmTyOModSI);
10536 cvtVOP3P(Inst,
Operands, OptionalIdx);
10538 cvtVOP3OpSel(Inst,
Operands, OptionalIdx);
10541 AMDGPUOperand::ImmTyOpSel);
10546 AMDGPUOperand::ImmTyDPP8);
10547 using namespace llvm::AMDGPU::DPP;
10551 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10553 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10555 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10557 AMDGPUOperand::ImmTyDppBoundCtrl);
10561 AMDGPUOperand::ImmTyDppFI);
10567 OptionalImmIndexMap OptionalIdx;
10571 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10572 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10579 if (TiedTo != -1) {
10584 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10586 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {
10594 Op.addImmOperands(Inst, 1);
10596 Op.addRegWithFPInputModsOperands(Inst, 2);
10597 }
else if (
Op.isDppFI()) {
10599 }
else if (
Op.isReg()) {
10600 Op.addRegOperands(Inst, 1);
10606 Op.addRegWithFPInputModsOperands(Inst, 2);
10607 }
else if (
Op.isReg()) {
10608 Op.addRegOperands(Inst, 1);
10609 }
else if (
Op.isDPPCtrl()) {
10610 Op.addImmOperands(Inst, 1);
10611 }
else if (
Op.isImm()) {
10613 OptionalIdx[
Op.getImmTy()] =
I;
10621 using namespace llvm::AMDGPU::DPP;
10625 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10627 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10629 AMDGPUOperand::ImmTyDppBoundCtrl);
10632 AMDGPUOperand::ImmTyDppFI);
10643 AMDGPUOperand::ImmTy
Type) {
10644 return parseStringOrIntWithPrefix(
10646 {
"BYTE_0",
"BYTE_1",
"BYTE_2",
"BYTE_3",
"WORD_0",
"WORD_1",
"DWORD"},
10651 return parseStringOrIntWithPrefix(
10652 Operands,
"dst_unused", {
"UNUSED_PAD",
"UNUSED_SEXT",
"UNUSED_PRESERVE"},
10653 AMDGPUOperand::ImmTySDWADstUnused);
10657 cvtSDWA(Inst,
Operands, SDWAInstType::VOP1);
10661 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2);
10664void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst,
10666 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
true,
true);
10669void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst,
10671 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
false,
true);
10679 SDWAInstType BasicInstType,
bool SkipDstVcc,
10681 using namespace llvm::AMDGPU::SDWA;
10683 OptionalImmIndexMap OptionalIdx;
10684 bool SkipVcc = SkipDstVcc || SkipSrcVcc;
10685 bool SkippedVcc =
false;
10689 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10690 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10694 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10695 if (SkipVcc && !SkippedVcc &&
Op.isReg() &&
10696 (
Op.getReg() == AMDGPU::VCC ||
Op.getReg() == AMDGPU::VCC_LO)) {
10702 if (BasicInstType == SDWAInstType::VOP2 &&
10708 if (BasicInstType == SDWAInstType::VOPC && Inst.
getNumOperands() == 0) {
10714 Op.addRegOrImmWithInputModsOperands(Inst, 2);
10715 }
else if (
Op.isImm()) {
10717 OptionalIdx[
Op.getImmTy()] =
I;
10721 SkippedVcc =
false;
10725 if (
Opc != AMDGPU::V_NOP_sdwa_gfx10 &&
Opc != AMDGPU::V_NOP_sdwa_gfx9 &&
10726 Opc != AMDGPU::V_NOP_sdwa_vi) {
10728 switch (BasicInstType) {
10729 case SDWAInstType::VOP1:
10732 AMDGPUOperand::ImmTyClamp, 0);
10736 AMDGPUOperand::ImmTyOModSI, 0);
10740 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10744 AMDGPUOperand::ImmTySDWADstUnused,
10745 DstUnused::UNUSED_PRESERVE);
10748 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10751 case SDWAInstType::VOP2:
10753 AMDGPUOperand::ImmTyClamp, 0);
10757 AMDGPUOperand::ImmTyOModSI, 0);
10760 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10762 AMDGPUOperand::ImmTySDWADstUnused,
10763 DstUnused::UNUSED_PRESERVE);
10765 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10767 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10770 case SDWAInstType::VOPC:
10773 AMDGPUOperand::ImmTyClamp, 0);
10775 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10777 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10784 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
10785 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
10786 auto *it = Inst.
begin();
10788 it, AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::src2));
10801#define GET_MATCHER_IMPLEMENTATION
10802#define GET_MNEMONIC_SPELL_CHECKER
10803#define GET_MNEMONIC_CHECKER
10804#include "AMDGPUGenAsmMatcher.inc"
10810 return parseTokenOp(
"addr64",
Operands);
10812 return parseNamedBit(
"done",
Operands, AMDGPUOperand::ImmTyDone,
true);
10814 return parseTokenOp(
"idxen",
Operands);
10816 return parseNamedBit(
"lds",
Operands, AMDGPUOperand::ImmTyLDS,
10819 return parseTokenOp(
"offen",
Operands);
10821 return parseTokenOp(
"off",
Operands);
10822 case MCK_row_95_en:
10823 return parseNamedBit(
"row_en",
Operands, AMDGPUOperand::ImmTyRowEn,
true);
10825 return parseNamedBit(
"gds",
Operands, AMDGPUOperand::ImmTyGDS);
10827 return parseNamedBit(
"tfe",
Operands, AMDGPUOperand::ImmTyTFE);
10829 return tryCustomParseOperand(
Operands, MCK);
10834unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &
Op,
10840 AMDGPUOperand &Operand = (AMDGPUOperand &)
Op;
10843 return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
10845 return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
10847 return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
10849 return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
10851 return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
10853 return Operand.isTFE() ? Match_Success : Match_InvalidOperand;
10855 return Operand.isDone() ? Match_Success : Match_InvalidOperand;
10856 case MCK_row_95_en:
10857 return Operand.isRowEn() ? Match_Success : Match_InvalidOperand;
10865 return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand;
10867 return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand;
10868 case MCK_SOPPBrTarget:
10869 return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand;
10870 case MCK_VReg32OrOff:
10871 return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
10872 case MCK_InterpSlot:
10873 return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
10874 case MCK_InterpAttr:
10875 return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
10876 case MCK_InterpAttrChan:
10877 return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand;
10879 case MCK_SReg_64_XEXEC:
10889 return Operand.isNull() ? Match_Success : Match_InvalidOperand;
10891 return Match_InvalidOperand;
10900 SMLoc S = getLoc();
10909 return Error(S,
"expected a 16-bit value");
10912 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyEndpgm));
10916bool AMDGPUOperand::isEndpgm()
const {
return isImmTy(ImmTyEndpgm); }
10922bool AMDGPUOperand::isSplitBarrier()
const {
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
SmallVector< int16_t, MAX_SRC_OPERANDS_NUM > OperandIndices
static bool checkWriteLane(const MCInst &Inst)
static bool getRegNum(StringRef Str, unsigned &Num)
static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands, unsigned i, unsigned Opc, AMDGPU::OpName OpName)
static constexpr RegInfo RegularRegisters[]
static const RegInfo * getRegularRegInfo(StringRef Str)
static ArrayRef< unsigned > getAllVariants()
static OperandIndices getSrcOperandIndices(unsigned Opcode, bool AddMandatoryLiterals=false)
static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name, const MCRegisterInfo *MRI)
static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
static const fltSemantics * getFltSemantics(unsigned Size)
static bool isRegularReg(RegisterKind Kind)
LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUAsmParser()
Force static initialization.
static bool ConvertOmodMul(int64_t &Mul)
#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)
static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi)
static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT)
static bool AMDGPUCheckMnemonic(StringRef Mnemonic, const FeatureBitset &AvailableFeatures, unsigned VariantID)
static void applyMnemonicAliases(StringRef &Mnemonic, const FeatureBitset &Features, unsigned VariantID)
constexpr unsigned MAX_SRC_OPERANDS_NUM
#define EXPR_RESOLVE_OR_ERROR(RESOLVED)
static bool ConvertOmodDiv(int64_t &Div)
static bool IsRevOpcode(const unsigned Opcode)
static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal, int64_t CntVal, bool Saturate, unsigned(*encode)(const IsaVersion &Version, unsigned, unsigned), unsigned(*decode)(const IsaVersion &Version, unsigned))
static MCRegister getSpecialRegForName(StringRef RegName)
static void addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands, AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx, AMDGPUOperand::ImmTy ImmT, int64_t Default=0, std::optional< unsigned > InsertAt=std::nullopt)
static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI)
static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum)
static const fltSemantics * getOpFltSemantics(uint8_t OperandType)
static bool isInvalidVOPDY(const OperandVector &Operands, uint64_t InvalidOprIdx)
static std::string AMDGPUMnemonicSpellCheck(StringRef S, const FeatureBitset &FBS, unsigned VariantID=0)
static LLVM_READNONE unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask, const unsigned XorMask)
static bool isSafeTruncation(int64_t Val, unsigned Size)
AMDHSA kernel descriptor MCExpr struct for use in MC layer.
Provides AMDGPU specific target descriptions.
Enums shared between the AMDGPU backend (LLVM) and the ELF linker (LLD) for the .amdgpu....
AMDHSA kernel descriptor definitions.
static bool parseExpr(MCAsmParser &MCParser, const MCExpr *&Value, raw_ostream &Err)
MC layer struct for AMDGPUMCKernelCodeT, provides MCExpr functionality where required.
@ AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
static llvm::Expected< InlineInfo > decode(GsymDataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr)
Decode an InlineInfo in Data at the specified offset.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Loop::LoopBounds::Direction Direction
static bool hasFeature(StringRef Feature, const FeatureBitset &FeatureBits, ArrayRef< SubtargetFeatureKV > ProcFeatures)
Register const TargetRegisterInfo * TRI
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static bool isReg(const MCInst &MI, unsigned OpNo)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
MachineRegisterInfo & MRI
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Interface definition for SIInstrInfo.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the SmallBitVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
static const char * getRegisterName(MCRegister Reg)
static const AMDGPUMCExpr * createMax(ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static unsigned getNumExpectedArgs(VariantKind Kind)
static const AMDGPUMCExpr * createLit(LitModifier Lit, int64_t Value, MCContext &Ctx)
static const AMDGPUMCExpr * create(VariantKind Kind, ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static const AMDGPUMCExpr * createExtraSGPRs(const MCExpr *VCCUsed, const MCExpr *FlatScrUsed, bool XNACKUsed, MCContext &Ctx)
Allow delayed MCExpr resolve of ExtraSGPRs (in case VCCUsed or FlatScrUsed are unresolvable but neede...
static const AMDGPUMCExpr * createAlignTo(const MCExpr *Value, const MCExpr *Align, MCContext &Ctx)
static std::optional< TargetID > parseTargetIDString(StringRef TargetIDDirective)
Parse and validate a TargetID from a full "<triple>-<processor>:<features>" directive string.
TargetIDSetting getXnackSetting() const
GPUKind getGPUKind() const
StringRef getTargetTripleString() const
std::string toString() const
TargetIDSetting getSramEccSetting() const
static const fltSemantics & IEEEsingle()
static const fltSemantics & BFloat()
static const fltSemantics & IEEEdouble()
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
opStatus
IEEE-754R 7: Default exception handling.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
const T & front() const
Get the first element.
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
StringRef getString() const
Get the string for the current token, this includes all characters (for example, the quotes on string...
bool is(TokenKind K) const
Container class for subtarget features.
constexpr bool test(unsigned I) const
constexpr FeatureBitset & flip(unsigned I)
void printExpr(raw_ostream &, const MCExpr &) const
virtual void Initialize(MCAsmParser &Parser)
Initialize the extension for parsing using the given Parser.
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc=SMLoc())
static const MCBinaryExpr * createDiv(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Context object for machine code objects.
LLVM_ABI MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Instances of this class represent a single low-level machine instruction.
unsigned getNumOperands() const
unsigned getOpcode() const
iterator insert(iterator I, const MCOperand &Op)
void addOperand(const MCOperand Op)
const MCOperand & getOperand(unsigned i) const
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
const int16_t * getRegClassByHwModeTable(unsigned ModeId) const
int16_t getOpRegClassID(const MCOperandInfo &OpInfo, unsigned HwModeId) const
Return the ID of the register class to use for OpInfo, for the active HwMode HwModeId.
Instances of this class represent operands of the MCInst class.
static MCOperand createExpr(const MCExpr *Val)
static MCOperand createReg(MCRegister Reg)
static MCOperand createImm(int64_t Val)
void setReg(MCRegister Reg)
Set the register number.
MCRegister getReg() const
Returns the register number.
const MCExpr * getExpr() const
MCParsedAsmOperand - This abstract class represents a source-level assembly instruction operand.
MCRegisterClass - Base class of TargetRegisterClass.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
unsigned getNumRegs() const
getNumRegs - Return the number of registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
Generic base class for all target subtargets.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
StringRef getName() const
getName - Get the symbol name.
bool isVariable() const
isVariable - Check if this is a variable symbol.
LLVM_ABI void setVariableValue(const MCExpr *Value)
void setRedefinable(bool Value)
Mark this symbol as redefinable.
const MCExpr * getVariableValue() const
Get the expression of the variable symbol.
MCTargetAsmParser - Generic interface to target specific assembly parsers.
uint64_t getScalarSizeInBits() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Ternary parse status returned by various parse* methods.
constexpr bool isFailure() const
static constexpr StatusTy Failure
constexpr bool isSuccess() const
static constexpr StatusTy Success
static constexpr StatusTy NoMatch
constexpr bool isNoMatch() const
constexpr unsigned id() const
Represents a location in source code.
static SMLoc getFromPointer(const char *Ptr)
constexpr const char * getPointer() const
constexpr bool isValid() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
bool contains(StringRef key) const
Check if the set contains the given key.
std::pair< typename Base::iterator, bool > insert(StringRef key)
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
std::pair< iterator, bool > insert(const ValueT &V)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask, const MCSubtargetInfo &STI)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI)
unsigned getTgtId(const StringRef Name)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char NumSGPRs[]
Key for Kernel::CodeProps::Metadata::mNumSGPRs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr char AssemblerDirectiveBegin[]
HSA metadata beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
HSA metadata ending assembler directive.
constexpr char AssemblerDirectiveBegin[]
Old HSA metadata beginning assembler directive for V2.
int64_t getHwregId(StringRef Name, const MCSubtargetInfo &STI)
@ FIXED_NUM_SGPRS_FOR_INIT_BUG
unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI, std::optional< bool > EnableWavefrontSize32)
unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI)
bool targetIDSettingsConflict(TargetIDSetting Lhs, TargetIDSetting Rhs)
Returns true if Lhs and Rhs are incompatible (both specific but different).
unsigned getLocalMemorySize(const MCSubtargetInfo &STI)
constexpr char AssemblerDirective[]
PAL metadata (old linear format) assembler directive.
constexpr char AssemblerDirectiveBegin[]
PAL metadata (new MsgPack format) beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
PAL metadata (new MsgPack format) ending assembler directive.
int64_t getMsgOpId(int64_t MsgId, StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a sendmsg operation to the operation portion of the immediate encoding.
int64_t getMsgId(StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a msg_id to the message portion of the immediate encoding.
uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId)
bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI)
bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, const MCSubtargetInfo &STI, bool Strict)
bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, bool Strict)
ArrayRef< GFXVersion > getGFXVersions()
constexpr unsigned COMPONENTS[]
constexpr const char *const ModMatrixFmt[]
constexpr const char *const ModMatrixScaleFmt[]
constexpr const char *const ModMatrixScale[]
bool isInlineValue(MCRegister Reg)
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI)
Is Reg - scalar register.
MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
FuncInfoFlags
Per-function flags packed into INFO_FLAGS entries.
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const int OPR_ID_UNSUPPORTED
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
unsigned getTemporalHintType(const MCInstrDesc TID)
LLVM_READONLY bool isLitExpr(const MCExpr *Expr)
bool isInlinableLiteralV2BF16(uint32_t Literal)
LLVM_ABI bool isCPUValidForSubArch(Triple::SubArchType SubArch, GPUKind AK)
Return true if the GPU AK is usable with the triple subarch SubArch.
unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool hasA16(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset, bool IsBuffer)
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
bool hasPackedD16(const MCSubtargetInfo &STI)
bool isGFX940(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2F16(uint32_t Literal)
bool isHsaAbi(const MCSubtargetInfo &STI)
bool isGFX11(const MCSubtargetInfo &STI)
const int OPR_VAL_INVALID
bool getSMEMIsBuffer(unsigned Opc)
bool isPackedSingleSGPRFP32Inst(unsigned Opc)
The opcode is a packed fp32 instruction which only reads low 32 bits of a scalar operand and propagat...
LLVM_ABI unsigned getAddressableNumSGPRs(GPUKind AK)
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
LLVM_ABI unsigned getTotalNumVGPRs(GPUKind AK, bool IsWave32)
CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
bool isSI(const MCSubtargetInfo &STI)
bool hasPrivateApertureRegs(const MCSubtargetInfo &STI)
unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt)
unsigned getWaitcntBitMask(const IsaVersion &Version)
bool isGFX9(const MCSubtargetInfo &STI)
unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST)
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this a KImm operand?
GPUKind
GPU kinds supported by the AMDGPU target.
bool isGFX90A(const MCSubtargetInfo &STI)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByEncoding(uint8_t DimEnc)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool isGFX12(const MCSubtargetInfo &STI)
unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Expcnt)
bool hasMAIInsts(const MCSubtargetInfo &STI)
constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo)
Is this an AMDGPU specific source operand?
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi, bool *IsBF16)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByAsmSuffix(StringRef AsmSuffix)
bool hasMIMG_R128(const MCSubtargetInfo &STI)
LLVM_ABI GPUKind parseArchAMDGCN(StringRef CPU)
bool hasG16(const MCSubtargetInfo &STI)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI)
LLVM_READONLY int64_t getLitValue(const MCExpr *Expr)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this floating-point operand?
bool isGFX10Plus(const MCSubtargetInfo &STI)
AMDGPU::TargetID TargetID
int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
@ OPERAND_REG_INLINE_C_FP64
@ OPERAND_REG_IMM_NOINLINE_FP16
@ OPERAND_REG_INLINE_C_BF16
@ OPERAND_REG_INLINE_C_V2BF16
@ OPERAND_REG_IMM_V2INT64
@ OPERAND_REG_IMM_V2INT16
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
@ OPERAND_REG_IMM_V2FP16_SPLAT
@ OPERAND_REG_INLINE_C_INT64
@ OPERAND_REG_INLINE_C_INT16
Operands with register or inline constant.
@ OPERAND_REG_IMM_NOINLINE_V2FP16
@ OPERAND_REG_INLINE_C_V2FP16
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
@ OPERAND_REG_INLINE_AC_FP32
@ OPERAND_REG_IMM_V2INT32
@ OPERAND_REG_INLINE_C_FP32
@ OPERAND_REG_INLINE_C_INT32
@ OPERAND_REG_INLINE_C_V2INT16
@ OPERAND_REG_INLINE_AC_FP64
@ OPERAND_REG_INLINE_C_FP16
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
constexpr bool isBF16SrcOperand(const MCOperandInfo &OpInfo)
Is this a scalar (i.e. not packed) bf16 source operand?
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
LLVM_ABI StringRef getArchNameAMDGCN(GPUKind AK)
bool hasGDS(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset)
bool isGFX9Plus(const MCSubtargetInfo &STI)
bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI)
const int OPR_ID_DUPLICATE
bool isVOPD(unsigned Opc)
VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Vmcnt)
unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt)
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
bool isVI(const MCSubtargetInfo &STI)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
MCRegister mc2PseudoReg(MCRegister Reg)
Convert hardware register Reg to a pseudo register.
unsigned hasKernargPreload(const MCSubtargetInfo &STI)
bool supportsWGP(const MCSubtargetInfo &STI)
LLVM_READNONE unsigned getOperandSize(const MCOperandInfo &OpInfo)
bool isCI(const MCSubtargetInfo &STI)
unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Lgkmcnt)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
bool isGFX1250Plus(const MCSubtargetInfo &STI)
bool hasPopsExitingWaveID(const MCSubtargetInfo &STI)
unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
bool hasVOPD(const MCSubtargetInfo &STI)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
bool isPermlane16(unsigned Opc)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ UNDEF
UNDEF - An undefined node.
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
void validate(const Triple &TT, const FeatureBitset &FeatureBits)
constexpr bool isAtomicRet(const T &...O)
constexpr bool isVOPC(const T &...O)
constexpr bool isVOP3(const T &...O)
constexpr bool isVOP1(const T &...O)
constexpr bool usesTENSOR_CNT(const T &...O)
constexpr bool isMAI(const T &...O)
constexpr bool isVOP2(const T &...O)
constexpr bool isSWMMAC(const T &...O)
constexpr bool isSOP2(const T &...O)
constexpr bool isFLAT(const T &...O)
constexpr bool isVOP3P(const T &...O)
constexpr bool isBuffer(const T &...O)
constexpr bool hasIntClamp(const T &...O)
constexpr bool isAtomicNoRet(const T &...O)
constexpr bool isSMRD(const T &...O)
constexpr bool isVOP3Like(const T &...O)
constexpr bool isMIMG(const T &...O)
constexpr bool isVMEM(const T &...O)
constexpr bool isImage(const T &...O)
constexpr bool isWMMA(const T &...O)
constexpr bool isVOPD3(const T &...O)
constexpr bool isGWS(const T &...O)
constexpr bool isMUBUF(const T &...O)
constexpr bool isSDWA(const T &...O)
constexpr bool isSOPC(const T &...O)
constexpr bool isDOT(const T &...O)
constexpr bool isVSAMPLE(const T &...O)
constexpr bool isDS(const T &...O)
constexpr bool isAtomic(const T &...O)
constexpr bool isGather4(const T &...O)
constexpr bool isPacked(const T &...O)
constexpr bool isDPP(const T &...O)
constexpr bool isSegmentSpecificFLAT(const T &...O)
@ Valid
The data is already valid.
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
EnumSet< Modifier > Modifiers
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
bool errorToBool(Error Err)
Helper for converting an Error to a bool.
StringMapEntry< Value * > ValueName
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
unsigned encode(MaybeAlign A)
Returns a representation of the alignment that encodes undefined as 0.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
static bool isMem(const MachineInstr &MI, unsigned Op)
LLVM_ABI std::pair< StringRef, StringRef > getToken(StringRef Source, StringRef Delimiters=" \t\n\v\f\r")
getToken - This function extracts one token from source, ignoring any leading characters that appear ...
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
testing::Matcher< const detail::ErrorHolder & > Failed()
LLVM_ABI void PrintError(const Twine &Msg)
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Target & getTheR600Target()
The target for R600 GPUs.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
SmallVectorImpl< std::unique_ptr< MCParsedAsmOperand > > OperandVector
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Target & getTheGCNTarget()
The target for GCN GPUs.
@ Sub
Subtraction of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
unsigned M0(unsigned Val)
ArrayRef(const T &OneElt) -> ArrayRef< T >
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Target & getTheGCNLegacyTarget()
The target for GCN GPUs, registered under the legacy "amdgcn" architecture name for use with -march.
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
void initDefault(const MCSubtargetInfo &STI, MCContext &Ctx, bool InitMCExpr=true)
void validate(const MCSubtargetInfo *STI, MCContext &Ctx)
uint32_t PrivateSegmentSize
SmallVector< std::pair< MCSymbol *, std::string >, 4 > IndirectCalls
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 8 > Calls
SmallVector< FuncInfo, 8 > Funcs
SmallVector< std::pair< MCSymbol *, std::string >, 4 > TypeIds
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 4 > Uses
Instruction set architecture version.
const MCExpr * compute_pgm_rsrc2
const MCExpr * kernarg_size
const MCExpr * kernarg_preload
const MCExpr * compute_pgm_rsrc3
const MCExpr * private_segment_fixed_size
const MCExpr * compute_pgm_rsrc1
static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, uint32_t Mask, MCContext &Ctx)
const MCExpr * group_segment_fixed_size
static MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, MCContext &Ctx)
const MCExpr * kernel_code_properties
RegisterMCAsmParser - Helper template for registering a target specific assembly parser,...
uint32_t group_segment_fixed_size
uint32_t private_segment_fixed_size
1.14.0