70 enum KindTy { Token, Immediate, Register, Expression } Kind;
72 SMLoc StartLoc, EndLoc;
73 const AMDGPUAsmParser *AsmParser;
76 AMDGPUOperand(KindTy Kind_,
const AMDGPUAsmParser *AsmParser_)
77 : Kind(Kind_), AsmParser(AsmParser_) {}
79 using Ptr = std::unique_ptr<AMDGPUOperand>;
87 bool hasFPModifiers()
const {
return Abs || Neg; }
88 bool hasIntModifiers()
const {
return Sext; }
89 bool hasModifiers()
const {
return hasFPModifiers() || hasIntModifiers(); }
90 bool isForcedLit()
const {
return Lit == LitModifier::Lit; }
91 bool isForcedLit64()
const {
return Lit == LitModifier::Lit64; }
93 int64_t getFPModifiersOperand()
const {
100 int64_t getIntModifiersOperand()
const {
106 int64_t getModifiersOperand()
const {
107 assert(!(hasFPModifiers() && hasIntModifiers()) &&
108 "fp and int modifiers should not be used simultaneously");
109 if (hasFPModifiers())
110 return getFPModifiersOperand();
111 if (hasIntModifiers())
112 return getIntModifiersOperand();
116 friend raw_ostream &
operator<<(raw_ostream &OS,
117 AMDGPUOperand::Modifiers Mods);
191 ImmTyMatrixAScaleFmt,
192 ImmTyMatrixBScaleFmt,
225 mutable int MCOpIdx = -1;
228 bool isToken()
const override {
return Kind == Token; }
230 bool isSymbolRefExpr()
const {
234 bool isImm()
const override {
return Kind == Immediate; }
236 bool isInlinableImm(MVT type)
const;
237 bool isLiteralImm(MVT type)
const;
239 bool isRegKind()
const {
return Kind == Register; }
241 bool isReg()
const override {
return isRegKind() && !hasModifiers(); }
243 bool isRegOrInline(
unsigned RCID, MVT type)
const {
244 return isRegClass(RCID) || isInlinableImm(type);
247 bool isRegOrInlineTarget(
unsigned TargetRCIdx, MVT type)
const {
248 return isRegClassTarget(TargetRCIdx) || isInlinableImm(type);
252 return isRegOrInline(RCID, type) || isLiteralImm(type);
255 bool isRegOrImmWithInputModsTarget(
unsigned TargetRCIdx, MVT type)
const {
256 return isRegOrInlineTarget(TargetRCIdx, type) || isLiteralImm(type);
259 template <
bool IsFake16>
bool isRegOrImmWithIntT16InputMods()
const {
261 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
264 bool isRegOrImmWithInt32InputMods()
const {
268 template <
bool IsFake16>
bool isRegOrInlineImmWithIntT16InputMods()
const {
269 return isRegOrInline(
270 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::i16);
273 bool isRegOrInlineImmWithInt32InputMods()
const {
274 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
277 bool isRegOrImmWithInt64InputMods()
const {
278 return isRegOrImmWithInputModsTarget(AMDGPU::VS_64_AlignTarget, MVT::i64);
281 bool isRegOrImmWithFP16InputMods()
const {
285 template <
bool IsFake16>
bool isRegOrImmWithFPT16InputMods()
const {
287 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
290 bool isRegOrImmWithFPT16_LO16InputMods()
const {
294 bool isRegOrImmWithFP32InputMods()
const {
298 bool isRegOrImmWithFP64InputMods()
const {
299 return isRegOrImmWithInputModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
302 template <
bool IsFake16>
bool isRegOrInlineImmWithFP16InputMods()
const {
303 return isRegOrInline(
304 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
307 bool isRegOrInlineImmWithFP32InputMods()
const {
308 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
311 bool isRegOrInlineImmWithFP64InputMods()
const {
312 return isRegOrInlineTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
315 bool isVRegWithInputMods(
unsigned RCID)
const {
return isRegClass(RCID); }
317 bool isVRegWithFP32InputMods()
const {
318 return isVRegWithInputMods(AMDGPU::VGPR_32RegClassID);
321 bool isVRegWithFP64InputMods()
const {
322 return isRegClassTarget(AMDGPU::VReg_64_AlignTarget);
325 bool isPackedFP16InputMods()
const {
329 bool isPackedVGPRFP32InputMods()
const {
333 bool isVReg32()
const {
return isRegClass(AMDGPU::VGPR_32RegClassID); }
335 bool isVReg32OrOff()
const {
return isOff() || isVReg32(); }
337 bool isRsrcReg32()
const {
return isRegClass(AMDGPU::RsrcReg32RegClassID); }
339 bool isNull()
const {
return isRegKind() &&
getReg() == AMDGPU::SGPR_NULL; }
341 bool isAV_LdSt_32_Align2_RegOp()
const {
342 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
343 isRegClass(AMDGPU::AGPR_32RegClassID);
346 bool isVRegWithInputMods()
const;
347 template <
bool IsFake16>
bool isT16_Lo128VRegWithInputMods()
const;
348 template <
bool IsFake16>
bool isT16VRegWithInputMods()
const;
349 bool isT16_LO16VRegWithInputMods()
const;
351 bool isSDWAOperand(MVT type)
const;
352 bool isSDWAFP16Operand()
const;
353 bool isSDWAFP32Operand()
const;
354 bool isSDWAInt16Operand()
const;
355 bool isSDWAInt32Operand()
const;
357 bool isImmTy(ImmTy ImmT)
const {
return isImm() &&
Imm.Type == ImmT; }
359 template <ImmTy Ty>
bool isImmTy()
const {
return isImmTy(Ty); }
361 bool isImmLiteral()
const {
return isImmTy(ImmTyNone); }
363 bool isImmModifier()
const {
return isImm() &&
Imm.Type != ImmTyNone; }
365 bool isOModSI()
const {
return isImmTy(ImmTyOModSI); }
366 bool isDim()
const {
return isImmTy(ImmTyDim); }
367 bool isR128A16()
const {
return isImmTy(ImmTyR128A16); }
368 bool isOff()
const {
return isImmTy(ImmTyOff); }
369 bool isExpTgt()
const {
return isImmTy(ImmTyExpTgt); }
370 bool isOffen()
const {
return isImmTy(ImmTyOffen); }
371 bool isIdxen()
const {
return isImmTy(ImmTyIdxen); }
372 bool isAddr64()
const {
return isImmTy(ImmTyAddr64); }
373 bool isSMEMOffsetMod()
const {
return isImmTy(ImmTySMEMOffsetMod); }
374 bool isFlatOffset()
const {
375 return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset);
377 bool isGDS()
const {
return isImmTy(ImmTyGDS); }
378 bool isLDS()
const {
return isImmTy(ImmTyLDS); }
379 bool isCPol()
const {
return isImmTy(ImmTyCPol); }
380 bool isIndexKey8bit()
const {
return isImmTy(ImmTyIndexKey8bit); }
381 bool isIndexKey16bit()
const {
return isImmTy(ImmTyIndexKey16bit); }
382 bool isIndexKey32bit()
const {
return isImmTy(ImmTyIndexKey32bit); }
383 bool isMatrixAFMT()
const {
return isImmTy(ImmTyMatrixAFMT); }
384 bool isMatrixBFMT()
const {
return isImmTy(ImmTyMatrixBFMT); }
385 bool isMatrixAScale()
const {
return isImmTy(ImmTyMatrixAScale); }
386 bool isMatrixBScale()
const {
return isImmTy(ImmTyMatrixBScale); }
387 bool isMatrixAScaleFmt()
const {
return isImmTy(ImmTyMatrixAScaleFmt); }
388 bool isMatrixBScaleFmt()
const {
return isImmTy(ImmTyMatrixBScaleFmt); }
389 bool isTFE()
const {
return isImmTy(ImmTyTFE); }
390 bool isFORMAT()
const {
return isImmTy(ImmTyFORMAT) &&
isUInt<7>(
getImm()); }
391 bool isDppFI()
const {
return isImmTy(ImmTyDppFI); }
392 bool isSDWADstSel()
const {
return isImmTy(ImmTySDWADstSel); }
393 bool isSDWASrc0Sel()
const {
return isImmTy(ImmTySDWASrc0Sel); }
394 bool isSDWASrc1Sel()
const {
return isImmTy(ImmTySDWASrc1Sel); }
395 bool isSDWADstUnused()
const {
return isImmTy(ImmTySDWADstUnused); }
396 bool isInterpSlot()
const {
return isImmTy(ImmTyInterpSlot); }
397 bool isInterpAttr()
const {
return isImmTy(ImmTyInterpAttr); }
398 bool isInterpAttrChan()
const {
return isImmTy(ImmTyInterpAttrChan); }
399 bool isOpSel()
const {
return isImmTy(ImmTyOpSel); }
400 bool isOpSelHi()
const {
return isImmTy(ImmTyOpSelHi); }
401 bool isNegLo()
const {
return isImmTy(ImmTyNegLo); }
402 bool isNegHi()
const {
return isImmTy(ImmTyNegHi); }
403 bool isDone()
const {
return isImmTy(ImmTyDone); }
404 bool isRowEn()
const {
return isImmTy(ImmTyRowEn); }
406 bool isRegOrImm()
const {
return isReg() || isImm(); }
408 bool isRegClass(
unsigned RCID)
const;
411 bool isRegClassTarget(
unsigned TargetRCIdx)
const;
415 bool isRegOrInlineNoMods(
unsigned RCID, MVT type)
const {
416 return isRegOrInline(RCID, type) && !hasModifiers();
419 bool isRegOrInlineNoModsTarget(
unsigned TargetRCIdx, MVT type)
const {
420 return isRegOrInlineTarget(TargetRCIdx, type) && !hasModifiers();
423 bool isSCSrcB16()
const {
424 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
427 bool isSCSrc_b32()
const {
428 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
431 bool isSCSrc_b64()
const {
432 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
435 bool isBoolReg()
const;
437 bool isSSrc_b32()
const {
438 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
441 bool isSSrc_b16()
const {
return isSCSrcB16() || isLiteralImm(MVT::i16); }
443 bool isSSrc_b64()
const {
446 return isSCSrc_b64() || isLiteralImm(MVT::i64) ||
447 (((
const MCTargetAsmParser *)AsmParser)
448 ->getAvailableFeatures()[AMDGPU::Feature64BitLiterals] &&
452 bool isSSrc_f32()
const {
453 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
456 bool isSSrc_bf16()
const {
return isSCSrcB16() || isLiteralImm(MVT::bf16); }
458 bool isSSrc_f16()
const {
return isSCSrcB16() || isLiteralImm(MVT::f16); }
460 bool isSSrc_NoInline_f16()
const {
return isSSrc_f16(); }
462 bool isSSrcOrLds_b32()
const {
463 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
464 isLiteralImm(MVT::i32) || isExpr();
467 bool isVCSrc_b32()
const {
468 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
471 bool isVCSrc_b32_Lo256()
const {
472 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo256RegClassID, MVT::i32);
475 bool isVCSrc_b64_Lo256()
const {
476 return isRegOrInlineNoMods(AMDGPU::VS_64_Lo256RegClassID, MVT::i64);
479 bool isVCSrc_b64()
const {
480 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::i64);
483 bool isVCSrcT_b16()
const {
484 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
487 bool isVCSrcTB16_Lo128()
const {
488 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
491 bool isVCSrcFake16B16_Lo128()
const {
492 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
495 bool isVCSrc_b16()
const {
496 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
499 bool isVCSrc_v2b16()
const {
return isVCSrc_b16(); }
501 bool isVCSrc_f32()
const {
502 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
505 bool isVCSrc_f64()
const {
506 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64);
509 bool isVCSrcTBF16()
const {
510 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
513 bool isVCSrcT_f16()
const {
514 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
517 bool isVCSrcT_bf16()
const {
518 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
521 bool isVCSrcTF16_LO16()
const {
522 return isRegOrInlineNoMods(AMDGPU::VS_16_LO16RegClassID, MVT::f16);
525 bool isVCSrcTBF16_Lo128()
const {
526 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
529 bool isVCSrcTF16_Lo128()
const {
530 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
533 bool isVCSrcFake16BF16_Lo128()
const {
534 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
537 bool isVCSrcFake16F16_Lo128()
const {
538 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
541 bool isVCSrc_bf16()
const {
542 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
545 bool isVCSrc_f16()
const {
546 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
549 bool isVCSrc_v2bf16()
const {
return isVCSrc_bf16(); }
551 bool isVCSrc_v2f16()
const {
return isVCSrc_f16(); }
553 bool isVSrc_b32()
const {
554 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
557 bool isVSrc_b64()
const {
return isVCSrc_f64() || isLiteralImm(MVT::i64); }
559 bool isVSrc_v2b64()
const {
560 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::i64) ||
561 isLiteralImm(MVT::i64);
564 bool isVSrc_v2f64()
const {
565 return isRegOrInlineNoMods(AMDGPU::VS_128RegClassID, MVT::f64) ||
566 isLiteralImm(MVT::f64);
569 bool isVSrcT_b16()
const {
return isVCSrcT_b16() || isLiteralImm(MVT::i16); }
571 bool isVSrcT_b16_Lo128()
const {
572 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
575 bool isVSrcFake16_b16_Lo128()
const {
576 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
579 bool isVSrc_b16()
const {
return isVCSrc_b16() || isLiteralImm(MVT::i16); }
581 bool isVSrc_v2b16()
const {
return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
583 bool isVSrc_v2f32()
const {
return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
585 bool isVCSrc_v2b32()
const {
return isVCSrc_b64(); }
587 bool isVSrc_v2b32()
const {
return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
589 bool isVSrc_f32()
const {
590 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
593 bool isVSrc_f64()
const {
594 return isRegOrInlineNoModsTarget(AMDGPU::VS_64_AlignTarget, MVT::f64) ||
595 isLiteralImm(MVT::f64);
598 bool isVSrcT_bf16()
const {
599 return isVCSrcTBF16() || isLiteralImm(MVT::bf16);
602 bool isVSrcT_f16()
const {
return isVCSrcT_f16() || isLiteralImm(MVT::f16); }
604 bool isVSrcT_f16_LO16()
const {
605 return isVCSrcTF16_LO16() || isLiteralImm(MVT::f16);
608 bool isVSrcT_bf16_Lo128()
const {
609 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
612 bool isVSrcT_f16_Lo128()
const {
613 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
616 bool isVSrcFake16_bf16_Lo128()
const {
617 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
620 bool isVSrcFake16_f16_Lo128()
const {
621 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
624 bool isVSrc_bf16()
const {
return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
626 bool isVSrc_f16()
const {
return isVCSrc_f16() || isLiteralImm(MVT::f16); }
628 bool isVSrc_v2bf16()
const {
629 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
632 bool isVSrc_v2f16()
const {
return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
634 bool isVSrc_v2f16_splat()
const {
return isVSrc_v2f16(); }
636 bool isVSrc_NoInline_v2f16()
const {
return isVSrc_v2f16(); }
638 bool isVISrc_64_bf16()
const {
639 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::bf16);
642 bool isVISrc_64_f16()
const {
643 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::f16);
646 bool isVISrc_64_b32()
const {
647 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::i32);
650 bool isVISrc_64_f64()
const {
651 return isRegOrInlineNoModsTarget(AMDGPU::VReg_64_AlignTarget, MVT::f64);
654 bool isVISrc_256_b32()
const {
655 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::i32);
658 bool isVISrc_256_f32()
const {
659 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::f32);
662 bool isVISrc_256_f64()
const {
663 return isRegOrInlineNoModsTarget(AMDGPU::VReg_256_AlignTarget, MVT::f64);
666 bool isVISrc_512_f64()
const {
667 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::f64);
670 bool isVISrc_128_b32()
const {
671 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::i32);
674 bool isVISrc_128_f32()
const {
675 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::f32);
678 bool isVISrc_512_b32()
const {
679 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::i32);
682 bool isVISrc_512_f32()
const {
683 return isRegOrInlineNoModsTarget(AMDGPU::VReg_512_AlignTarget, MVT::f32);
686 bool isVISrc_1024_b32()
const {
687 return isRegOrInlineNoModsTarget(AMDGPU::VReg_1024_AlignTarget, MVT::i32);
690 bool isVISrc_1024_f32()
const {
691 return isRegOrInlineNoModsTarget(AMDGPU::VReg_1024_AlignTarget, MVT::f32);
694 bool isAISrc_64_f64()
const {
695 return isRegOrInlineNoModsTarget(AMDGPU::AReg_64_AlignTarget, MVT::f64);
698 bool isAISrc_128_b32()
const {
699 return isRegOrInlineNoModsTarget(AMDGPU::AReg_128_AlignTarget, MVT::i32);
702 bool isAISrc_128_f32()
const {
703 return isRegOrInlineNoModsTarget(AMDGPU::AReg_128_AlignTarget, MVT::f32);
706 bool isVISrc_128_bf16()
const {
707 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::bf16);
710 bool isVISrc_128_f16()
const {
711 return isRegOrInlineNoModsTarget(AMDGPU::VReg_128_AlignTarget, MVT::f16);
714 bool isAISrc_256_f64()
const {
715 return isRegOrInlineNoModsTarget(AMDGPU::AReg_256_AlignTarget, MVT::f64);
718 bool isAISrc_512_b32()
const {
719 return isRegOrInlineNoModsTarget(AMDGPU::AReg_512_AlignTarget, MVT::i32);
722 bool isAISrc_512_f32()
const {
723 return isRegOrInlineNoModsTarget(AMDGPU::AReg_512_AlignTarget, MVT::f32);
726 bool isAISrc_1024_b32()
const {
727 return isRegOrInlineNoModsTarget(AMDGPU::AReg_1024_AlignTarget, MVT::i32);
730 bool isAISrc_1024_f32()
const {
731 return isRegOrInlineNoModsTarget(AMDGPU::AReg_1024_AlignTarget, MVT::f32);
734 bool isKImmFP32()
const {
return isLiteralImm(MVT::f32); }
736 bool isKImmFP16()
const {
return isLiteralImm(MVT::f16); }
738 bool isKImmFP64()
const {
return isLiteralImm(MVT::f64); }
740 bool isMem()
const override {
return false; }
742 bool isExpr()
const {
return Kind == Expression; }
744 bool isSOPPBrTarget()
const {
return isExpr() || isImm(); }
746 bool isSWaitCnt()
const;
747 bool isDepCtr()
const;
748 bool isSDelayALU()
const;
749 bool isHwreg()
const;
750 bool isSendMsg()
const;
751 bool isWaitEvent()
const;
752 bool isSplitBarrier()
const;
753 bool isSwizzle()
const;
754 bool isSMRDOffset8()
const;
755 bool isSMEMOffset()
const;
756 bool isSMRDLiteralOffset()
const;
758 bool isDPPCtrl()
const;
760 bool isGPRIdxMode()
const;
761 bool isS16Imm()
const;
762 bool isU16Imm()
const;
763 bool isEndpgm()
const;
765 auto getPredicate(std::function<
bool(
const AMDGPUOperand &
Op)>
P)
const {
766 return [
this,
P]() {
return P(*
this); };
771 return StringRef(Tok.Data, Tok.Length);
779 void setImm(int64_t Val) {
784 ImmTy getImmTy()
const {
789 MCRegister
getReg()
const override {
794 SMLoc getStartLoc()
const override {
return StartLoc; }
796 SMLoc getEndLoc()
const override {
return EndLoc; }
798 int getMCOpIdx()
const {
return MCOpIdx; }
800 Modifiers getModifiers()
const {
801 assert(isRegKind() || isImmTy(ImmTyNone));
802 return isRegKind() ?
Reg.Mods :
Imm.Mods;
805 void setModifiers(Modifiers Mods) {
806 assert(isRegKind() || isImmTy(ImmTyNone));
813 bool hasModifiers()
const {
return getModifiers().hasModifiers(); }
815 bool hasFPModifiers()
const {
return getModifiers().hasFPModifiers(); }
817 bool hasIntModifiers()
const {
return getModifiers().hasIntModifiers(); }
819 bool isForcedLit()
const {
820 return isImmLiteral() && getModifiers().isForcedLit();
823 bool isForcedLit64()
const {
824 return isImmLiteral() && getModifiers().isForcedLit64();
829 void addImmOperands(MCInst &Inst,
unsigned N,
830 bool ApplyModifiers =
true)
const;
832 void addLiteralImmOperand(MCInst &Inst, int64_t Val,
833 bool ApplyModifiers)
const;
835 void addRegOperands(MCInst &Inst,
unsigned N)
const;
837 void addRegOrImmOperands(MCInst &Inst,
unsigned N)
const {
839 addRegOperands(Inst,
N);
841 addImmOperands(Inst,
N);
844 void addRegOrImmWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
845 Modifiers Mods = getModifiers();
848 addRegOperands(Inst,
N);
850 addImmOperands(Inst,
N,
false);
854 void addRegOrImmWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
855 assert(!hasIntModifiers());
856 addRegOrImmWithInputModsOperands(Inst,
N);
859 void addRegWithInputModsOperands(MCInst &Inst,
unsigned N)
const {
860 Modifiers Mods = getModifiers();
863 addRegOperands(Inst,
N);
866 void addRegWithFPInputModsOperands(MCInst &Inst,
unsigned N)
const {
867 assert(!hasIntModifiers());
868 addRegWithInputModsOperands(Inst,
N);
871 static void printImmTy(raw_ostream &OS, ImmTy
Type) {
874 case ImmTyNone: OS <<
"None";
break;
875 case ImmTyGDS: OS <<
"GDS";
break;
876 case ImmTyLDS: OS <<
"LDS";
break;
877 case ImmTyOffen: OS <<
"Offen";
break;
878 case ImmTyIdxen: OS <<
"Idxen";
break;
879 case ImmTyAddr64: OS <<
"Addr64";
break;
880 case ImmTyOffset: OS <<
"Offset";
break;
881 case ImmTyInstOffset: OS <<
"InstOffset";
break;
882 case ImmTyOffset0: OS <<
"Offset0";
break;
883 case ImmTyOffset1: OS <<
"Offset1";
break;
884 case ImmTySMEMOffsetMod: OS <<
"SMEMOffsetMod";
break;
885 case ImmTyCPol: OS <<
"CPol";
break;
886 case ImmTyIndexKey8bit: OS <<
"index_key";
break;
887 case ImmTyIndexKey16bit: OS <<
"index_key";
break;
888 case ImmTyIndexKey32bit: OS <<
"index_key";
break;
889 case ImmTyTFE: OS <<
"TFE";
break;
890 case ImmTyIsAsync: OS <<
"IsAsync";
break;
891 case ImmTyD16: OS <<
"D16";
break;
892 case ImmTyFORMAT: OS <<
"FORMAT";
break;
893 case ImmTyClamp: OS <<
"Clamp";
break;
894 case ImmTyOModSI: OS <<
"OModSI";
break;
895 case ImmTyDPP8: OS <<
"DPP8";
break;
896 case ImmTyDppCtrl: OS <<
"DppCtrl";
break;
897 case ImmTyDppRowMask: OS <<
"DppRowMask";
break;
898 case ImmTyDppBankMask: OS <<
"DppBankMask";
break;
899 case ImmTyDppBoundCtrl: OS <<
"DppBoundCtrl";
break;
900 case ImmTyDppFI: OS <<
"DppFI";
break;
901 case ImmTySDWADstSel: OS <<
"SDWADstSel";
break;
902 case ImmTySDWASrc0Sel: OS <<
"SDWASrc0Sel";
break;
903 case ImmTySDWASrc1Sel: OS <<
"SDWASrc1Sel";
break;
904 case ImmTySDWADstUnused: OS <<
"SDWADstUnused";
break;
905 case ImmTyDMask: OS <<
"DMask";
break;
906 case ImmTyDim: OS <<
"Dim";
break;
907 case ImmTyUNorm: OS <<
"UNorm";
break;
908 case ImmTyDA: OS <<
"DA";
break;
909 case ImmTyR128A16: OS <<
"R128A16";
break;
910 case ImmTyA16: OS <<
"A16";
break;
911 case ImmTyLWE: OS <<
"LWE";
break;
912 case ImmTyOff: OS <<
"Off";
break;
913 case ImmTyExpTgt: OS <<
"ExpTgt";
break;
914 case ImmTyExpCompr: OS <<
"ExpCompr";
break;
915 case ImmTyExpVM: OS <<
"ExpVM";
break;
916 case ImmTyDone: OS <<
"Done";
break;
917 case ImmTyRowEn: OS <<
"RowEn";
break;
918 case ImmTyHwreg: OS <<
"Hwreg";
break;
919 case ImmTySendMsg: OS <<
"SendMsg";
break;
920 case ImmTyWaitEvent: OS <<
"WaitEvent";
break;
921 case ImmTyInterpSlot: OS <<
"InterpSlot";
break;
922 case ImmTyInterpAttr: OS <<
"InterpAttr";
break;
923 case ImmTyInterpAttrChan: OS <<
"InterpAttrChan";
break;
924 case ImmTyOpSel: OS <<
"OpSel";
break;
925 case ImmTyOpSelHi: OS <<
"OpSelHi";
break;
926 case ImmTyNegLo: OS <<
"NegLo";
break;
927 case ImmTyNegHi: OS <<
"NegHi";
break;
928 case ImmTySwizzle: OS <<
"Swizzle";
break;
929 case ImmTyGprIdxMode: OS <<
"GprIdxMode";
break;
930 case ImmTyHigh: OS <<
"High";
break;
931 case ImmTyBLGP: OS <<
"BLGP";
break;
932 case ImmTyCBSZ: OS <<
"CBSZ";
break;
933 case ImmTyABID: OS <<
"ABID";
break;
934 case ImmTyEndpgm: OS <<
"Endpgm";
break;
935 case ImmTyWaitVDST: OS <<
"WaitVDST";
break;
936 case ImmTyWaitEXP: OS <<
"WaitEXP";
break;
937 case ImmTyWaitVAVDst: OS <<
"WaitVAVDst";
break;
938 case ImmTyWaitVMVSrc: OS <<
"WaitVMVSrc";
break;
939 case ImmTyBitOp3: OS <<
"BitOp3";
break;
940 case ImmTyMatrixAFMT: OS <<
"ImmTyMatrixAFMT";
break;
941 case ImmTyMatrixBFMT: OS <<
"ImmTyMatrixBFMT";
break;
942 case ImmTyMatrixAScale: OS <<
"ImmTyMatrixAScale";
break;
943 case ImmTyMatrixBScale: OS <<
"ImmTyMatrixBScale";
break;
944 case ImmTyMatrixAScaleFmt: OS <<
"ImmTyMatrixAScaleFmt";
break;
945 case ImmTyMatrixBScaleFmt: OS <<
"ImmTyMatrixBScaleFmt";
break;
946 case ImmTyMatrixAReuse: OS <<
"ImmTyMatrixAReuse";
break;
947 case ImmTyMatrixBReuse: OS <<
"ImmTyMatrixBReuse";
break;
948 case ImmTyScaleSel: OS <<
"ScaleSel" ;
break;
949 case ImmTyByteSel: OS <<
"ByteSel" ;
break;
954 void print(raw_ostream &OS,
const MCAsmInfo &MAI)
const override {
958 <<
" mods: " <<
Reg.Mods <<
'>';
962 if (getImmTy() != ImmTyNone) {
964 printImmTy(OS, getImmTy());
966 OS <<
" mods: " <<
Imm.Mods <<
'>';
979 static AMDGPUOperand::Ptr CreateImm(
const AMDGPUAsmParser *AsmParser,
980 int64_t Val, SMLoc Loc,
981 ImmTy
Type = ImmTyNone,
982 bool IsFPImm =
false) {
983 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
985 Op->Imm.IsFPImm = IsFPImm;
987 Op->Imm.Mods = Modifiers();
993 static AMDGPUOperand::Ptr CreateToken(
const AMDGPUAsmParser *AsmParser,
994 StringRef Str, SMLoc Loc,
995 bool HasExplicitEncodingSize =
true) {
996 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
997 Res->Tok.Data = Str.data();
998 Res->Tok.Length = Str.size();
1004 static AMDGPUOperand::Ptr CreateReg(
const AMDGPUAsmParser *AsmParser,
1005 MCRegister
Reg, SMLoc S, SMLoc
E) {
1006 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1007 Op->Reg.RegNo =
Reg;
1008 Op->Reg.Mods = Modifiers();
1014 static AMDGPUOperand::Ptr CreateExpr(
const AMDGPUAsmParser *AsmParser,
1015 const class MCExpr *Expr, SMLoc S) {
1016 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1025 OS <<
"abs:" << Mods.Abs <<
" neg: " << Mods.Neg <<
" sext:" << Mods.Sext;
1034#define GET_REGISTER_MATCHER
1035#include "AMDGPUGenAsmMatcher.inc"
1036#undef GET_REGISTER_MATCHER
1037#undef GET_SUBTARGET_FEATURE_NAME
1042class KernelScopeInfo {
1043 int SgprIndexUnusedMin = -1;
1044 int VgprIndexUnusedMin = -1;
1045 int AgprIndexUnusedMin = -1;
1049 void usesSgprAt(
int i) {
1050 if (i >= SgprIndexUnusedMin) {
1051 SgprIndexUnusedMin = ++i;
1054 Ctx->getOrCreateSymbol(
Twine(
".kernel.sgpr_count"));
1060 void usesVgprAt(
int i) {
1061 if (i >= VgprIndexUnusedMin) {
1062 VgprIndexUnusedMin = ++i;
1065 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1067 VgprIndexUnusedMin);
1073 void usesAgprAt(
int i) {
1078 if (i >= AgprIndexUnusedMin) {
1079 AgprIndexUnusedMin = ++i;
1082 Ctx->getOrCreateSymbol(
Twine(
".kernel.agpr_count"));
1087 Ctx->getOrCreateSymbol(
Twine(
".kernel.vgpr_count"));
1089 VgprIndexUnusedMin);
1096 KernelScopeInfo() =
default;
1100 MSTI = Ctx->getSubtargetInfo();
1102 usesSgprAt(SgprIndexUnusedMin = -1);
1103 usesVgprAt(VgprIndexUnusedMin = -1);
1105 usesAgprAt(AgprIndexUnusedMin = -1);
1109 void usesRegister(RegisterKind RegKind,
unsigned DwordRegIndex,
1110 unsigned RegWidth) {
1113 usesSgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1116 usesAgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1119 usesVgprAt(DwordRegIndex +
divideCeil(RegWidth, 32) - 1);
1128 MCAsmParser &Parser;
1130 unsigned ForcedEncodingSize = 0;
1131 bool ForcedDPP =
false;
1132 bool ForcedSDWA =
false;
1133 KernelScopeInfo KernelScope;
1134 const unsigned HwMode;
1136 const AMDGPU::IsaVersion ISA;
1141#define GET_ASSEMBLER_HEADER
1142#include "AMDGPUGenAsmMatcher.inc"
1147 unsigned getRegOperandSize(
const MCInstrDesc &
Desc,
unsigned OpNo)
const {
1149 int16_t RCID = MII.getOpRegClassID(
Desc.operands()[OpNo], HwMode);
1153 std::optional<AMDGPU::InfoSectionData> InfoData;
1160 bool TargetDirectiveEmitted =
false;
1169 SmallVector<unsigned> OpcodeStream;
1171 OpcodeStreamSymbols;
1172 SmallPtrSet<const MCSymbol *, 8> AMDHSAKernelSymbols;
1175 void checkKernelPrologues();
1178 void createConstantSymbol(StringRef Id, int64_t Val);
1180 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1181 bool OutOfRangeError(SMRange
Range);
1197 bool calculateGPRBlocks(
const FeatureBitset &Features,
const MCExpr *VCCUsed,
1198 const MCExpr *FlatScrUsed,
bool XNACKUsed,
1199 std::optional<bool> EnableWavefrontSize32,
1200 const MCExpr *NextFreeVGPR, SMRange VGPRRange,
1201 const MCExpr *NextFreeSGPR, SMRange SGPRRange,
1202 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks);
1203 bool ParseDirectiveAMDGCNTarget();
1204 bool ParseDirectiveAMDHSACodeObjectVersion();
1205 bool ParseDirectiveAMDHSAKernel();
1206 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1207 bool ParseDirectiveAMDKernelCodeT();
1209 bool subtargetHasRegister(
const MCRegisterInfo &MRI, MCRegister
Reg);
1210 bool ParseDirectiveAMDGPUHsaKernel();
1212 bool ParseDirectiveISAVersion();
1213 bool ParseDirectiveHSAMetadata();
1214 bool ParseDirectivePALMetadataBegin();
1215 bool ParseDirectivePALMetadata();
1216 bool ParseDirectiveAMDGPULDS();
1217 bool ParseDirectiveAMDGPUInfo();
1221 bool ParseToEndDirective(
const char *AssemblerDirectiveBegin,
1222 const char *AssemblerDirectiveEnd,
1223 std::string &CollectString);
1225 bool AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
1226 RegisterKind RegKind, MCRegister Reg1,
1227 RegisterKind RegKind1, SMLoc Loc);
1228 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1229 unsigned &RegNum,
unsigned &RegWidth,
1230 bool RestoreOnFailure =
false);
1231 bool ParseAMDGPURegister(RegisterKind &RegKind, MCRegister &
Reg,
1232 unsigned &RegNum,
unsigned &RegWidth,
1233 SmallVectorImpl<AsmToken> &Tokens);
1234 MCRegister ParseRegularReg(RegisterKind &RegKind,
unsigned &RegNum,
1236 SmallVectorImpl<AsmToken> &Tokens);
1237 MCRegister ParseSpecialReg(RegisterKind &RegKind,
unsigned &RegNum,
1239 SmallVectorImpl<AsmToken> &Tokens);
1240 MCRegister ParseRegList(RegisterKind &RegKind,
unsigned &RegNum,
1242 SmallVectorImpl<AsmToken> &Tokens);
1243 bool ParseRegRange(
unsigned &Num,
unsigned &Width,
unsigned &SubReg);
1244 MCRegister getRegularReg(RegisterKind RegKind,
unsigned RegNum,
1245 unsigned SubReg,
unsigned RegWidth, SMLoc Loc);
1248 bool isRegister(
const AsmToken &Token,
const AsmToken &NextToken)
const;
1249 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1250 void initializeGprCountSymbol(RegisterKind RegKind);
1251 bool updateGprCountSymbols(RegisterKind RegKind,
unsigned DwordRegIndex,
1257 OperandMode_Default,
1261 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1263 AMDGPUAsmParser(
const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1264 const MCInstrInfo &MII)
1265 : MCTargetAsmParser(STI, MII), Parser(_Parser),
1266 HwMode(STI.getHwMode(MCSubtargetInfo::HwMode_RegInfo)),
1271 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1273 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1274 createConstantSymbol(
".amdgcn.gfx_generation_number", ISA.Major);
1275 createConstantSymbol(
".amdgcn.gfx_generation_minor", ISA.Minor);
1276 createConstantSymbol(
".amdgcn.gfx_generation_stepping", ISA.Stepping);
1278 createConstantSymbol(
".option.machine_version_major", ISA.Major);
1279 createConstantSymbol(
".option.machine_version_minor", ISA.Minor);
1280 createConstantSymbol(
".option.machine_version_stepping", ISA.Stepping);
1282 if (ISA.Major >= 6 &&
isHsaAbi(getSTI())) {
1283 initializeGprCountSymbol(IS_VGPR);
1284 initializeGprCountSymbol(IS_SGPR);
1289 createConstantSymbol(Symbol, Code);
1291 createConstantSymbol(
"UC_VERSION_W64_BIT", 0x2000);
1292 createConstantSymbol(
"UC_VERSION_W32_BIT", 0x4000);
1293 createConstantSymbol(
"UC_VERSION_MDP_BIT", 0x8000);
1333 bool hasBVHRayTracingInsts()
const {
1334 return getFeatureBits()[AMDGPU::FeatureBVHRayTracingInsts];
1337 bool isWave32()
const {
return getAvailableFeatures()[Feature_isWave32Bit]; }
1339 bool isWave64()
const {
return getAvailableFeatures()[Feature_isWave64Bit]; }
1341 bool hasInv2PiInlineImm()
const {
1342 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1345 bool has64BitLiterals()
const {
1346 return getFeatureBits()[AMDGPU::Feature64BitLiterals];
1349 bool hasFlatOffsets()
const {
1350 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1353 bool hasTrue16Insts()
const {
1354 return getFeatureBits()[AMDGPU::FeatureTrue16BitInsts];
1358 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1361 bool hasSGPR102_SGPR103()
const {
return !
isVI() && !
isGFX9(); }
1363 bool hasSGPR104_SGPR105()
const {
return isGFX10Plus(); }
1365 bool hasIntClamp()
const {
return getFeatureBits()[AMDGPU::FeatureIntClamp]; }
1367 bool hasPartialNSAEncoding()
const {
1368 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1371 bool hasGloballyAddressableScratch()
const {
1372 return getFeatureBits()[AMDGPU::FeatureGloballyAddressableScratch];
1385 AMDGPUTargetStreamer &getTargetStreamer() {
1386 MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1387 return static_cast<AMDGPUTargetStreamer &
>(TS);
1393 return const_cast<AMDGPUAsmParser *
>(
this)->MCTargetAsmParser::getContext();
1396 const MCRegisterInfo *getMRI()
const {
1400 const MCInstrInfo *getMII()
const {
return &MII; }
1404 int16_t getTargetRegClass(
unsigned TargetRCIdx)
const {
1410 const FeatureBitset &getFeatureBits()
const {
1411 return getSTI().getFeatureBits();
1414 void setForcedEncodingSize(
unsigned Size) { ForcedEncodingSize =
Size; }
1415 void setForcedDPP(
bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1416 void setForcedSDWA(
bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1418 unsigned getForcedEncodingSize()
const {
return ForcedEncodingSize; }
1419 bool isForcedVOP3()
const {
return ForcedEncodingSize == 64; }
1420 bool isForcedDPP()
const {
return ForcedDPP; }
1421 bool isForcedSDWA()
const {
return ForcedSDWA; }
1422 ArrayRef<unsigned> getMatchedVariants()
const;
1423 StringRef getMatchedVariantName()
const;
1425 std::unique_ptr<AMDGPUOperand> parseRegister(
bool RestoreOnFailure =
false);
1426 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1427 bool RestoreOnFailure);
1428 bool parseRegister(MCRegister &
Reg, SMLoc &StartLoc, SMLoc &EndLoc)
override;
1429 ParseStatus tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
1430 SMLoc &EndLoc)
override;
1431 unsigned checkTargetMatchPredicate(MCInst &Inst)
override;
1432 unsigned validateTargetOperandClass(MCParsedAsmOperand &
Op,
1433 unsigned Kind)
override;
1434 bool matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
1437 bool MatchingInlineAsm)
override;
1438 bool ParseDirective(AsmToken DirectiveID)
override;
1439 void doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc)
override;
1440 void onEndOfFile()
override;
1442 OperandMode
Mode = OperandMode_Default);
1443 StringRef parseMnemonicSuffix(StringRef Name);
1444 bool parseInstruction(ParseInstructionInfo &Info, StringRef Name,
1450 ParseStatus parseIntWithPrefix(
const char *Prefix, int64_t &
Int);
1454 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1455 std::function<
bool(int64_t &)> ConvertResult =
nullptr);
1457 ParseStatus parseOperandArrayWithPrefix(
1459 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1460 bool (*ConvertResult)(int64_t &) =
nullptr);
1464 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1465 bool IgnoreNegative =
false);
1466 unsigned getCPolKind(StringRef Id, StringRef Mnemo,
bool &Disabling)
const;
1470 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &
Value,
1474 ArrayRef<const char *> Ids,
1478 ArrayRef<const char *> Ids,
1479 AMDGPUOperand::ImmTy
Type);
1482 bool isOperandModifier(
const AsmToken &Token,
1483 const AsmToken &NextToken)
const;
1484 bool isRegOrOperandModifier(
const AsmToken &Token,
1485 const AsmToken &NextToken)
const;
1486 bool isNamedOperandModifier(
const AsmToken &Token,
1487 const AsmToken &NextToken)
const;
1488 bool isOpcodeModifierWithVal(
const AsmToken &Token,
1489 const AsmToken &NextToken)
const;
1490 bool parseSP3NegModifier();
1497 bool AllowImm =
true);
1499 bool AllowImm =
true);
1505 AMDGPUOperand::ImmTy ImmTy);
1510 AMDGPUOperand::ImmTy
Type);
1514 AMDGPUOperand::ImmTy
Type);
1518 AMDGPUOperand::ImmTy
Type);
1522 ParseStatus parseDfmtNfmt(int64_t &
Format);
1523 ParseStatus parseUfmt(int64_t &
Format);
1524 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1526 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1529 ParseStatus parseSymbolicOrNumericFormat(int64_t &
Format);
1530 ParseStatus parseNumericFormat(int64_t &
Format);
1534 bool tryParseFmt(
const char *Pref, int64_t MaxVal, int64_t &Val);
1535 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr,
1540 bool parseCnt(int64_t &IntVal);
1543 bool parseDepCtr(int64_t &IntVal,
unsigned &Mask);
1544 void depCtrError(SMLoc Loc,
int ErrorId, StringRef DepCtrName);
1547 bool parseDelay(int64_t &Delay);
1553 struct OperandInfoTy {
1556 bool IsSymbolic =
false;
1557 bool IsDefined =
false;
1559 constexpr OperandInfoTy(int64_t Val) : Val(Val) {}
1562 struct StructuredOpField : OperandInfoTy {
1566 bool IsDefined =
false;
1568 constexpr StructuredOpField(StringLiteral Id, StringLiteral Desc,
1569 unsigned Width, int64_t
Default)
1570 : OperandInfoTy(
Default), Id(Id), Desc(Desc), Width(Width) {}
1571 virtual ~StructuredOpField() =
default;
1573 bool Error(AMDGPUAsmParser &Parser,
const Twine &Err)
const {
1574 Parser.Error(Loc,
"invalid " + Desc +
": " + Err);
1578 virtual bool validate(AMDGPUAsmParser &Parser)
const {
1580 return Error(Parser,
"not supported on this GPU");
1582 return Error(Parser,
"only " + Twine(Width) +
"-bit values are legal");
1590 bool parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
1591 OperandInfoTy &Stream);
1592 bool validateSendMsg(
const OperandInfoTy &
Msg,
const OperandInfoTy &
Op,
1593 const OperandInfoTy &Stream);
1595 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &
Offset,
1596 OperandInfoTy &Width);
1601 static SMLoc getLaterLoc(SMLoc a, SMLoc b);
1608 SMLoc getOperandLoc(std::function<
bool(
const AMDGPUOperand &)>
Test,
1610 SMLoc getImmLoc(AMDGPUOperand::ImmTy
Type,
1614 bool validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
1619 bool validateBF16InlineConst(
const MCInst &Inst,
1622 bool validateConstantBusLimitations(
const MCInst &Inst,
1624 std::optional<unsigned> checkVOPDRegBankConstraints(
const MCInst &Inst,
1627 bool tryVOPD(
const MCInst &Inst);
1628 bool tryVOPD3(
const MCInst &Inst);
1629 bool tryAnotherVOPDEncoding(
const MCInst &Inst);
1631 bool validateIntClampSupported(
const MCInst &Inst);
1632 bool validateMIMGAtomicDMask(
const MCInst &Inst);
1633 bool validateMIMGGatherDMask(
const MCInst &Inst);
1635 bool validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc);
1636 bool validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc);
1637 bool validateMIMGD16(
const MCInst &Inst);
1639 bool validateTensorR128(
const MCInst &Inst);
1640 bool validateMIMGMSAA(
const MCInst &Inst);
1641 bool validateOpSel(
const MCInst &Inst);
1642 bool validateTrue16OpSel(
const MCInst &Inst);
1643 bool validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName);
1645 bool validateVccOperand(MCRegister
Reg)
const;
1650 bool validateAGPRLdSt(
const MCInst &Inst)
const;
1651 bool validateVGPRAlign(
const MCInst &Inst)
const;
1655 bool validateDivScale(
const MCInst &Inst);
1660 const unsigned CPol);
1665 bool validateClusterBarrierIsFirst(
const MCInst &Inst,
1668 unsigned getConstantBusLimit(
unsigned Opcode)
const;
1669 bool usesConstantBus(
const MCInst &Inst,
unsigned OpIdx);
1670 bool isInlineConstant(
const MCInst &Inst,
unsigned OpIdx)
const;
1671 MCRegister findImplicitSGPRReadInVOP(
const MCInst &Inst)
const;
1673 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS);
1674 bool isSupportedMnemo(StringRef Mnemo,
const FeatureBitset &FBS,
1675 ArrayRef<unsigned> Variants);
1676 bool checkUnsupportedInstruction(StringRef Name, SMLoc IDLoc);
1678 bool isId(
const StringRef Id)
const;
1679 bool isId(
const AsmToken &Token,
const StringRef Id)
const;
1681 StringRef getId()
const;
1682 bool trySkipId(
const StringRef Id);
1683 bool trySkipId(
const StringRef Pref,
const StringRef Id);
1687 bool parseString(StringRef &Val,
1688 const StringRef ErrMsg =
"expected a string");
1689 bool parseId(StringRef &Val,
const StringRef ErrMsg =
"");
1695 StringRef getTokenStr()
const;
1696 AsmToken peekToken(
bool ShouldSkipSpace =
true);
1698 SMLoc getLoc()
const;
1702 void onBeginOfFile()
override;
1706 void emitTargetDirective();
1707 bool parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc)
override;
1719 bool parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
1720 const unsigned MaxVal,
const Twine &ErrMsg,
1722 bool parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
1723 const unsigned MinVal,
const unsigned MaxVal,
1724 const StringRef ErrMsg);
1726 bool parseSwizzleOffset(int64_t &
Imm);
1727 bool parseSwizzleMacro(int64_t &
Imm);
1728 bool parseSwizzleQuadPerm(int64_t &
Imm);
1729 bool parseSwizzleBitmaskPerm(int64_t &
Imm);
1730 bool parseSwizzleBroadcast(int64_t &
Imm);
1731 bool parseSwizzleSwap(int64_t &
Imm);
1732 bool parseSwizzleReverse(int64_t &
Imm);
1733 bool parseSwizzleFFT(int64_t &
Imm);
1734 bool parseSwizzleRotate(int64_t &
Imm);
1737 int64_t parseGPRIdxMacro();
1740 cvtMubufImpl(Inst,
Operands,
false);
1743 cvtMubufImpl(Inst,
Operands,
true);
1749 OptionalImmIndexMap &OptionalIdx);
1758 OptionalImmIndexMap &OptionalIdx);
1760 OptionalImmIndexMap &OptionalIdx);
1764 void cvtOpSelHelper(MCInst &Inst,
unsigned OpSel);
1766 bool parseDimId(
unsigned &Encoding);
1768 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1772 int64_t parseDPPCtrlSel(StringRef Ctrl);
1773 int64_t parseDPPCtrlPerm();
1779 bool IsDPP8 =
false);
1785 AMDGPUOperand::ImmTy
Type);
1793 enum class SDWAInstType :
unsigned {
VOP1 = 0,
VOP2 = 1,
VOPC = 2 };
1796 SDWAInstType BasicInstType,
bool SkipDstVcc =
false,
1797 bool SkipSrcVcc =
false);
1907bool AMDGPUOperand::isInlinableImm(
MVT type)
const {
1917 if (!isImmTy(ImmTyNone)) {
1922 if (getModifiers().
Lit != LitModifier::None)
1932 if (type == MVT::f64 || type == MVT::i64) {
1934 AsmParser->hasInv2PiInlineImm());
1937 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
1956 APFloat::rmNearestTiesToEven, &Lost);
1963 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1965 AsmParser->hasInv2PiInlineImm());
1970 static_cast<int32_t
>(FPLiteral.bitcastToAPInt().getZExtValue()),
1971 AsmParser->hasInv2PiInlineImm());
1975 if (type == MVT::f64 || type == MVT::i64) {
1977 AsmParser->hasInv2PiInlineImm());
1986 static_cast<int16_t
>(
Literal.getLoBits(16).getSExtValue()), type,
1987 AsmParser->hasInv2PiInlineImm());
1991 static_cast<int32_t
>(
Literal.getLoBits(32).getZExtValue()),
1992 AsmParser->hasInv2PiInlineImm());
1995bool AMDGPUOperand::isLiteralImm(MVT type)
const {
1997 if (!isImmTy(ImmTyNone)) {
2002 (type == MVT::i64 || type == MVT::f64) && AsmParser->has64BitLiterals();
2007 if (type == MVT::f64 && hasFPModifiers()) {
2027 if (type == MVT::f64) {
2032 if (type == MVT::i64) {
2045 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2046 : (type == MVT::v2i16) ? MVT::f32
2047 : (type == MVT::v2f32) ? MVT::f32
2050 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64,
Imm.Val));
2054bool AMDGPUOperand::isRegClassTarget(
unsigned TargetRCIdx)
const {
2057 int16_t RCID = AsmParser->getTargetRegClass(TargetRCIdx);
2058 return RCID >= 0 && isRegClass(RCID);
2061bool AMDGPUOperand::isRegClass(
unsigned RCID)
const {
2062 return isRegKind() &&
2063 AsmParser->getMRI()->getRegClass(RCID).contains(
getReg());
2066bool AMDGPUOperand::isVRegWithInputMods()
const {
2067 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2069 (AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP] &&
2070 isRegClassTarget(AMDGPU::VReg_64_AlignTarget));
2073template <
bool IsFake16>
2074bool AMDGPUOperand::isT16_Lo128VRegWithInputMods()
const {
2075 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2076 : AMDGPU::VGPR_16_Lo128RegClassID);
2079template <
bool IsFake16>
bool AMDGPUOperand::isT16VRegWithInputMods()
const {
2080 return isRegClass(IsFake16 ? AMDGPU::VGPR_32RegClassID
2081 : AMDGPU::VGPR_16RegClassID);
2084bool AMDGPUOperand::isT16_LO16VRegWithInputMods()
const {
2085 return isRegClass(AMDGPU::VGPR_16_LO16RegClassID);
2088bool AMDGPUOperand::isSDWAOperand(MVT type)
const {
2089 if (AsmParser->isVI())
2091 if (AsmParser->isGFX9Plus())
2092 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2096bool AMDGPUOperand::isSDWAFP16Operand()
const {
2097 return isSDWAOperand(MVT::f16);
2100bool AMDGPUOperand::isSDWAFP32Operand()
const {
2101 return isSDWAOperand(MVT::f32);
2104bool AMDGPUOperand::isSDWAInt16Operand()
const {
2105 return isSDWAOperand(MVT::i16);
2108bool AMDGPUOperand::isSDWAInt32Operand()
const {
2109 return isSDWAOperand(MVT::i32);
2112bool AMDGPUOperand::isBoolReg()
const {
2113 return isReg() && ((AsmParser->isWave64() && isSCSrc_b64()) ||
2114 (AsmParser->isWave32() && isSCSrc_b32()));
2118 unsigned Size)
const {
2119 assert(isImmTy(ImmTyNone) &&
Imm.Mods.hasFPModifiers());
2134void AMDGPUOperand::addImmOperands(MCInst &Inst,
unsigned N,
2135 bool ApplyModifiers)
const {
2145 addLiteralImmOperand(Inst,
Imm.Val,
2146 ApplyModifiers & isImmTy(ImmTyNone) &&
2147 Imm.Mods.hasFPModifiers());
2149 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2154void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val,
2155 bool ApplyModifiers)
const {
2156 const auto &InstDesc = AsmParser->getMII()->get(Inst.
getOpcode());
2161 if (ApplyModifiers) {
2163 const unsigned Size =
2165 Val = applyInputFPModifiers(Val,
Size);
2169 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2171 bool CanUse64BitLiterals =
2174 MCContext &Ctx = AsmParser->getContext();
2185 if (
Lit == LitModifier::None &&
2187 AsmParser->hasInv2PiInlineImm())) {
2195 bool HasMandatoryLiteral =
2198 if (
Literal.getLoBits(32) != 0 &&
2199 (InstDesc.getSize() != 4 || !AsmParser->has64BitLiterals()) &&
2200 !HasMandatoryLiteral) {
2201 const_cast<AMDGPUAsmParser *
>(AsmParser)->
Warning(
2203 "Can't encode literal as exact 64-bit floating-point operand. "
2204 "Low 32-bits will be set to zero");
2205 Val &= 0xffffffff00000000u;
2211 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2217 Lit = LitModifier::Lit64;
2218 }
else if (
Lit == LitModifier::Lit) {
2232 if (CanUse64BitLiterals &&
Lit == LitModifier::None &&
2234 Lit = LitModifier::Lit64;
2241 if (
Lit == LitModifier::None && AsmParser->hasInv2PiInlineImm() &&
2242 Literal == 0x3fc45f306725feed) {
2282 Val = FPLiteral.bitcastToAPInt().getZExtValue();
2289 if (
Lit != LitModifier::None) {
2320 if (
Lit == LitModifier::None &&
2330 if (!AsmParser->has64BitLiterals() ||
Lit == LitModifier::Lit)
2338 if (
Lit == LitModifier::None &&
2346 if (!AsmParser->has64BitLiterals()) {
2347 Val =
static_cast<uint64_t>(Val) << 32;
2354 if (
Lit == LitModifier::Lit ||
2356 Val =
static_cast<uint64_t>(Val) << 32;
2360 if (
Lit == LitModifier::Lit)
2387 if (
Lit != LitModifier::None) {
2395void AMDGPUOperand::addRegOperands(MCInst &Inst,
unsigned N)
const {
2401bool AMDGPUOperand::isInlineValue()
const {
2409void AMDGPUAsmParser::createConstantSymbol(StringRef Id, int64_t Val) {
2420 if (Is == IS_VGPR) {
2425 return AMDGPU::VGPR_32RegClassID;
2427 return AMDGPU::VReg_64RegClassID;
2429 return AMDGPU::VReg_96RegClassID;
2431 return AMDGPU::VReg_128RegClassID;
2433 return AMDGPU::VReg_160RegClassID;
2435 return AMDGPU::VReg_192RegClassID;
2437 return AMDGPU::VReg_224RegClassID;
2439 return AMDGPU::VReg_256RegClassID;
2441 return AMDGPU::VReg_288RegClassID;
2443 return AMDGPU::VReg_320RegClassID;
2445 return AMDGPU::VReg_352RegClassID;
2447 return AMDGPU::VReg_384RegClassID;
2449 return AMDGPU::VReg_512RegClassID;
2451 return AMDGPU::VReg_1024RegClassID;
2453 }
else if (Is == IS_TTMP) {
2458 return AMDGPU::TTMP_32RegClassID;
2460 return AMDGPU::TTMP_64RegClassID;
2462 return AMDGPU::TTMP_128RegClassID;
2464 return AMDGPU::TTMP_256RegClassID;
2466 return AMDGPU::TTMP_512RegClassID;
2468 }
else if (Is == IS_SGPR) {
2473 return AMDGPU::SGPR_32RegClassID;
2475 return AMDGPU::SGPR_64RegClassID;
2477 return AMDGPU::SGPR_96RegClassID;
2479 return AMDGPU::SGPR_128RegClassID;
2481 return AMDGPU::SGPR_160RegClassID;
2483 return AMDGPU::SGPR_192RegClassID;
2485 return AMDGPU::SGPR_224RegClassID;
2487 return AMDGPU::SGPR_256RegClassID;
2489 return AMDGPU::SGPR_288RegClassID;
2491 return AMDGPU::SGPR_320RegClassID;
2493 return AMDGPU::SGPR_352RegClassID;
2495 return AMDGPU::SGPR_384RegClassID;
2497 return AMDGPU::SGPR_512RegClassID;
2499 }
else if (Is == IS_AGPR) {
2504 return AMDGPU::AGPR_32RegClassID;
2506 return AMDGPU::AReg_64RegClassID;
2508 return AMDGPU::AReg_96RegClassID;
2510 return AMDGPU::AReg_128RegClassID;
2512 return AMDGPU::AReg_160RegClassID;
2514 return AMDGPU::AReg_192RegClassID;
2516 return AMDGPU::AReg_224RegClassID;
2518 return AMDGPU::AReg_256RegClassID;
2520 return AMDGPU::AReg_288RegClassID;
2522 return AMDGPU::AReg_320RegClassID;
2524 return AMDGPU::AReg_352RegClassID;
2526 return AMDGPU::AReg_384RegClassID;
2528 return AMDGPU::AReg_512RegClassID;
2530 return AMDGPU::AReg_1024RegClassID;
2538 .
Case(
"exec", AMDGPU::EXEC)
2539 .
Case(
"vcc", AMDGPU::VCC)
2540 .
Case(
"flat_scratch", AMDGPU::FLAT_SCR)
2541 .
Case(
"xnack_mask", AMDGPU::XNACK_MASK)
2542 .
Case(
"shared_base", AMDGPU::SRC_SHARED_BASE)
2543 .
Case(
"src_shared_base", AMDGPU::SRC_SHARED_BASE)
2544 .
Case(
"shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2545 .
Case(
"src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2546 .
Case(
"private_base", AMDGPU::SRC_PRIVATE_BASE)
2547 .
Case(
"src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2548 .
Case(
"private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2549 .
Case(
"src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2550 .
Case(
"src_flat_scratch_base_lo", AMDGPU::SRC_FLAT_SCRATCH_BASE_LO)
2551 .
Case(
"src_flat_scratch_base_hi", AMDGPU::SRC_FLAT_SCRATCH_BASE_HI)
2552 .
Case(
"pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2553 .
Case(
"src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2554 .
Case(
"lds_direct", AMDGPU::LDS_DIRECT)
2555 .
Case(
"src_lds_direct", AMDGPU::LDS_DIRECT)
2556 .
Case(
"m0", AMDGPU::M0)
2557 .
Case(
"vccz", AMDGPU::SRC_VCCZ)
2558 .
Case(
"src_vccz", AMDGPU::SRC_VCCZ)
2559 .
Case(
"execz", AMDGPU::SRC_EXECZ)
2560 .
Case(
"src_execz", AMDGPU::SRC_EXECZ)
2561 .
Case(
"scc", AMDGPU::SRC_SCC)
2562 .
Case(
"src_scc", AMDGPU::SRC_SCC)
2563 .
Case(
"tba", AMDGPU::TBA)
2564 .
Case(
"tma", AMDGPU::TMA)
2565 .
Case(
"flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2566 .
Case(
"flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2567 .
Case(
"xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2568 .
Case(
"xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2569 .
Case(
"vcc_lo", AMDGPU::VCC_LO)
2570 .
Case(
"vcc_hi", AMDGPU::VCC_HI)
2571 .
Case(
"exec_lo", AMDGPU::EXEC_LO)
2572 .
Case(
"exec_hi", AMDGPU::EXEC_HI)
2573 .
Case(
"tma_lo", AMDGPU::TMA_LO)
2574 .
Case(
"tma_hi", AMDGPU::TMA_HI)
2575 .
Case(
"tba_lo", AMDGPU::TBA_LO)
2576 .
Case(
"tba_hi", AMDGPU::TBA_HI)
2577 .
Case(
"pc", AMDGPU::PC_REG)
2578 .
Case(
"null", AMDGPU::SGPR_NULL)
2582bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2583 SMLoc &EndLoc,
bool RestoreOnFailure) {
2584 auto R = parseRegister();
2588 RegNo =
R->getReg();
2589 StartLoc =
R->getStartLoc();
2590 EndLoc =
R->getEndLoc();
2594bool AMDGPUAsmParser::parseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2596 return ParseRegister(
Reg, StartLoc, EndLoc,
false);
2599ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &
Reg, SMLoc &StartLoc,
2601 bool Result = ParseRegister(
Reg, StartLoc, EndLoc,
true);
2602 bool PendingErrors = getParser().hasPendingError();
2603 getParser().clearPendingErrors();
2611bool AMDGPUAsmParser::AddNextRegisterToList(MCRegister &
Reg,
unsigned &RegWidth,
2612 RegisterKind RegKind,
2614 RegisterKind RegKind1, SMLoc Loc) {
2616 if (RegKind == IS_SGPR) {
2617 unsigned RegIdx = (
Reg - AMDGPU::SGPR0) + RegWidth / 32;
2618 if ((RegIdx == 106 && Reg1 == AMDGPU::VCC_LO) ||
2619 (RegIdx == 107 && Reg1 == AMDGPU::VCC_HI)) {
2625 if (RegKind != RegKind1) {
2626 Error(Loc,
"registers in a list must be of the same kind");
2632 if (
Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2637 if (
Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2638 Reg = AMDGPU::FLAT_SCR;
2642 if (
Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2643 Reg = AMDGPU::XNACK_MASK;
2647 if (
Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2652 if (
Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2657 if (
Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2662 Error(Loc,
"register does not fit in the list");
2668 if (Reg1 !=
Reg + RegWidth / 32) {
2669 Error(Loc,
"registers in a list must have consecutive indices");
2685 {{
"v"}, IS_VGPR}, {{
"s"}, IS_SGPR}, {{
"ttmp"}, IS_TTMP},
2686 {{
"acc"}, IS_AGPR}, {{
"a"}, IS_AGPR},
2690 return Kind == IS_VGPR || Kind == IS_SGPR || Kind == IS_TTMP ||
2696 if (Str.starts_with(
Reg.Name))
2702 return !Str.getAsInteger(10, Num);
2705bool AMDGPUAsmParser::isRegister(
const AsmToken &Token,
2706 const AsmToken &NextToken)
const {
2721 StringRef RegSuffix = Str.substr(
RegName.size());
2722 if (!RegSuffix.
empty()) {
2739bool AMDGPUAsmParser::isRegister() {
2740 return isRegister(
getToken(), peekToken());
2743MCRegister AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
unsigned RegNum,
2744 unsigned SubReg,
unsigned RegWidth,
2748 unsigned AlignSize = 1;
2749 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2755 if (RegNum % AlignSize != 0) {
2756 Error(Loc,
"invalid register alignment");
2757 return MCRegister();
2760 unsigned RegIdx = RegNum / AlignSize;
2763 Error(Loc,
"invalid or unsupported register size");
2764 return MCRegister();
2768 const MCRegisterClass &RC =
TRI->getRegClass(RCID);
2769 if (RegIdx >= RC.
getNumRegs() || (RegKind == IS_VGPR && RegIdx > 255)) {
2770 Error(Loc,
"register index is out of range");
2771 return AMDGPU::NoRegister;
2774 if (RegKind == IS_VGPR && !
isGFX1250Plus() && RegIdx + RegWidth / 32 > 256) {
2775 Error(Loc,
"register index is out of range");
2776 return MCRegister();
2785 Error(Loc,
"invalid subregister");
2791bool AMDGPUAsmParser::ParseRegRange(
unsigned &Num,
unsigned &RegWidth,
2793 int64_t RegLo, RegHi;
2797 SMLoc FirstIdxLoc = getLoc();
2804 SecondIdxLoc = getLoc();
2815 Error(FirstIdxLoc,
"invalid register index");
2820 Error(SecondIdxLoc,
"invalid register index");
2824 if (RegLo > RegHi) {
2825 Error(FirstIdxLoc,
"first register index should not exceed second index");
2829 if (RegHi == RegLo) {
2830 StringRef RegSuffix = getTokenStr();
2831 if (RegSuffix ==
".l") {
2832 SubReg = AMDGPU::lo16;
2834 }
else if (RegSuffix ==
".h") {
2835 SubReg = AMDGPU::hi16;
2840 Num =
static_cast<unsigned>(RegLo);
2841 RegWidth = 32 * ((RegHi - RegLo) + 1);
2846MCRegister AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
2849 SmallVectorImpl<AsmToken> &Tokens) {
2855 RegKind = IS_SPECIAL;
2862MCRegister AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
2865 SmallVectorImpl<AsmToken> &Tokens) {
2867 StringRef
RegName = getTokenStr();
2868 auto Loc = getLoc();
2872 Error(Loc,
"invalid register name");
2873 return MCRegister();
2881 unsigned SubReg = NoSubRegister;
2882 bool IsRange =
false;
2883 if (!RegSuffix.
empty()) {
2885 SubReg = AMDGPU::lo16;
2887 SubReg = AMDGPU::hi16;
2891 Error(Loc,
"invalid register index");
2892 return MCRegister();
2898 if (!ParseRegRange(RegNum, RegWidth, SubReg))
2899 return MCRegister();
2903 MCRegister
Reg = getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
2904 const MCRegisterInfo &
TRI = *
getContext().getRegisterInfo();
2905 if (RegKind == IS_SGPR && IsRange
2906 ? (
TRI.isSubRegister(
Reg, VCC_LO) ||
TRI.isSubRegister(
Reg, VCC_HI))
2907 : (
Reg == VCC_LO ||
Reg == VCC_HI)) {
2908 Error(Loc,
"register index is out of range");
2909 return MCRegister();
2915MCRegister AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
2916 unsigned &RegNum,
unsigned &RegWidth,
2917 SmallVectorImpl<AsmToken> &Tokens) {
2919 auto ListLoc = getLoc();
2922 "expected a register or a list of registers")) {
2923 return MCRegister();
2928 auto Loc = getLoc();
2929 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth))
2930 return MCRegister();
2931 if (RegWidth != 32) {
2932 Error(Loc,
"expected a single 32-bit register");
2933 return MCRegister();
2937 RegisterKind NextRegKind;
2939 unsigned NextRegNum, NextRegWidth;
2942 if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth,
2944 return MCRegister();
2946 if (NextRegWidth != 32) {
2947 Error(Loc,
"expected a single 32-bit register");
2948 return MCRegister();
2950 if (!AddNextRegisterToList(
Reg, RegWidth, RegKind, NextReg, NextRegKind,
2952 return MCRegister();
2956 "expected a comma or a closing square bracket")) {
2957 return MCRegister();
2961 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
2966bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
2967 MCRegister &
Reg,
unsigned &RegNum,
2969 SmallVectorImpl<AsmToken> &Tokens) {
2970 auto Loc = getLoc();
2974 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
2976 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
2978 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
2983 assert(Parser.hasPendingError());
2987 if (!subtargetHasRegister(*
TRI,
Reg)) {
2988 if (
Reg == AMDGPU::SGPR_NULL) {
2989 Error(Loc,
"'null' operand is not supported on this GPU");
2992 " register not available on this GPU");
3000bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
3001 MCRegister &
Reg,
unsigned &RegNum,
3003 bool RestoreOnFailure ) {
3007 if (ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth, Tokens)) {
3008 if (RestoreOnFailure) {
3009 while (!Tokens.
empty()) {
3018std::optional<StringRef>
3019AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3022 return StringRef(
".amdgcn.next_free_vgpr");
3024 return StringRef(
".amdgcn.next_free_sgpr");
3026 return std::nullopt;
3030void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3031 auto SymbolName = getGprCountSymbolName(RegKind);
3032 assert(SymbolName &&
"initializing invalid register kind");
3038bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3039 unsigned DwordRegIndex,
3040 unsigned RegWidth) {
3045 auto SymbolName = getGprCountSymbolName(RegKind);
3050 int64_t NewMax = DwordRegIndex +
divideCeil(RegWidth, 32) - 1;
3054 return !
Error(getLoc(),
3055 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3059 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3061 if (OldCount <= NewMax)
3067std::unique_ptr<AMDGPUOperand>
3068AMDGPUAsmParser::parseRegister(
bool RestoreOnFailure) {
3070 SMLoc StartLoc = Tok.getLoc();
3071 SMLoc EndLoc = Tok.getEndLoc();
3072 RegisterKind RegKind;
3074 unsigned RegNum, RegWidth;
3076 if (!ParseAMDGPURegister(RegKind,
Reg, RegNum, RegWidth)) {
3080 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3083 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3084 return AMDGPUOperand::CreateReg(
this,
Reg, StartLoc, EndLoc);
3091 if (isRegister() || isModifier())
3094 if (
Lit == LitModifier::None) {
3095 if (trySkipId(
"lit"))
3096 Lit = LitModifier::Lit;
3097 else if (trySkipId(
"lit64"))
3098 Lit = LitModifier::Lit64;
3100 if (
Lit != LitModifier::None) {
3103 ParseStatus S = parseImm(
Operands, HasSP3AbsModifier,
Lit);
3112 const auto &NextTok = peekToken();
3115 bool Negate =
false;
3123 AMDGPUOperand::Modifiers Mods;
3131 StringRef Num = getTokenStr();
3134 APFloat RealVal(APFloat::IEEEdouble());
3135 auto roundMode = APFloat::rmNearestTiesToEven;
3136 if (
errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3139 RealVal.changeSign();
3142 AMDGPUOperand::CreateImm(
this, RealVal.bitcastToAPInt().getZExtValue(),
3143 S, AMDGPUOperand::ImmTyNone,
true));
3144 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3145 Op.setModifiers(Mods);
3154 if (HasSP3AbsModifier) {
3163 if (getParser().parsePrimaryExpr(Expr, EndLoc,
nullptr))
3166 if (Parser.parseExpression(Expr))
3170 if (Expr->evaluateAsAbsolute(IntVal)) {
3172 return Error(S,
"literal value out of range");
3173 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
3174 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3175 Op.setModifiers(Mods);
3177 if (
Lit != LitModifier::None)
3179 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
3192 if (
auto R = parseRegister()) {
3202 ParseStatus Res = parseReg(
Operands);
3210bool AMDGPUAsmParser::isNamedOperandModifier(
const AsmToken &Token,
3211 const AsmToken &NextToken)
const {
3214 return str ==
"abs" || str ==
"neg" || str ==
"sext";
3219bool AMDGPUAsmParser::isOpcodeModifierWithVal(
const AsmToken &Token,
3220 const AsmToken &NextToken)
const {
3224bool AMDGPUAsmParser::isOperandModifier(
const AsmToken &Token,
3225 const AsmToken &NextToken)
const {
3226 return isNamedOperandModifier(Token, NextToken) || Token.
is(
AsmToken::Pipe);
3229bool AMDGPUAsmParser::isRegOrOperandModifier(
const AsmToken &Token,
3230 const AsmToken &NextToken)
const {
3231 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3248bool AMDGPUAsmParser::isModifier() {
3251 AsmToken NextToken[2];
3252 peekTokens(NextToken);
3256 bool IsOpcodeModifier = isOpcodeModifierWithVal(Tok, NextToken[0]) &&
3259 return isOperandModifier(Tok, NextToken[0]) ||
3261 isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3287bool AMDGPUAsmParser::parseSP3NegModifier() {
3289 AsmToken NextToken[2];
3290 peekTokens(NextToken);
3293 (isRegister(NextToken[0], NextToken[1]) ||
3311 return Error(getLoc(),
"invalid syntax, expected 'neg' modifier");
3313 SP3Neg = parseSP3NegModifier();
3316 Neg = trySkipId(
"neg");
3318 return Error(Loc,
"expected register or immediate");
3322 Abs = trySkipId(
"abs");
3327 if (trySkipId(
"lit")) {
3328 Lit = LitModifier::Lit;
3331 }
else if (trySkipId(
"lit64")) {
3332 Lit = LitModifier::Lit64;
3335 if (!has64BitLiterals())
3336 return Error(Loc,
"lit64 is not supported on this GPU");
3342 return Error(Loc,
"expected register or immediate");
3351 return (SP3Neg || Neg || SP3Abs || Abs ||
Lit != LitModifier::None)
3355 if (
Lit != LitModifier::None && !
Operands.back()->isImm())
3356 Error(Loc,
"expected immediate with lit modifier");
3358 if (SP3Abs && !skipToken(
AsmToken::Pipe,
"expected vertical bar"))
3364 if (
Lit != LitModifier::None &&
3368 AMDGPUOperand::Modifiers Mods;
3369 Mods.Abs = Abs || SP3Abs;
3370 Mods.Neg = Neg || SP3Neg;
3373 if (Mods.hasFPModifiers() ||
Lit != LitModifier::None) {
3374 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3376 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3377 Op.setModifiers(Mods);
3385 bool Sext = trySkipId(
"sext");
3386 if (Sext && !skipToken(
AsmToken::LParen,
"expected left paren after sext"))
3401 AMDGPUOperand::Modifiers Mods;
3404 if (Mods.hasIntModifiers()) {
3405 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands.back());
3407 return Error(
Op.getStartLoc(),
"expected an absolute expression");
3408 Op.setModifiers(Mods);
3415 return parseRegOrImmWithFPInputMods(
Operands,
false);
3419 return parseRegOrImmWithIntInputMods(
Operands,
false);
3426 if (!trySkipId(
"rsrcidx"))
3432 SMLoc RegLoc = getLoc();
3433 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3441 if (!
Reg->isRsrcReg32())
3442 return Error(RegLoc,
"rsrcidx operand must be a 32-bit SGPR or VGPR");
3452 auto Loc = getLoc();
3453 if (trySkipId(
"off")) {
3455 AMDGPUOperand::CreateImm(
this, 0, Loc, AMDGPUOperand::ImmTyOff,
false));
3462 std::unique_ptr<AMDGPUOperand>
Reg = parseRegister();
3471unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3476 return Match_InvalidOperand;
3478 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3479 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3482 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::dst_sel);
3484 if (!
Op.isImm() ||
Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3485 return Match_InvalidOperand;
3493 if (tryAnotherVOPDEncoding(Inst))
3494 return Match_InvalidOperand;
3496 return Match_Success;
3500 static const unsigned Variants[] = {
3509ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants()
const {
3510 if (isForcedDPP() && isForcedVOP3()) {
3514 if (getForcedEncodingSize() == 32) {
3519 if (isForcedVOP3()) {
3524 if (isForcedSDWA()) {
3530 if (isForcedDPP()) {
3538StringRef AMDGPUAsmParser::getMatchedVariantName()
const {
3539 if (isForcedDPP() && isForcedVOP3())
3542 if (getForcedEncodingSize() == 32)
3558AMDGPUAsmParser::findImplicitSGPRReadInVOP(
const MCInst &Inst)
const {
3562 case AMDGPU::FLAT_SCR:
3564 case AMDGPU::VCC_LO:
3565 case AMDGPU::VCC_HI:
3572 return MCRegister();
3579bool AMDGPUAsmParser::isInlineConstant(
const MCInst &Inst,
3580 unsigned OpIdx)
const {
3588 const MCOperand &MO = Inst.
getOperand(OpIdx);
3638unsigned AMDGPUAsmParser::getConstantBusLimit(
unsigned Opcode)
const {
3644 case AMDGPU::V_LSHLREV_B64_e64:
3645 case AMDGPU::V_LSHLREV_B64_gfx10:
3646 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3647 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3648 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3649 case AMDGPU::V_LSHRREV_B64_e64:
3650 case AMDGPU::V_LSHRREV_B64_gfx10:
3651 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3652 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3653 case AMDGPU::V_ASHRREV_I64_e64:
3654 case AMDGPU::V_ASHRREV_I64_gfx10:
3655 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3656 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3657 case AMDGPU::V_LSHL_B64_e64:
3658 case AMDGPU::V_LSHR_B64_e64:
3659 case AMDGPU::V_ASHR_I64_e64:
3672 bool AddMandatoryLiterals =
false) {
3675 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3679 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immX) : -1;
3681 return {getNamedOperandIdx(Opcode, OpName::src0X),
3682 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3683 getNamedOperandIdx(Opcode, OpName::vsrc2X),
3684 getNamedOperandIdx(Opcode, OpName::src0Y),
3685 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3686 getNamedOperandIdx(Opcode, OpName::vsrc2Y),
3691 return {getNamedOperandIdx(Opcode, OpName::src0),
3692 getNamedOperandIdx(Opcode, OpName::src1),
3693 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3696bool AMDGPUAsmParser::usesConstantBus(
const MCInst &Inst,
unsigned OpIdx) {
3697 const MCOperand &MO = Inst.
getOperand(OpIdx);
3699 return !isInlineConstant(Inst, OpIdx);
3706 return isSGPR(PReg,
TRI) && PReg != SGPR_NULL;
3717 const unsigned Opcode = Inst.
getOpcode();
3718 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3721 if (!LaneSelOp.
isReg())
3724 return LaneSelReg ==
M0 || LaneSelReg == M0_gfxpre11;
3727bool AMDGPUAsmParser::validateConstantBusLimitations(
3729 const unsigned Opcode = Inst.
getOpcode();
3730 const MCInstrDesc &
Desc = MII.
get(Opcode);
3731 MCRegister LastSGPR;
3732 unsigned ConstantBusUseCount = 0;
3733 unsigned NumLiterals = 0;
3734 unsigned LiteralSize;
3750 SmallDenseSet<MCRegister> SGPRsUsed;
3751 MCRegister SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3753 SGPRsUsed.
insert(SGPRUsed);
3754 ++ConstantBusUseCount;
3759 unsigned ConstantBusLimit = getConstantBusLimit(Opcode);
3761 for (
int OpIdx : OpIndices) {
3765 const MCOperand &MO = Inst.
getOperand(OpIdx);
3766 if (usesConstantBus(Inst, OpIdx)) {
3775 if (SGPRsUsed.
insert(LastSGPR).second) {
3776 ++ConstantBusUseCount;
3796 if (NumLiterals == 0) {
3799 }
else if (LiteralSize !=
Size) {
3805 if (ConstantBusUseCount + NumLiterals > ConstantBusLimit) {
3807 "invalid operand (violates constant bus restrictions)");
3814std::optional<unsigned>
3815AMDGPUAsmParser::checkVOPDRegBankConstraints(
const MCInst &Inst,
bool AsVOPD3) {
3817 const unsigned Opcode = Inst.
getOpcode();
3823 auto getVRegIdx = [&](unsigned,
unsigned OperandIdx) {
3824 const MCOperand &Opr = Inst.
getOperand(OperandIdx);
3833 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1170 ||
3834 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 ||
3835 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 ||
3836 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx13 ||
3837 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 ||
3838 Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx13;
3842 for (
auto OpName : {OpName::src0X, OpName::src0Y}) {
3843 int I = getNamedOperandIdx(Opcode, OpName);
3847 int64_t
Imm =
Op.getImm();
3853 for (
auto OpName : {OpName::vsrc1X, OpName::vsrc1Y, OpName::vsrc2X,
3854 OpName::vsrc2Y, OpName::imm}) {
3855 int I = getNamedOperandIdx(Opcode, OpName);
3865 auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(
3866 getVRegIdx, *
TRI, SkipSrc, AllowSameVGPR, AsVOPD3);
3868 return InvalidCompOprIdx;
3871bool AMDGPUAsmParser::validateVOPD(
const MCInst &Inst,
3878 for (
const std::unique_ptr<MCParsedAsmOperand> &Operand :
Operands) {
3879 AMDGPUOperand &
Op = (AMDGPUOperand &)*Operand;
3880 if ((
Op.isRegKind() ||
Op.isImmTy(AMDGPUOperand::ImmTyNone)) &&
3882 Error(
Op.getStartLoc(),
"ABS not allowed in VOPD3 instructions");
3886 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst, AsVOPD3);
3887 if (!InvalidCompOprIdx.has_value())
3890 auto CompOprIdx = *InvalidCompOprIdx;
3893 std::max(InstInfo[
VOPD::X].getIndexInParsedOperands(CompOprIdx),
3894 InstInfo[
VOPD::Y].getIndexInParsedOperands(CompOprIdx));
3897 auto Loc = ((AMDGPUOperand &)*
Operands[ParsedIdx]).getStartLoc();
3898 if (CompOprIdx == VOPD::Component::DST) {
3900 Error(Loc,
"dst registers must be distinct");
3902 Error(Loc,
"one dst register must be even and the other odd");
3904 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
3905 Error(Loc, Twine(
"src") + Twine(CompSrcIdx) +
3906 " operands must use different VGPR banks");
3914bool AMDGPUAsmParser::tryVOPD3(
const MCInst &Inst) {
3916 auto InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
false);
3917 if (!InvalidCompOprIdx.has_value())
3921 InvalidCompOprIdx = checkVOPDRegBankConstraints(Inst,
true);
3922 if (InvalidCompOprIdx.has_value()) {
3927 if (*InvalidCompOprIdx == VOPD::Component::DST)
3940bool AMDGPUAsmParser::tryVOPD(
const MCInst &Inst) {
3941 const unsigned Opcode = Inst.
getOpcode();
3956 for (
auto OpName : {OpName::src0X_modifiers, OpName::src0Y_modifiers,
3957 OpName::vsrc1X_modifiers, OpName::vsrc1Y_modifiers,
3958 OpName::vsrc2X_modifiers, OpName::vsrc2Y_modifiers}) {
3959 int I = getNamedOperandIdx(Opcode, OpName);
3966 return !tryVOPD3(Inst);
3971bool AMDGPUAsmParser::tryAnotherVOPDEncoding(
const MCInst &Inst) {
3976 return tryVOPD(Inst);
3977 return tryVOPD3(Inst);
3980bool AMDGPUAsmParser::validateIntClampSupported(
const MCInst &Inst) {
3985 int ClampIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::clamp);
3993bool AMDGPUAsmParser::validateMIMGDataSize(
const MCInst &Inst, SMLoc IDLoc) {
4001 int VDataIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdata);
4002 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4003 int TFEIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::tfe);
4008 if ((DMaskIdx == -1 || TFEIdx == -1) &&
4009 hasBVHRayTracingInsts())
4012 unsigned VDataSize = getRegOperandSize(
Desc, VDataIdx);
4013 unsigned TFESize = (TFEIdx != -1 && Inst.
getOperand(TFEIdx).
getImm()) ? 1 : 0;
4018 bool IsPackedD16 =
false;
4021 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4022 IsPackedD16 = D16Idx >= 0;
4024 DataSize = (DataSize + 1) / 2;
4027 if ((VDataSize / 4) == DataSize + TFESize)
4032 Modifiers = IsPackedD16 ?
"dmask and d16" :
"dmask";
4034 Modifiers = IsPackedD16 ?
"dmask, d16 and tfe" :
"dmask and tfe";
4036 Error(IDLoc, Twine(
"image data size does not match ") + Modifiers);
4040bool AMDGPUAsmParser::validateMIMGAddrSize(
const MCInst &Inst, SMLoc IDLoc) {
4049 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4051 int VAddr0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr0);
4052 AMDGPU::OpName RSrcOpName =
4054 int SrsrcIdx = AMDGPU::getNamedOperandIdx(
Opc, RSrcOpName);
4055 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4056 int A16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::a16);
4060 assert(SrsrcIdx > VAddr0Idx);
4063 if (BaseOpcode->
BVH) {
4064 if (IsA16 == BaseOpcode->
A16)
4066 Error(IDLoc,
"image address size does not match a16");
4072 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
4073 unsigned ActualAddrSize =
4074 IsNSA ? SrsrcIdx - VAddr0Idx : getRegOperandSize(
Desc, VAddr0Idx) / 4;
4076 unsigned ExpectedAddrSize =
4080 if (hasPartialNSAEncoding() &&
4082 int VAddrLastIdx = SrsrcIdx - 1;
4083 unsigned VAddrLastSize = getRegOperandSize(
Desc, VAddrLastIdx) / 4;
4085 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
4088 if (ExpectedAddrSize > 12)
4089 ExpectedAddrSize = 16;
4094 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
4098 if (ActualAddrSize == ExpectedAddrSize)
4101 Error(IDLoc,
"image address size does not match dim and a16");
4105bool AMDGPUAsmParser::validateMIMGAtomicDMask(
const MCInst &Inst) {
4112 if (!
Desc.mayLoad() || !
Desc.mayStore())
4115 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4122 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
4125bool AMDGPUAsmParser::validateMIMGGatherDMask(
const MCInst &Inst) {
4132 int DMaskIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dmask);
4140 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4143bool AMDGPUAsmParser::validateMIMGDim(
const MCInst &Inst,
4157 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4158 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4165bool AMDGPUAsmParser::validateMIMGMSAA(
const MCInst &Inst) {
4172 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4175 if (!BaseOpcode->
MSAA)
4178 int DimIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dim);
4184 return DimInfo->
MSAA;
4189 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4190 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4191 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4201bool AMDGPUAsmParser::validateMovrels(
const MCInst &Inst,
4209 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4220 Error(getOperandLoc(
Operands, Src0Idx),
"source operand must be a VGPR");
4224bool AMDGPUAsmParser::validateMAIAccWrite(
const MCInst &Inst,
4229 if (
Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4232 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4243 "source operand must be either a VGPR or an inline constant");
4250bool AMDGPUAsmParser::validateMAISrc2(
const MCInst &Inst,
4255 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4258 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4262 if (Inst.
getOperand(Src2Idx).
isImm() && isInlineConstant(Inst, Src2Idx)) {
4264 "inline constants are not allowed for this operand");
4271bool AMDGPUAsmParser::validateMFMA(
const MCInst &Inst,
4279 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
4280 if (BlgpIdx != -1) {
4281 if (
const MFMA_F8F6F4_Info *Info = AMDGPU::isMFMA_F8F6F4(
Opc)) {
4282 int CbszIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
4292 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4294 "wrong register tuple size for cbsz value " + Twine(CBSZ));
4299 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4301 "wrong register tuple size for blgp value " + Twine(BLGP));
4309 const int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4319 if (Src2Reg == DstReg)
4324 .getSizeInBits() <= 128)
4327 if (
TRI->regsOverlap(Src2Reg, DstReg)) {
4329 "source 2 operand must not partially overlap with dst");
4336bool AMDGPUAsmParser::validateDivScale(
const MCInst &Inst) {
4340 case V_DIV_SCALE_F32_gfx6_gfx7:
4341 case V_DIV_SCALE_F32_vi:
4342 case V_DIV_SCALE_F32_gfx10:
4343 case V_DIV_SCALE_F64_gfx6_gfx7:
4344 case V_DIV_SCALE_F64_vi:
4345 case V_DIV_SCALE_F64_gfx10:
4352 {AMDGPU::OpName::src0_modifiers, AMDGPU::OpName::src2_modifiers,
4353 AMDGPU::OpName::src2_modifiers}) {
4364bool AMDGPUAsmParser::validateMIMGD16(
const MCInst &Inst) {
4371 int D16Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::d16);
4380bool AMDGPUAsmParser::validateTensorR128(
const MCInst &Inst) {
4386 int R128Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::r128);
4393 case AMDGPU::V_SUBREV_F32_e32:
4394 case AMDGPU::V_SUBREV_F32_e64:
4395 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4396 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4397 case AMDGPU::V_SUBREV_F32_e32_vi:
4398 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4399 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4400 case AMDGPU::V_SUBREV_F32_e64_vi:
4402 case AMDGPU::V_SUBREV_CO_U32_e32:
4403 case AMDGPU::V_SUBREV_CO_U32_e64:
4404 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4405 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4407 case AMDGPU::V_SUBBREV_U32_e32:
4408 case AMDGPU::V_SUBBREV_U32_e64:
4409 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4410 case AMDGPU::V_SUBBREV_U32_e32_vi:
4411 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4412 case AMDGPU::V_SUBBREV_U32_e64_vi:
4414 case AMDGPU::V_SUBREV_U32_e32:
4415 case AMDGPU::V_SUBREV_U32_e64:
4416 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4417 case AMDGPU::V_SUBREV_U32_e32_vi:
4418 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4419 case AMDGPU::V_SUBREV_U32_e64_vi:
4421 case AMDGPU::V_SUBREV_F16_e32:
4422 case AMDGPU::V_SUBREV_F16_e64:
4423 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4424 case AMDGPU::V_SUBREV_F16_e32_vi:
4425 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4426 case AMDGPU::V_SUBREV_F16_e64_vi:
4428 case AMDGPU::V_SUBREV_U16_e32:
4429 case AMDGPU::V_SUBREV_U16_e64:
4430 case AMDGPU::V_SUBREV_U16_e32_vi:
4431 case AMDGPU::V_SUBREV_U16_e64_vi:
4433 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4434 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4435 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4437 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4438 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4440 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4441 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4443 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4444 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4446 case AMDGPU::V_LSHRREV_B32_e32:
4447 case AMDGPU::V_LSHRREV_B32_e64:
4448 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4449 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4450 case AMDGPU::V_LSHRREV_B32_e32_vi:
4451 case AMDGPU::V_LSHRREV_B32_e64_vi:
4452 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4453 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4455 case AMDGPU::V_ASHRREV_I32_e32:
4456 case AMDGPU::V_ASHRREV_I32_e64:
4457 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4458 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4459 case AMDGPU::V_ASHRREV_I32_e32_vi:
4460 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4461 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4462 case AMDGPU::V_ASHRREV_I32_e64_vi:
4464 case AMDGPU::V_LSHLREV_B32_e32:
4465 case AMDGPU::V_LSHLREV_B32_e64:
4466 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4467 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4468 case AMDGPU::V_LSHLREV_B32_e32_vi:
4469 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4470 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4471 case AMDGPU::V_LSHLREV_B32_e64_vi:
4473 case AMDGPU::V_LSHLREV_B16_e32:
4474 case AMDGPU::V_LSHLREV_B16_e64:
4475 case AMDGPU::V_LSHLREV_B16_e32_vi:
4476 case AMDGPU::V_LSHLREV_B16_e64_vi:
4477 case AMDGPU::V_LSHLREV_B16_gfx10:
4479 case AMDGPU::V_LSHRREV_B16_e32:
4480 case AMDGPU::V_LSHRREV_B16_e64:
4481 case AMDGPU::V_LSHRREV_B16_e32_vi:
4482 case AMDGPU::V_LSHRREV_B16_e64_vi:
4483 case AMDGPU::V_LSHRREV_B16_gfx10:
4485 case AMDGPU::V_ASHRREV_I16_e32:
4486 case AMDGPU::V_ASHRREV_I16_e64:
4487 case AMDGPU::V_ASHRREV_I16_e32_vi:
4488 case AMDGPU::V_ASHRREV_I16_e64_vi:
4489 case AMDGPU::V_ASHRREV_I16_gfx10:
4491 case AMDGPU::V_LSHLREV_B64_e64:
4492 case AMDGPU::V_LSHLREV_B64_gfx10:
4493 case AMDGPU::V_LSHLREV_B64_vi:
4495 case AMDGPU::V_LSHRREV_B64_e64:
4496 case AMDGPU::V_LSHRREV_B64_gfx10:
4497 case AMDGPU::V_LSHRREV_B64_vi:
4499 case AMDGPU::V_ASHRREV_I64_e64:
4500 case AMDGPU::V_ASHRREV_I64_gfx10:
4501 case AMDGPU::V_ASHRREV_I64_vi:
4503 case AMDGPU::V_PK_LSHLREV_B16:
4504 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4505 case AMDGPU::V_PK_LSHLREV_B16_vi:
4507 case AMDGPU::V_PK_LSHRREV_B16:
4508 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4509 case AMDGPU::V_PK_LSHRREV_B16_vi:
4510 case AMDGPU::V_PK_ASHRREV_I16:
4511 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4512 case AMDGPU::V_PK_ASHRREV_I16_vi:
4519bool AMDGPUAsmParser::validateLdsDirect(
const MCInst &Inst,
4521 const unsigned Opcode = Inst.
getOpcode();
4530 for (
auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4531 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4535 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4539 "lds_direct is not supported on this GPU");
4545 "lds_direct cannot be used with this instruction");
4549 if (SrcName != OpName::src0) {
4551 "lds_direct may be used as src0 only");
4561 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
4562 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4563 if (
Op.isFlatOffset())
4564 return Op.getStartLoc();
4569bool AMDGPUAsmParser::validateOffset(
const MCInst &Inst,
4572 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4577 return validateFlatOffset(Inst,
Operands);
4580 return validateSMEMOffset(Inst,
Operands);
4585 const unsigned OffsetSize = 24;
4586 if (!
isUIntN(OffsetSize - 1,
Op.getImm())) {
4588 Twine(
"expected a ") + Twine(OffsetSize - 1) +
4589 "-bit unsigned offset for buffer ops");
4593 const unsigned OffsetSize = 16;
4594 if (!
isUIntN(OffsetSize,
Op.getImm())) {
4596 Twine(
"expected a ") + Twine(OffsetSize) +
"-bit unsigned offset");
4603bool AMDGPUAsmParser::validateFlatOffset(
const MCInst &Inst,
4609 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4613 if (!hasFlatOffsets() &&
Op.getImm() != 0) {
4615 "flat offset modifier is not supported on this GPU");
4622 bool AllowNegative =
4624 if (!
isIntN(OffsetSize,
Op.getImm()) || (!AllowNegative &&
Op.getImm() < 0)) {
4626 Twine(
"expected a ") +
4627 (AllowNegative ? Twine(OffsetSize) +
"-bit signed offset"
4628 : Twine(OffsetSize - 1) +
"-bit unsigned offset"));
4637 for (
unsigned i = 2, e =
Operands.size(); i != e; ++i) {
4638 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
4639 if (
Op.isSMEMOffset() ||
Op.isSMEMOffsetMod())
4640 return Op.getStartLoc();
4645bool AMDGPUAsmParser::validateSMEMOffset(
const MCInst &Inst,
4654 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4670 ?
"expected a 23-bit unsigned offset for buffer ops"
4671 :
isGFX12Plus() ?
"expected a 24-bit signed offset"
4672 : (
isVI() || IsBuffer) ?
"expected a 20-bit unsigned offset"
4673 :
"expected a 21-bit signed offset");
4683bool AMDGPUAsmParser::validateBF16InlineConst(
const MCInst &Inst,
4685 if (!getFeatureBits()[AMDGPU::FeatureBF16InlineConstFromUpperFP32])
4700 const int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4704 const MCOperandInfo &Src0Info =
Desc.operands()[Src0Idx];
4709 if (!
Src0.isImm() ||
4711 hasInv2PiInlineImm()))
4716 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
4717 if (ModsIdx != -1 &&
4723 "bf16 inline constant is read from the high half of the fp32 inline "
4724 "constant on this GPU; use the e64 encoding with op_sel:[1,0]");
4728bool AMDGPUAsmParser::validateSOPLiteral(
const MCInst &Inst,
4731 const MCInstrDesc &
Desc = MII.
get(Opcode);
4735 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4736 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4738 const int OpIndices[] = {Src0Idx, Src1Idx};
4740 unsigned NumExprs = 0;
4741 unsigned NumLiterals = 0;
4744 for (
int OpIdx : OpIndices) {
4748 const MCOperand &MO = Inst.
getOperand(OpIdx);
4752 std::optional<int64_t>
Imm;
4755 }
else if (MO.
isExpr()) {
4764 if (!
Imm.has_value()) {
4766 }
else if (!isInlineConstant(Inst, OpIdx)) {
4770 if (NumLiterals == 0 || LiteralValue !=
Value) {
4778 if (NumLiterals + NumExprs <= 1)
4782 "only one unique literal operand is allowed");
4786bool AMDGPUAsmParser::validateOpSel(
const MCInst &Inst) {
4789 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4797 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4798 if (OpSelIdx != -1) {
4802 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4803 if (OpSelHiIdx != -1) {
4812 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4819 if (getFeatureBits()[AMDGPU::FeatureGFX950Insts] &&
4820 (
Opc == AMDGPU::V_PK_ADD_F32_vi ||
Opc == AMDGPU::V_PK_MUL_F32_vi ||
4821 Opc == AMDGPU::V_PK_FMA_F32_vi)) {
4822 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4826 if (
Opc == AMDGPU::V_PK_FMA_F32_vi &&
4827 (((OpSel >> 1) & 1) || ((OpSel >> 2) & 1)))
4830 else if ((OpSel & 3) == 2)
4838 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
4839 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4840 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4841 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
4850 auto VerifyOneSGPR = [
OpSel, OpSelHi](
unsigned Index) ->
bool {
4852 return ((OpSel & Mask) == 0) && ((OpSelHi &
Mask) == 0);
4862 int Src2Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2);
4863 if (Src2Idx != -1) {
4874bool AMDGPUAsmParser::validateTrue16OpSel(
const MCInst &Inst) {
4875 if (!hasTrue16Insts())
4877 const MCRegisterInfo *MRI = getMRI();
4879 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
4885 if (OpSelOpValue == 0)
4887 unsigned OpCount = 0;
4888 for (AMDGPU::OpName OpName : {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
4889 AMDGPU::OpName::src2, AMDGPU::OpName::vdst}) {
4890 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), OpName);
4897 bool OpSelOpIsHi = ((OpSelOpValue & (1 << OpCount)) != 0);
4898 if (OpSelOpIsHi != VGPRSuffixIsHi)
4907bool AMDGPUAsmParser::validateNeg(
const MCInst &Inst, AMDGPU::OpName OpName) {
4908 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
4920 int NegIdx = AMDGPU::getNamedOperandIdx(
Opc, OpName);
4931 const AMDGPU::OpName SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
4932 AMDGPU::OpName::src1_modifiers,
4933 AMDGPU::OpName::src2_modifiers};
4935 for (
unsigned i = 0; i < 3; ++i) {
4945bool AMDGPUAsmParser::validateDPP(
const MCInst &Inst,
4948 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp_ctrl);
4949 if (DppCtrlIdx >= 0) {
4953 getSTI().
hasFeature(AMDGPU::FeatureDPALU_DPP) &&
4957 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl,
Operands);
4958 Error(S,
isGFX12() ?
"DP ALU dpp only supports row_share"
4959 :
"DP ALU dpp only supports row_newbcast");
4964 int Dpp8Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::dpp8);
4965 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
4968 int Src1Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src1);
4974 "invalid operand for instruction");
4979 "src1 immediate operand invalid for instruction");
4989bool AMDGPUAsmParser::validateVccOperand(MCRegister
Reg)
const {
4990 return (
Reg == AMDGPU::VCC && isWave64()) ||
4991 (
Reg == AMDGPU::VCC_LO && isWave32());
4995bool AMDGPUAsmParser::validateVOPLiteral(
const MCInst &Inst,
4998 const MCInstrDesc &
Desc = MII.
get(Opcode);
4999 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
5006 std::optional<unsigned> LiteralOpIdx;
5009 for (
int OpIdx : OpIndices) {
5013 const MCOperand &MO = Inst.
getOperand(OpIdx);
5019 std::optional<int64_t>
Imm;
5025 bool IsAnotherLiteral =
false;
5026 bool IsForcedLit = findMCOperand(
Operands, OpIdx).isForcedLit();
5027 bool IsForcedLit64 = findMCOperand(
Operands, OpIdx).isForcedLit64();
5028 if (!
Imm.has_value()) {
5030 IsAnotherLiteral =
true;
5031 }
else if (IsForcedLit || IsForcedLit64 || !isInlineConstant(Inst, OpIdx)) {
5036 HasMandatoryLiteral);
5048 (IsForcedLit64 && !HasMandatoryLiteral)) &&
5049 (!has64BitLiterals() ||
Desc.getSize() != 4)) {
5051 "invalid operand for instruction");
5056 if (!IsForcedFP64 && (IsForcedLit64 || !IsValid32Op) &&
5057 OpIdx != getNamedOperandIdx(Opcode, OpName::src0)) {
5059 "invalid operand for instruction");
5064 if (IsValid32Op && !IsForcedFP64 && !IsForcedLit64) {
5065 Value =
static_cast<uint32_t
>(
5073 if (IsAnotherLiteral && !HasMandatoryLiteral &&
5074 !getFeatureBits()[FeatureVOP3Literal]) {
5076 "literal operands are not supported");
5080 if (LiteralOpIdx && IsAnotherLiteral) {
5082 getOperandLoc(
Operands, *LiteralOpIdx)),
5083 "only one unique literal operand is allowed");
5087 if (IsAnotherLiteral)
5088 LiteralOpIdx = OpIdx;
5097 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), Name);
5111bool AMDGPUAsmParser::validateAGPRLdSt(
const MCInst &Inst)
const {
5117 ? AMDGPU::OpName::data0
5118 : AMDGPU::OpName::vdata;
5120 const MCRegisterInfo *MRI = getMRI();
5121 int DstAreg =
IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
5125 int Data2Areg =
IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
5126 if (Data2Areg >= 0 && Data2Areg != DataAreg)
5130 auto FB = getFeatureBits();
5131 if (FB[AMDGPU::FeatureGFX90AInsts]) {
5132 if (DataAreg < 0 || DstAreg < 0)
5134 return DstAreg == DataAreg;
5137 return DstAreg < 1 && DataAreg < 1;
5140bool AMDGPUAsmParser::validateVGPRAlign(
const MCInst &Inst)
const {
5141 auto FB = getFeatureBits();
5142 if (!FB[AMDGPU::FeatureRequiresAlignedVGPRs])
5146 const MCRegisterInfo *MRI = getMRI();
5149 if (FB[AMDGPU::FeatureGFX90AInsts] &&
Opc == AMDGPU::DS_READ_B96_TR_B6_vi)
5152 if (FB[AMDGPU::FeatureGFX1250Insts]) {
5156 case AMDGPU::DS_LOAD_TR6_B96:
5157 case AMDGPU::DS_LOAD_TR6_B96_gfx12:
5161 case AMDGPU::GLOBAL_LOAD_TR6_B96:
5162 case AMDGPU::GLOBAL_LOAD_TR6_B96_gfx1250: {
5166 int VAddrIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vaddr);
5167 if (VAddrIdx != -1) {
5170 if ((
Sub - AMDGPU::VGPR0) & 1)
5175 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR:
5176 case AMDGPU::GLOBAL_LOAD_TR6_B96_SADDR_gfx1250:
5181 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5182 const MCRegisterClass &AGPR32 = MRI->
getRegClass(AMDGPU::AGPR_32RegClassID);
5202 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
5203 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
5205 return Op.getStartLoc();
5210bool AMDGPUAsmParser::validateBLGP(
const MCInst &Inst,
5213 int BlgpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
5216 SMLoc BLGPLoc = getBLGPLoc(
Operands);
5219 bool IsNeg = StringRef(BLGPLoc.
getPointer()).starts_with(
"neg:");
5220 auto FB = getFeatureBits();
5221 bool UsesNeg =
false;
5222 if (FB[AMDGPU::FeatureGFX940Insts]) {
5224 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
5225 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
5226 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
5227 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
5232 if (IsNeg == UsesNeg)
5235 Error(BLGPLoc, UsesNeg ?
"invalid modifier: blgp is not supported"
5236 :
"invalid modifier: neg is not supported");
5241bool AMDGPUAsmParser::validateWaitCnt(
const MCInst &Inst,
5247 if (
Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
5248 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
5249 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
5250 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
5253 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::sdst);
5256 if (
Reg == AMDGPU::SGPR_NULL)
5259 Error(getOperandLoc(
Operands, Src0Idx),
"src0 must be null");
5263bool AMDGPUAsmParser::validateDS(
const MCInst &Inst,
5268 return validateGWS(Inst,
Operands);
5273 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::gds);
5278 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS,
Operands);
5279 Error(S,
"gds modifier is not supported on this GPU");
5287bool AMDGPUAsmParser::validateGWS(
const MCInst &Inst,
5289 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
5293 if (
Opc != AMDGPU::DS_GWS_INIT_vi &&
Opc != AMDGPU::DS_GWS_BARRIER_vi &&
5294 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
5297 const MCRegisterInfo *MRI = getMRI();
5298 const MCRegisterClass &VGPR32 = MRI->
getRegClass(AMDGPU::VGPR_32RegClassID);
5300 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::data0);
5303 auto RegIdx =
Reg - (VGPR32.
contains(
Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
5305 Error(getOperandLoc(
Operands, Data0Pos),
"vgpr must be even aligned");
5312bool AMDGPUAsmParser::validateCoherencyBits(
const MCInst &Inst,
5316 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::cpol);
5324 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5327 Error(S,
"scale_offset is not supported on this GPU");
5330 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5333 Error(S,
"nv is not supported on this GPU");
5338 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5341 Error(S,
"scale_offset is not supported for this instruction");
5345 return validateTHAndScopeBits(Inst,
Operands, CPol);
5349 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5350 Error(S,
"cache policy is not supported for SMRD instructions");
5354 Error(IDLoc,
"invalid cache policy for SMEM instruction");
5361 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5365 "scc modifier is not supported for this instruction on this GPU");
5376 :
"instruction must use glc");
5381 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5384 &CStr.data()[CStr.find(
isGFX940() ?
"sc0" :
"glc")]);
5386 :
"instruction must not use glc");
5394bool AMDGPUAsmParser::validateTHAndScopeBits(
const MCInst &Inst,
5396 const unsigned CPol) {
5401 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol,
Operands);
5408 return PrintError(
"th:TH_ATOMIC_RETURN requires a destination operand");
5413 return PrintError(
"instruction must use th:TH_ATOMIC_RETURN");
5421 return PrintError(
"invalid th value for SMEM instruction");
5428 return PrintError(
"scope and th combination is not valid");
5434 return PrintError(
"invalid th value for atomic instructions");
5437 return PrintError(
"invalid th value for store instructions");
5440 return PrintError(
"invalid th value for load instructions");
5446bool AMDGPUAsmParser::validateTFE(
const MCInst &Inst,
5450 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE,
Operands);
5452 Error(Loc,
"TFE modifier has no meaning for store instructions");
5460bool AMDGPUAsmParser::validateWMMA(
const MCInst &Inst,
5466 int AFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
5470 int BFmtIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
5473 auto validateFmt = [&](
unsigned Fmt, AMDGPU::OpName SrcOp) ->
bool {
5474 int SrcIdx = AMDGPU::getNamedOperandIdx(
Opc, SrcOp);
5483 "wrong register tuple size for " +
5488 if (!validateFmt(AFmt, AMDGPU::OpName::src0) ||
5489 !validateFmt(BFmt, AMDGPU::OpName::src1))
5493 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
5494 if (AScaleIdx == -1)
5498 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
5502 "invalid matrix and scale format combination");
5509bool AMDGPUAsmParser::validateMonitorSleep(
const MCInst &Inst,
5512 if (
Opc != AMDGPU::S_MONITOR_SLEEP_gfx12 ||
5513 !getSTI().
hasFeature(AMDGPU::FeatureNoSleepForever))
5516 int ImmIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::simm16);
5519 "sleep forever is unsuported on the target");
5526bool AMDGPUAsmParser::validateClusterBarrierIsFirst(
5529 if (
Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx12 &&
5530 Opc != AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM_gfx13)
5533 int Src0Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0);
5540 "s_barrier_signal_isfirst does not support user_cluster_barrier_id (-3)");
5544bool AMDGPUAsmParser::validateScaleSel(
const MCInst &Inst,
5547 int ScaleSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::scale_sel);
5548 if (ScaleSelIdx == -1)
5552 case AMDGPU::V_CVT_SCALE_PK16_F16_FP6_e64_gfx1250:
5553 case AMDGPU::V_CVT_SCALE_PK16_BF16_FP6_e64_gfx1250:
5554 case AMDGPU::V_CVT_SCALE_PK16_F16_BF6_e64_gfx1250:
5555 case AMDGPU::V_CVT_SCALE_PK16_BF16_BF6_e64_gfx1250:
5556 case AMDGPU::V_CVT_SCALE_PK16_F32_FP6_e64_gfx1250:
5557 case AMDGPU::V_CVT_SCALE_PK16_F32_BF6_e64_gfx1250:
5558 case AMDGPU::V_CVT_SCALE_PK8_F16_FP4_e64_gfx1250:
5559 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP4_e64_gfx1250:
5560 case AMDGPU::V_CVT_SCALE_PK8_F32_FP4_e64_gfx1250:
5563 case AMDGPU::V_CVT_SCALE_PK8_F16_FP8_e64_gfx1250:
5564 case AMDGPU::V_CVT_SCALE_PK8_BF16_FP8_e64_gfx1250:
5565 case AMDGPU::V_CVT_SCALE_PK8_F16_BF8_e64_gfx1250:
5566 case AMDGPU::V_CVT_SCALE_PK8_BF16_BF8_e64_gfx1250:
5567 case AMDGPU::V_CVT_SCALE_PK8_F32_FP8_e64_gfx1250:
5568 case AMDGPU::V_CVT_SCALE_PK8_F32_BF8_e64_gfx1250:
5575 if (getSTI().
hasFeature(AMDGPU::FeatureBlock16ConversionScaleInsts))
5579 if (ScaleSel < MaxSel)
5583 "scale_sel maximum supported value is " + Twine(MaxSel - 1));
5587bool AMDGPUAsmParser::validateInstruction(
const MCInst &Inst, SMLoc IDLoc,
5589 if (!validateLdsDirect(Inst,
Operands))
5591 if (!validateTrue16OpSel(Inst)) {
5593 "op_sel operand conflicts with 16-bit operand suffix");
5596 if (!validateSOPLiteral(Inst,
Operands))
5598 if (!validateVOPLiteral(Inst,
Operands)) {
5601 if (!validateConstantBusLimitations(Inst,
Operands)) {
5604 if (!validateVOPD(Inst,
Operands)) {
5607 if (!validateIntClampSupported(Inst)) {
5609 "integer clamping is not supported on this GPU");
5612 if (!validateOpSel(Inst)) {
5614 "invalid op_sel operand");
5617 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5619 "invalid neg_lo operand");
5622 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5624 "invalid neg_hi operand");
5627 if (!validateDPP(Inst,
Operands)) {
5631 if (!validateMIMGD16(Inst)) {
5633 "d16 modifier is not supported on this GPU");
5636 if (!validateMIMGDim(Inst,
Operands)) {
5637 Error(IDLoc,
"missing dim operand");
5640 if (!validateTensorR128(Inst)) {
5642 "instruction must set modifier r128=0");
5645 if (!validateMIMGMSAA(Inst)) {
5647 "invalid dim; must be MSAA type");
5650 if (!validateMIMGDataSize(Inst, IDLoc)) {
5653 if (!validateMIMGAddrSize(Inst, IDLoc))
5655 if (!validateMIMGAtomicDMask(Inst)) {
5657 "invalid atomic image dmask");
5660 if (!validateMIMGGatherDMask(Inst)) {
5662 "invalid image_gather dmask: only one bit must be set");
5665 if (!validateMovrels(Inst,
Operands)) {
5668 if (!validateOffset(Inst,
Operands)) {
5671 if (!validateBF16InlineConst(Inst,
Operands)) {
5674 if (!validateMAIAccWrite(Inst,
Operands)) {
5677 if (!validateMAISrc2(Inst,
Operands)) {
5680 if (!validateMFMA(Inst,
Operands)) {
5683 if (!validateCoherencyBits(Inst,
Operands, IDLoc)) {
5687 if (!validateAGPRLdSt(Inst)) {
5690 getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5691 ?
"invalid register class: data and dst should be all VGPR or AGPR"
5692 :
"invalid register class: agpr loads and stores not supported on "
5696 if (!validateVGPRAlign(Inst)) {
5697 Error(IDLoc,
"invalid register class: vgpr tuples must be 64 bit aligned");
5704 if (!validateBLGP(Inst,
Operands)) {
5708 if (!validateDivScale(Inst)) {
5709 Error(IDLoc,
"ABS not allowed in VOP3B instructions");
5712 if (!validateWaitCnt(Inst,
Operands)) {
5715 if (!validateTFE(Inst,
Operands)) {
5718 if (!validateWMMA(Inst,
Operands)) {
5721 if (!validateMonitorSleep(Inst,
Operands)) {
5724 if (!validateClusterBarrierIsFirst(Inst,
Operands)) {
5727 if (!validateScaleSel(Inst,
Operands)) {
5736 unsigned VariantID = 0);
5740 unsigned VariantID);
5742bool AMDGPUAsmParser::isSupportedMnemo(
StringRef Mnemo,
5747bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5748 const FeatureBitset &FBS,
5749 ArrayRef<unsigned> Variants) {
5750 for (
auto Variant : Variants) {
5758bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5760 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5763 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5768 getParser().clearPendingErrors();
5772 StringRef VariantName = getMatchedVariantName();
5773 if (!VariantName.
empty() && isSupportedMnemo(Mnemo, FBS)) {
5774 return Error(IDLoc, Twine(VariantName,
5775 " variant of this instruction is not supported"));
5779 if (
isGFX10Plus() && getFeatureBits()[AMDGPU::FeatureWavefrontSize64] &&
5780 !getFeatureBits()[AMDGPU::FeatureWavefrontSize32]) {
5782 FeatureBitset FeaturesWS32 = getFeatureBits();
5783 FeaturesWS32.
flip(AMDGPU::FeatureWavefrontSize64)
5784 .
flip(AMDGPU::FeatureWavefrontSize32);
5785 FeatureBitset AvailableFeaturesWS32 =
5786 ComputeAvailableFeatures(FeaturesWS32);
5788 if (isSupportedMnemo(Mnemo, AvailableFeaturesWS32, getMatchedVariants()))
5789 return Error(IDLoc,
"instruction requires wavesize=32");
5793 if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
5794 return Error(IDLoc,
"instruction not supported on this GPU (" +
5795 getSTI().
getCPU() +
")" +
": " + Mnemo);
5800 return Error(IDLoc,
"invalid instruction" + Suggestion);
5806 const auto &
Op = ((AMDGPUOperand &)*
Operands[InvalidOprIdx]);
5807 if (
Op.isToken() && InvalidOprIdx > 1) {
5808 const auto &PrevOp = ((AMDGPUOperand &)*
Operands[InvalidOprIdx - 1]);
5809 return PrevOp.isToken() && PrevOp.getToken() ==
"::";
5814bool AMDGPUAsmParser::matchAndEmitInstruction(SMLoc IDLoc,
unsigned &Opcode,
5818 bool MatchingInlineAsm) {
5821 unsigned Result = Match_Success;
5826 auto atLeastAsSpecific = [](
unsigned New,
unsigned Cur) {
5827 auto rank = [](
unsigned M) {
5828 return M == Match_MnemonicFail ? 1
5829 :
M == Match_InvalidOperand ? 2
5830 :
M == Match_MissingFeature ? 3
5833 return rank(New) >= rank(Cur);
5836 for (
auto Variant : getMatchedVariants()) {
5839 MatchInstructionImpl(
Operands, Inst, EI, MatchingInlineAsm, Variant);
5840 if (R == Match_Success || atLeastAsSpecific(R, Result)) {
5844 if (R == Match_Success)
5848 if (Result == Match_Success) {
5849 if (!validateInstruction(Inst, IDLoc,
Operands)) {
5852 emitTargetDirective();
5853 Out.emitInstruction(Inst, getSTI());
5860 if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
5867 case Match_MissingFeature:
5871 return Error(IDLoc,
"operands are not valid for this GPU or mode");
5873 case Match_InvalidOperand: {
5874 SMLoc ErrorLoc = IDLoc;
5875 if (ErrorInfo != ~0ULL) {
5876 if (ErrorInfo >=
Operands.size()) {
5877 return Error(IDLoc,
"too few operands for instruction");
5879 AMDGPUOperand &ErrorOp = (AMDGPUOperand &)*
Operands[ErrorInfo];
5880 ErrorLoc = ErrorOp.getStartLoc();
5881 if (ErrorLoc == SMLoc())
5885 return Error(ErrorLoc,
"invalid VOPDY instruction");
5887 return Error(ErrorLoc,
"invalid operand for instruction");
5890 case Match_MnemonicFail:
5896bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
5901 if (getParser().parseAbsoluteExpression(Tmp)) {
5904 Ret =
static_cast<uint32_t
>(Tmp);
5908bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
5909 if (!getSTI().getTargetTriple().isAMDGCN())
5910 return TokError(
"directive only supported for amdgcn architecture");
5912 std::string TargetIDDirective;
5913 SMLoc TargetStart = getTok().getLoc();
5914 if (getParser().parseEscapedString(TargetIDDirective))
5917 std::optional<AMDGPU::TargetID> MaybeParsed =
5920 return getParser().Error(TargetStart,
5921 "malformed target id '" + TargetIDDirective +
"'");
5924 const Triple &
TT = getSTI().getTargetTriple();
5930 return getParser().Error(
5931 TargetStart,
"target id '" + TargetIDDirective +
5932 "' specifies a processor that is not valid for "
5934 TT.getArchName() +
"'");
5937 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
5938 getTargetStreamer().getTargetID();
5941 const Triple &STITriple = getSTI().getTargetTriple();
5942 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
5943 return getParser().Error(
5944 TargetStart,
".amdgcn_target " + Twine(ParsedTargetID.
toString()) +
5945 " is incompatible with " +
5946 Twine(CurrentTargetID->toString()));
5950 StringRef DirectiveProcessor =
5953 if (DirectiveISA != ISA) {
5954 return getParser().Error(TargetStart,
5955 ".amdgcn_target directive processor " +
5956 Twine(DirectiveProcessor) +
5957 " does not match the specified processor " +
5958 Twine(getSTI().
getCPU()));
5964 CurrentTargetID->getXnackSetting())) {
5966 ".amdgcn_target directive has conflicting xnack settings");
5970 CurrentTargetID->getSramEccSetting())) {
5972 ".amdgcn_target directive has conflicting sramecc settings");
5978 getTargetStreamer().getTargetID()->setXnackSetting(
5980 getTargetStreamer().getTargetID()->setSramEccSetting(
5986bool AMDGPUAsmParser::OutOfRangeError(SMRange
Range) {
5990bool AMDGPUAsmParser::calculateGPRBlocks(
5991 const FeatureBitset &Features,
const MCExpr *VCCUsed,
5992 const MCExpr *FlatScrUsed,
bool XNACKUsed,
5993 std::optional<bool> EnableWavefrontSize32,
const MCExpr *NextFreeVGPR,
5994 SMRange VGPRRange,
const MCExpr *NextFreeSGPR, SMRange SGPRRange,
5995 const MCExpr *&VGPRBlocks,
const MCExpr *&SGPRBlocks) {
6000 const MCExpr *
NumSGPRs = NextFreeSGPR;
6001 int64_t EvaluatedSGPRs;
6003 if (
ISA.Major >= 10)
6008 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
ISA.Major >= 8 &&
6009 !Features.
test(FeatureSGPRInitBug) &&
6010 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6011 return OutOfRangeError(SGPRRange);
6013 const MCExpr *ExtraSGPRs =
6017 if (
NumSGPRs->evaluateAsAbsolute(EvaluatedSGPRs) &&
6018 (
ISA.Major <= 7 || Features.
test(FeatureSGPRInitBug)) &&
6019 static_cast<uint64_t>(EvaluatedSGPRs) > MaxAddressableNumSGPRs)
6020 return OutOfRangeError(SGPRRange);
6022 if (Features.
test(FeatureSGPRInitBug))
6029 auto GetNumGPRBlocks = [&Ctx](
const MCExpr *NumGPR,
6030 unsigned Granule) ->
const MCExpr * {
6034 const MCExpr *AlignToGPR =
6036 const MCExpr *DivGPR =
6042 VGPRBlocks = GetNumGPRBlocks(
6051bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
6052 if (!getSTI().getTargetTriple().isAMDGCN())
6053 return TokError(
"directive only supported for amdgcn architecture");
6056 return TokError(
"directive only supported for amdhsa OS");
6058 StringRef KernelName;
6059 if (getParser().parseIdentifier(KernelName))
6066 AMDGPU::MCKernelDescriptor KD =
6076 const MCExpr *NextFreeVGPR = ZeroExpr;
6078 const MCExpr *NamedBarCnt = ZeroExpr;
6083 const MCExpr *NextFreeSGPR = ZeroExpr;
6086 unsigned ImpliedUserSGPRCount = 0;
6090 std::optional<unsigned> ExplicitUserSGPRCount;
6091 const MCExpr *ReserveVCC = OneExpr;
6092 const MCExpr *ReserveFlatScr = OneExpr;
6093 std::optional<bool> EnableWavefrontSize32;
6100 SMRange IDRange = getTok().getLocRange();
6101 if (!parseId(ID,
"expected .amdhsa_ directive or .end_amdhsa_kernel"))
6104 if (ID ==
".end_amdhsa_kernel")
6107 if (!Seen.
insert(ID).second)
6108 return TokError(
".amdhsa_ directives cannot be repeated");
6110 SMLoc ValStart = getLoc();
6111 const MCExpr *ExprVal;
6112 if (getParser().parseExpression(ExprVal))
6114 SMLoc ValEnd = getLoc();
6115 SMRange ValRange = SMRange(ValStart, ValEnd);
6119 bool EvaluatableExpr;
6120 if ((EvaluatableExpr = ExprVal->evaluateAsAbsolute(IVal))) {
6122 return OutOfRangeError(ValRange);
6126#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
6127 if (!isUInt<ENTRY##_WIDTH>(Val)) \
6128 return OutOfRangeError(RANGE); \
6129 AMDGPU::MCKernelDescriptor::bits_set(FIELD, VALUE, ENTRY##_SHIFT, ENTRY, \
6134#define EXPR_RESOLVE_OR_ERROR(RESOLVED) \
6136 return Error(IDRange.Start, "directive should have resolvable expression", \
6139 if (ID ==
".amdhsa_group_segment_fixed_size") {
6142 return OutOfRangeError(ValRange);
6144 }
else if (ID ==
".amdhsa_private_segment_fixed_size") {
6147 return OutOfRangeError(ValRange);
6149 }
else if (ID ==
".amdhsa_kernarg_size") {
6151 return OutOfRangeError(ValRange);
6153 }
else if (ID ==
".amdhsa_user_sgpr_count") {
6155 ExplicitUserSGPRCount = Val;
6156 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_buffer") {
6160 "directive is not supported with architected flat scratch",
6163 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
6166 ImpliedUserSGPRCount += 4;
6167 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_length") {
6170 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6173 return OutOfRangeError(ValRange);
6177 ImpliedUserSGPRCount += Val;
6178 PreloadLength = Val;
6180 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_preload_offset") {
6183 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6186 return OutOfRangeError(ValRange);
6190 PreloadOffset = Val;
6191 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_ptr") {
6194 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, ExprVal,
6197 ImpliedUserSGPRCount += 2;
6198 }
else if (ID ==
".amdhsa_user_sgpr_queue_ptr") {
6201 KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, ExprVal,
6204 ImpliedUserSGPRCount += 2;
6205 }
else if (ID ==
".amdhsa_user_sgpr_kernarg_segment_ptr") {
6208 KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
6211 ImpliedUserSGPRCount += 2;
6212 }
else if (ID ==
".amdhsa_user_sgpr_dispatch_id") {
6215 KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, ExprVal,
6218 ImpliedUserSGPRCount += 2;
6219 }
else if (ID ==
".amdhsa_user_sgpr_flat_scratch_init") {
6222 "directive is not supported with architected flat scratch",
6226 KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT,
6229 ImpliedUserSGPRCount += 2;
6230 }
else if (ID ==
".amdhsa_user_sgpr_private_segment_size") {
6233 KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
6236 ImpliedUserSGPRCount += 1;
6237 }
else if (ID ==
".amdhsa_wavefront_size32") {
6240 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6241 EnableWavefrontSize32 = Val;
6243 KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, ExprVal,
6245 }
else if (ID ==
".amdhsa_uses_dynamic_stack") {
6247 KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, ExprVal,
6249 }
else if (ID ==
".amdhsa_system_sgpr_private_segment_wavefront_offset") {
6252 "directive is not supported with architected flat scratch",
6255 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6257 }
else if (ID ==
".amdhsa_enable_private_segment") {
6261 "directive is not supported without architected flat scratch",
6264 COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, ExprVal,
6266 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_x") {
6268 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, ExprVal,
6270 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_y") {
6272 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, ExprVal,
6274 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_id_z") {
6276 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, ExprVal,
6278 }
else if (ID ==
".amdhsa_system_sgpr_workgroup_info") {
6280 COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, ExprVal,
6282 }
else if (ID ==
".amdhsa_system_vgpr_workitem_id") {
6284 COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, ExprVal,
6286 }
else if (ID ==
".amdhsa_next_free_vgpr") {
6287 VGPRRange = ValRange;
6288 NextFreeVGPR = ExprVal;
6289 }
else if (ID ==
".amdhsa_next_free_sgpr") {
6290 SGPRRange = ValRange;
6291 NextFreeSGPR = ExprVal;
6292 }
else if (ID ==
".amdhsa_accum_offset") {
6294 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6295 AccumOffset = ExprVal;
6296 }
else if (ID ==
".amdhsa_named_barrier_count") {
6298 return Error(IDRange.
Start,
"directive requires gfx1250+", IDRange);
6299 NamedBarCnt = ExprVal;
6300 }
else if (ID ==
".amdhsa_reserve_vcc") {
6302 return OutOfRangeError(ValRange);
6303 ReserveVCC = ExprVal;
6304 }
else if (ID ==
".amdhsa_reserve_flat_scratch") {
6306 return Error(IDRange.
Start,
"directive requires gfx7+", IDRange);
6309 "directive is not supported with architected flat scratch",
6312 return OutOfRangeError(ValRange);
6313 ReserveFlatScr = ExprVal;
6314 }
else if (ID ==
".amdhsa_reserve_xnack_mask") {
6316 return Error(IDRange.
Start,
"directive requires gfx8+", IDRange);
6318 return OutOfRangeError(ValRange);
6319 bool XnackOn = getTargetStreamer().getTargetID()->isXnackOnOrAny();
6320 if (Val != XnackOn) {
6321 return getParser().Error(
6323 ".amdhsa_reserve_xnack_mask does not match target id", IDRange);
6325 }
else if (ID ==
".amdhsa_float_round_mode_32") {
6327 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, ExprVal,
6329 }
else if (ID ==
".amdhsa_float_round_mode_16_64") {
6331 COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, ExprVal,
6333 }
else if (ID ==
".amdhsa_float_denorm_mode_32") {
6335 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, ExprVal,
6337 }
else if (ID ==
".amdhsa_float_denorm_mode_16_64") {
6339 COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, ExprVal,
6341 }
else if (ID ==
".amdhsa_dx10_clamp") {
6342 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6343 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6346 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, ExprVal,
6348 }
else if (ID ==
".amdhsa_ieee_mode") {
6349 if (!getSTI().
hasFeature(AMDGPU::FeatureDX10ClampAndIEEEMode))
6350 return Error(IDRange.
Start,
"directive unsupported on gfx1170+",
6353 COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, ExprVal,
6355 }
else if (ID ==
".amdhsa_fp16_overflow") {
6357 return Error(IDRange.
Start,
"directive requires gfx9+", IDRange);
6359 COMPUTE_PGM_RSRC1_GFX9_PLUS_FP16_OVFL, ExprVal,
6361 }
else if (ID ==
".amdhsa_tg_split") {
6363 return Error(IDRange.
Start,
"directive requires gfx90a+", IDRange);
6366 }
else if (ID ==
".amdhsa_workgroup_processor_mode") {
6369 "directive unsupported on " + getSTI().
getCPU(), IDRange);
6371 COMPUTE_PGM_RSRC1_GFX10_PLUS_WGP_MODE, ExprVal,
6373 }
else if (ID ==
".amdhsa_memory_ordered") {
6375 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6377 COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED, ExprVal,
6379 }
else if (ID ==
".amdhsa_forward_progress") {
6381 return Error(IDRange.
Start,
"directive requires gfx10+", IDRange);
6383 COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS, ExprVal,
6385 }
else if (ID ==
".amdhsa_shared_vgpr_count") {
6387 if (
ISA.Major < 10 ||
ISA.Major >= 12)
6388 return Error(IDRange.
Start,
"directive requires gfx10 or gfx11",
6390 SharedVGPRCount = Val;
6392 COMPUTE_PGM_RSRC3_GFX10_GFX11_SHARED_VGPR_COUNT, ExprVal,
6394 }
else if (ID ==
".amdhsa_inst_pref_size") {
6396 return Error(IDRange.
Start,
"directive requires gfx11+", IDRange);
6397 if (
ISA.Major == 11) {
6399 COMPUTE_PGM_RSRC3_GFX11_INST_PREF_SIZE, ExprVal,
6403 COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, ExprVal,
6406 }
else if (ID ==
".amdhsa_exception_fp_ieee_invalid_op") {
6409 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION,
6411 }
else if (ID ==
".amdhsa_exception_fp_denorm_src") {
6413 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
6415 }
else if (ID ==
".amdhsa_exception_fp_ieee_div_zero") {
6418 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO,
6420 }
else if (ID ==
".amdhsa_exception_fp_ieee_overflow") {
6422 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
6424 }
else if (ID ==
".amdhsa_exception_fp_ieee_underflow") {
6426 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
6428 }
else if (ID ==
".amdhsa_exception_fp_ieee_inexact") {
6430 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
6432 }
else if (ID ==
".amdhsa_exception_int_div_zero") {
6434 COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
6436 }
else if (ID ==
".amdhsa_round_robin_scheduling") {
6438 return Error(IDRange.
Start,
"directive requires gfx12+", IDRange);
6440 COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, ExprVal,
6443 return Error(IDRange.
Start,
"unknown .amdhsa_kernel directive", IDRange);
6446#undef PARSE_BITS_ENTRY
6449 if (!Seen.
contains(
".amdhsa_next_free_vgpr"))
6450 return TokError(
".amdhsa_next_free_vgpr directive is required");
6452 if (!Seen.
contains(
".amdhsa_next_free_sgpr"))
6453 return TokError(
".amdhsa_next_free_sgpr directive is required");
6455 unsigned UserSGPRCount = ExplicitUserSGPRCount.value_or(ImpliedUserSGPRCount);
6457 return TokError(
"too many user SGPRs enabled, found " +
6458 Twine(UserSGPRCount) +
", but only " +
6464 if (PreloadLength) {
6470 const MCExpr *VGPRBlocks;
6471 const MCExpr *SGPRBlocks;
6472 if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
6473 getTargetStreamer().getTargetID()->isXnackOnOrAny(),
6474 EnableWavefrontSize32, NextFreeVGPR, VGPRRange,
6475 NextFreeSGPR, SGPRRange, VGPRBlocks, SGPRBlocks))
6478 int64_t EvaluatedVGPRBlocks;
6479 bool VGPRBlocksEvaluatable =
6480 VGPRBlocks->evaluateAsAbsolute(EvaluatedVGPRBlocks);
6481 if (VGPRBlocksEvaluatable &&
6483 static_cast<uint64_t>(EvaluatedVGPRBlocks))) {
6484 return OutOfRangeError(VGPRRange);
6488 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT,
6489 COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT,
getContext());
6491 int64_t EvaluatedSGPRBlocks;
6492 if (SGPRBlocks->evaluateAsAbsolute(EvaluatedSGPRBlocks) &&
6494 static_cast<uint64_t>(EvaluatedSGPRBlocks)))
6495 return OutOfRangeError(SGPRRange);
6498 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT,
6499 COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
getContext());
6501 if (ExplicitUserSGPRCount && ImpliedUserSGPRCount > *ExplicitUserSGPRCount)
6502 return TokError(
"amdgpu_user_sgpr_count smaller than implied by "
6503 "enabled user SGPRs");
6509 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT_SHIFT,
6510 COMPUTE_PGM_RSRC2_GFX125_USER_SGPR_COUNT,
getContext());
6515 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT_SHIFT,
6516 COMPUTE_PGM_RSRC2_GFX6_GFX120_USER_SGPR_COUNT,
getContext());
6521 return TokError(
"Kernarg size should be resolvable");
6523 if (PreloadLength && kernarg_size &&
6524 (PreloadLength * 4 + PreloadOffset * 4 > kernarg_size))
6525 return TokError(
"Kernarg preload length + offset is larger than the "
6526 "kernarg segment size");
6529 if (!Seen.
contains(
".amdhsa_accum_offset"))
6530 return TokError(
".amdhsa_accum_offset directive is required");
6531 int64_t EvaluatedAccum;
6532 bool AccumEvaluatable = AccumOffset->evaluateAsAbsolute(EvaluatedAccum);
6533 uint64_t UEvaluatedAccum = EvaluatedAccum;
6534 if (AccumEvaluatable &&
6535 (UEvaluatedAccum < 4 || UEvaluatedAccum > 256 || (UEvaluatedAccum & 3)))
6536 return TokError(
"accum_offset should be in range [4..256] in "
6539 int64_t EvaluatedNumVGPR;
6540 if (NextFreeVGPR->evaluateAsAbsolute(EvaluatedNumVGPR) &&
6544 return TokError(
"accum_offset exceeds total VGPR allocation");
6550 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT,
6551 COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,
6557 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT_SHIFT,
6558 COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT,
6561 if (
ISA.Major >= 10 &&
ISA.Major < 12) {
6563 if (SharedVGPRCount && EnableWavefrontSize32 && *EnableWavefrontSize32) {
6564 return TokError(
"shared_vgpr_count directive not valid on "
6565 "wavefront size 32");
6568 if (VGPRBlocksEvaluatable &&
6569 (SharedVGPRCount * 2 +
static_cast<uint64_t>(EvaluatedVGPRBlocks) >
6571 return TokError(
"shared_vgpr_count*2 + "
6572 "compute_pgm_rsrc1.GRANULATED_WORKITEM_VGPR_COUNT cannot "
6577 emitTargetDirective();
6578 getTargetStreamer().EmitAmdhsaKernelDescriptor(getSTI(), KernelName, KD,
6579 NextFreeVGPR, NextFreeSGPR,
6580 ReserveVCC, ReserveFlatScr);
6584bool AMDGPUAsmParser::ParseDirectiveAMDHSACodeObjectVersion() {
6586 if (ParseAsAbsoluteExpression(
Version))
6589 getTargetStreamer().EmitDirectiveAMDHSACodeObjectVersion(
Version);
6590 emitTargetDirective();
6594bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
6595 AMDGPUMCKernelCodeT &
C) {
6598 if (ID ==
"max_scratch_backing_memory_byte_size") {
6599 Parser.eatToEndOfStatement();
6603 SmallString<40> ErrStr;
6604 raw_svector_ostream Err(ErrStr);
6605 if (!
C.ParseKernelCodeT(ID, getParser(), Err)) {
6606 return TokError(Err.
str());
6610 if (ID ==
"enable_wavefront_size32") {
6613 return TokError(
"enable_wavefront_size32=1 is only allowed on GFX10+");
6615 return TokError(
"enable_wavefront_size32=1 requires +WavefrontSize32");
6618 return TokError(
"enable_wavefront_size32=0 requires +WavefrontSize64");
6622 if (ID ==
"wavefront_size") {
6623 if (
C.wavefront_size == 5) {
6625 return TokError(
"wavefront_size=5 is only allowed on GFX10+");
6627 return TokError(
"wavefront_size=5 requires +WavefrontSize32");
6628 }
else if (
C.wavefront_size == 6) {
6630 return TokError(
"wavefront_size=6 requires +WavefrontSize64");
6637bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
6638 AMDGPUMCKernelCodeT KernelCode;
6648 if (!parseId(ID,
"expected value identifier or .end_amd_kernel_code_t"))
6651 if (ID ==
".end_amd_kernel_code_t")
6654 if (ParseAMDKernelCodeTValue(ID, KernelCode))
6659 getTargetStreamer().EmitAMDKernelCodeT(KernelCode);
6664bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
6665 StringRef KernelName;
6666 if (!parseId(KernelName,
"expected symbol name"))
6669 getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
6676bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
6677 if (!getSTI().getTargetTriple().isAMDGCN()) {
6678 return Error(getLoc(),
6679 ".amd_amdgpu_isa directive is not available on non-amdgcn "
6683 StringRef TargetIDDirective = getLexer().getTok().getStringContents();
6685 std::optional<AMDGPU::TargetID> MaybeParsed =
6688 return Error(getParser().getTok().getLoc(),
6689 "malformed target id '" + TargetIDDirective +
"'");
6692 const Triple &
TT = getSTI().getTargetTriple();
6698 return Error(getParser().getTok().getLoc(),
6699 "target id '" + TargetIDDirective +
6700 "' specifies a processor that is not valid for subarch '" +
6701 TT.getArchName() +
"'");
6704 const std::optional<AMDGPU::TargetID> &CurrentTargetID =
6705 getTargetStreamer().getTargetID();
6708 const Triple &STITriple = getSTI().getTargetTriple();
6709 if (!DirectiveTriple.isCompatibleWith(STITriple)) {
6710 return Error(getParser().getTok().getLoc(),
6711 ".amd_amdgpu_isa " + Twine(ParsedTargetID.
toString()) +
6712 " is incompatible with " +
6713 Twine(CurrentTargetID->toString()));
6717 StringRef DirectiveProcessor =
6720 if (DirectiveISA != ISA) {
6721 return Error(getParser().getTok().getLoc(),
6722 ".amd_amdgpu_isa directive processor " +
6723 Twine(DirectiveProcessor) +
6724 " does not match the specified processor " +
6725 Twine(getSTI().
getCPU()));
6728 getTargetStreamer().EmitISAVersion();
6734bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
6737 std::string HSAMetadataString;
6742 if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
6743 return Error(getLoc(),
"invalid HSA metadata");
6750bool AMDGPUAsmParser::ParseToEndDirective(
const char *AssemblerDirectiveBegin,
6751 const char *AssemblerDirectiveEnd,
6752 std::string &CollectString) {
6754 raw_string_ostream CollectStream(CollectString);
6756 getLexer().setSkipSpace(
false);
6758 bool FoundEnd =
false;
6761 CollectStream << getTokenStr();
6765 if (trySkipId(AssemblerDirectiveEnd)) {
6770 CollectStream << Parser.parseStringToEndOfStatement()
6771 <<
getContext().getAsmInfo().getSeparatorString();
6773 Parser.eatToEndOfStatement();
6776 getLexer().setSkipSpace(
true);
6779 return TokError(Twine(
"expected directive ") +
6780 Twine(AssemblerDirectiveEnd) + Twine(
" not found"));
6787bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
6793 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6794 if (!PALMetadata->setFromString(
String))
6795 return Error(getLoc(),
"invalid PAL metadata");
6800bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
6803 Twine(
" directive is "
6804 "not available on non-amdpal OSes"))
6808 auto *PALMetadata = getTargetStreamer().getPALMetadata();
6809 PALMetadata->setLegacy();
6812 if (ParseAsAbsoluteExpression(
Key)) {
6813 return TokError(Twine(
"invalid value in ") +
6817 return TokError(Twine(
"expected an even number of values in ") +
6820 if (ParseAsAbsoluteExpression(
Value)) {
6821 return TokError(Twine(
"invalid value in ") +
6824 PALMetadata->setRegister(
Key,
Value);
6833bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
6834 if (getParser().checkForValidSection())
6838 SMLoc NameLoc = getLoc();
6839 if (getParser().parseIdentifier(Name))
6840 return TokError(
"expected identifier in directive");
6843 if (getParser().parseComma())
6849 SMLoc SizeLoc = getLoc();
6850 if (getParser().parseAbsoluteExpression(
Size))
6853 return Error(SizeLoc,
"size must be non-negative");
6854 if (
Size > LocalMemorySize)
6855 return Error(SizeLoc,
"size is too large");
6859 SMLoc AlignLoc = getLoc();
6860 if (getParser().parseAbsoluteExpression(Alignment))
6863 return Error(AlignLoc,
"alignment must be a power of two");
6868 if (Alignment >= 1u << 31)
6869 return Error(AlignLoc,
"alignment is too large");
6875 Symbol->redefineIfPossible();
6876 if (!
Symbol->isUndefined())
6877 return Error(NameLoc,
"invalid symbol redefinition");
6879 getTargetStreamer().emitAMDGPULDS(Symbol,
Size,
Align(Alignment));
6883bool AMDGPUAsmParser::ParseDirectiveAMDGPUInfo() {
6884 if (getParser().checkForValidSection())
6888 if (getParser().parseIdentifier(FuncName))
6889 return TokError(
"expected symbol name after .amdgpu_info");
6892 AMDGPU::InfoSectionData ParsedInfoData;
6893 AMDGPU::FuncInfo FI;
6895 bool HasScalarAttrs =
false;
6902 SMLoc IDLoc = getLoc();
6903 if (!parseId(ID,
"expected directive or .end_amdgpu_info"))
6906 if (ID ==
".end_amdgpu_info")
6914 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6916 if (Dir ==
"flags") {
6918 if (getParser().parseAbsoluteExpression(Val))
6921 FI.
UsesVCC = !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_VCC);
6923 !!(
Flags & AMDGPU::FuncInfoFlags::FUNC_USES_FLAT_SCRATCH);
6925 HasScalarAttrs =
true;
6926 }
else if (Dir ==
"num_sgpr") {
6928 if (getParser().parseAbsoluteExpression(Val))
6930 FI.
NumSGPR =
static_cast<uint32_t
>(Val);
6931 HasScalarAttrs =
true;
6932 }
else if (Dir ==
"num_vgpr") {
6934 if (getParser().parseAbsoluteExpression(Val))
6937 HasScalarAttrs =
true;
6938 }
else if (Dir ==
"num_agpr") {
6940 if (getParser().parseAbsoluteExpression(Val))
6943 HasScalarAttrs =
true;
6944 }
else if (Dir ==
"private_segment_size") {
6946 if (getParser().parseAbsoluteExpression(Val))
6949 HasScalarAttrs =
true;
6950 }
else if (Dir ==
"use") {
6952 if (getParser().parseIdentifier(ResName))
6953 return TokError(
"expected resource symbol for .amdgpu_use");
6954 ParsedInfoData.
Uses.push_back(
6955 {FuncSym,
getContext().getOrCreateSymbol(ResName)});
6956 }
else if (Dir ==
"call") {
6958 if (getParser().parseIdentifier(DstName))
6959 return TokError(
"expected callee symbol for .amdgpu_call");
6960 ParsedInfoData.
Calls.push_back(
6961 {FuncSym,
getContext().getOrCreateSymbol(DstName)});
6962 }
else if (Dir ==
"indirect_call") {
6964 if (getParser().parseEscapedString(TypeId))
6965 return TokError(
"expected type ID string for .amdgpu_indirect_call");
6966 ParsedInfoData.
IndirectCalls.push_back({FuncSym, std::move(TypeId)});
6967 }
else if (Dir ==
"typeid") {
6969 if (getParser().parseEscapedString(TypeId))
6970 return TokError(
"expected type ID string for .amdgpu_typeid");
6971 ParsedInfoData.
TypeIds.push_back({FuncSym, std::move(TypeId)});
6973 return Error(IDLoc,
"unknown .amdgpu_info directive '" + ID +
"'");
6978 ParsedInfoData.
Funcs.push_back(std::move(FI));
6980 AMDGPU::InfoSectionData &
Data = InfoData ? *InfoData : InfoData.emplace();
6981 for (AMDGPU::FuncInfo &Func : ParsedInfoData.
Funcs)
6982 Data.Funcs.push_back(std::move(Func));
6983 for (std::pair<MCSymbol *, MCSymbol *> &Use : ParsedInfoData.
Uses)
6984 Data.Uses.push_back(Use);
6985 for (std::pair<MCSymbol *, MCSymbol *> &
Call : ParsedInfoData.
Calls)
6987 for (std::pair<MCSymbol *, std::string> &
IndirectCall :
6990 for (std::pair<MCSymbol *, std::string> &TypeId : ParsedInfoData.
TypeIds)
6991 Data.TypeIds.push_back(std::move(TypeId));
6996void AMDGPUAsmParser::doBeforeLabelEmit(MCSymbol *Symbol, SMLoc IDLoc) {
7003void AMDGPUAsmParser::checkKernelPrologues() {
7004 if (getFeatureBits()[AMDGPU::FeatureRequiresInitialUnclausedVmem]) {
7005 static const unsigned Required[] = {S_MOV_B64_gfx12, V_NOP_e32_gfx12,
7006 GLOBAL_PREFETCH_B8_SADDR_gfx1250};
7007 for (
auto [Sym, Loc,
Offset] : OpcodeStreamSymbols) {
7008 if (!AMDHSAKernelSymbols.
contains(Sym))
7010 ArrayRef<unsigned> Prologue =
ArrayRef(OpcodeStream).drop_front(
Offset);
7011 if (!Prologue.
empty() && Prologue.
front() == S_SETREG_IMM32_B32_gfx12)
7015 "' does not begin with the required prologue "
7016 "sequence: s_mov_b64 followed by v_nop and "
7017 "global_prefetch_b8");
7021 OpcodeStream.
clear();
7022 OpcodeStreamSymbols.clear();
7023 AMDHSAKernelSymbols.
clear();
7026void AMDGPUAsmParser::onEndOfFile() {
7027 emitTargetDirective();
7028 checkKernelPrologues();
7030 getTargetStreamer().emitAMDGPUInfo(*InfoData);
7033bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
7034 StringRef IDVal = DirectiveID.
getString();
7037 if (IDVal ==
".amdhsa_kernel")
7038 return ParseDirectiveAMDHSAKernel();
7040 if (IDVal ==
".amdhsa_code_object_version")
7041 return ParseDirectiveAMDHSACodeObjectVersion();
7045 return ParseDirectiveHSAMetadata();
7047 if (IDVal ==
".amd_kernel_code_t")
7048 return ParseDirectiveAMDKernelCodeT();
7050 if (IDVal ==
".amdgpu_hsa_kernel")
7051 return ParseDirectiveAMDGPUHsaKernel();
7053 if (IDVal ==
".amd_amdgpu_isa")
7054 return ParseDirectiveISAVersion();
7058 Twine(
" directive is "
7059 "not available on non-amdhsa OSes"))
7064 if (IDVal ==
".amdgcn_target")
7065 return ParseDirectiveAMDGCNTarget();
7067 if (IDVal ==
".amdgpu_lds")
7068 return ParseDirectiveAMDGPULDS();
7070 if (IDVal ==
".amdgpu_info")
7071 return ParseDirectiveAMDGPUInfo();
7074 return ParseDirectivePALMetadataBegin();
7077 return ParseDirectivePALMetadata();
7082bool AMDGPUAsmParser::subtargetHasRegister(
const MCRegisterInfo &MRI,
7089 return hasSGPR104_SGPR105();
7092 case SRC_SHARED_BASE_LO:
7093 case SRC_SHARED_BASE:
7094 case SRC_SHARED_LIMIT_LO:
7095 case SRC_SHARED_LIMIT:
7097 case SRC_PRIVATE_BASE_LO:
7098 case SRC_PRIVATE_BASE:
7099 case SRC_PRIVATE_LIMIT_LO:
7100 case SRC_PRIVATE_LIMIT:
7102 case SRC_FLAT_SCRATCH_BASE_LO:
7103 case SRC_FLAT_SCRATCH_BASE_HI:
7104 return hasGloballyAddressableScratch();
7105 case SRC_POPS_EXITING_WAVE_ID:
7118 getTargetStreamer().getTargetID()->isXnackSupported();
7148 return hasSGPR102_SGPR103();
7156 ParseStatus Res = parseVOPD(
Operands);
7161 Res = MatchOperandParserImpl(
Operands, Mnemonic);
7173 SMLoc LBraceLoc = getLoc();
7178 auto Loc = getLoc();
7181 Error(Loc,
"expected a register");
7185 RBraceLoc = getLoc();
7190 "expected a comma or a closing square bracket"))
7194 if (
Operands.size() - Prefix > 1) {
7196 AMDGPUOperand::CreateToken(
this,
"[", LBraceLoc));
7197 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"]", RBraceLoc));
7206StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
7208 setForcedEncodingSize(0);
7209 setForcedDPP(
false);
7210 setForcedSDWA(
false);
7212 if (
Name.consume_back(
"_e64_dpp")) {
7214 setForcedEncodingSize(64);
7217 if (
Name.consume_back(
"_e64")) {
7218 setForcedEncodingSize(64);
7221 if (
Name.consume_back(
"_e32")) {
7222 setForcedEncodingSize(32);
7225 if (
Name.consume_back(
"_dpp")) {
7229 if (
Name.consume_back(
"_sdwa")) {
7230 setForcedSDWA(
true);
7238 unsigned VariantID);
7244 Name = parseMnemonicSuffix(Name);
7250 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, NameLoc));
7252 bool IsMIMG = Name.starts_with(
"image_");
7255 OperandMode
Mode = OperandMode_Default;
7257 Mode = OperandMode_NSA;
7261 checkUnsupportedInstruction(Name, NameLoc);
7262 if (!Parser.hasPendingError()) {
7265 :
"not a valid operand.";
7285ParseStatus AMDGPUAsmParser::parseTokenOp(StringRef Name,
7288 if (!trySkipId(Name))
7291 Operands.push_back(AMDGPUOperand::CreateToken(
this, Name, S));
7295ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
const char *Prefix,
7304ParseStatus AMDGPUAsmParser::parseIntWithPrefix(
7306 std::function<
bool(int64_t &)> ConvertResult) {
7310 ParseStatus Res = parseIntWithPrefix(Prefix,
Value);
7314 if (ConvertResult && !ConvertResult(
Value)) {
7315 Error(S,
"invalid " + StringRef(Prefix) +
" value.");
7318 Operands.push_back(AMDGPUOperand::CreateImm(
this,
Value, S, ImmTy));
7322ParseStatus AMDGPUAsmParser::parseOperandArrayWithPrefix(
7324 bool (*ConvertResult)(int64_t &)) {
7333 const unsigned MaxSize = 4;
7337 for (
int I = 0;; ++
I) {
7339 SMLoc Loc = getLoc();
7343 if (
Op != 0 &&
Op != 1)
7344 return Error(Loc,
"invalid " + StringRef(Prefix) +
" value.");
7351 if (
I + 1 == MaxSize)
7352 return Error(getLoc(),
"expected a closing square bracket");
7358 Operands.push_back(AMDGPUOperand::CreateImm(
this, Val, S, ImmTy));
7362ParseStatus AMDGPUAsmParser::parseNamedBit(StringRef Name,
7364 AMDGPUOperand::ImmTy ImmTy,
7365 bool IgnoreNegative) {
7369 if (trySkipId(Name)) {
7371 }
else if (trySkipId(
"no", Name)) {
7380 return Error(S,
"r128 modifier is not supported on this GPU");
7381 if (Name ==
"a16" && !
hasA16())
7382 return Error(S,
"a16 modifier is not supported on this GPU");
7384 if (Bit == 0 && Name ==
"gds") {
7387 return Error(S,
"nogds is not allowed");
7390 if (
isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
7391 ImmTy = AMDGPUOperand::ImmTyR128A16;
7393 Operands.push_back(AMDGPUOperand::CreateImm(
this, Bit, S, ImmTy));
7397unsigned AMDGPUAsmParser::getCPolKind(StringRef Id, StringRef Mnemo,
7398 bool &Disabling)
const {
7399 Disabling =
Id.consume_front(
"no");
7402 return StringSwitch<unsigned>(Id)
7409 return StringSwitch<unsigned>(Id)
7419 SMLoc StringLoc = getLoc();
7421 int64_t CPolVal = 0;
7441 ResScope = parseScope(
Operands, Scope);
7454 if (trySkipId(
"nv")) {
7458 }
else if (trySkipId(
"no",
"nv")) {
7465 if (trySkipId(
"scale_offset")) {
7469 }
else if (trySkipId(
"no",
"scale_offset")) {
7482 Operands.push_back(AMDGPUOperand::CreateImm(
this, CPolVal, StringLoc,
7483 AMDGPUOperand::ImmTyCPol));
7488 SMLoc OpLoc = getLoc();
7489 unsigned Enabled = 0, Seen = 0;
7493 unsigned CPol = getCPolKind(getId(), Mnemo, Disabling);
7500 return Error(S,
"dlc modifier is not supported on this GPU");
7503 return Error(S,
"scc modifier is not supported on this GPU");
7506 return Error(S,
"duplicate cache policy modifier");
7518 AMDGPUOperand::CreateImm(
this,
Enabled, OpLoc, AMDGPUOperand::ImmTyCPol));
7527 ParseStatus Res = parseStringOrIntWithPrefix(
7528 Operands,
"scope", {
"SCOPE_CU",
"SCOPE_SE",
"SCOPE_DEV",
"SCOPE_SYS"},
7542 ParseStatus Res = parseStringWithPrefix(
"th",
Value, StringLoc);
7546 if (
Value ==
"TH_DEFAULT")
7548 else if (
Value ==
"TH_STORE_LU" ||
Value ==
"TH_LOAD_WB" ||
7549 Value ==
"TH_LOAD_NT_WB") {
7550 return Error(StringLoc,
"invalid th value");
7551 }
else if (
Value.consume_front(
"TH_ATOMIC_")) {
7553 }
else if (
Value.consume_front(
"TH_LOAD_")) {
7555 }
else if (
Value.consume_front(
"TH_STORE_")) {
7558 return Error(StringLoc,
"invalid th value");
7561 if (
Value ==
"BYPASS")
7566 TH |= StringSwitch<int64_t>(
Value)
7576 .Default(0xffffffff);
7578 TH |= StringSwitch<int64_t>(
Value)
7589 .Default(0xffffffff);
7592 if (TH == 0xffffffff)
7593 return Error(StringLoc,
"invalid th value");
7600 AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx,
7601 AMDGPUOperand::ImmTy ImmT, int64_t
Default = 0,
7602 std::optional<unsigned> InsertAt = std::nullopt) {
7603 auto i = OptionalIdx.find(ImmT);
7604 if (i != OptionalIdx.end()) {
7605 unsigned Idx = i->second;
7606 const AMDGPUOperand &
Op =
7607 static_cast<const AMDGPUOperand &
>(*
Operands[
Idx]);
7611 Op.addImmOperands(Inst, 1);
7613 if (InsertAt.has_value())
7620ParseStatus AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
7626 StringLoc = getLoc();
7631ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7637 SMLoc StringLoc = getLoc();
7641 Value = getTokenStr();
7645 if (
Value == Ids[IntVal])
7650 if (IntVal < 0 || IntVal >= (int64_t)Ids.
size())
7651 return Error(StringLoc,
"invalid " + Twine(Name) +
" value");
7656ParseStatus AMDGPUAsmParser::parseStringOrIntWithPrefix(
7658 AMDGPUOperand::ImmTy
Type) {
7662 ParseStatus Res = parseStringOrIntWithPrefix(
Operands, Name, Ids, IntVal);
7664 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S,
Type));
7673bool AMDGPUAsmParser::tryParseFmt(
const char *Pref, int64_t MaxVal,
7676 SMLoc Loc = getLoc();
7678 auto Res = parseIntWithPrefix(Pref, Val);
7684 if (Val < 0 || Val > MaxVal) {
7685 Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7694 AMDGPUOperand::ImmTy ImmTy) {
7695 const char *Pref =
"index_key";
7697 SMLoc Loc = getLoc();
7698 auto Res = parseIntWithPrefix(Pref, ImmVal);
7702 if ((ImmTy == AMDGPUOperand::ImmTyIndexKey16bit ||
7703 ImmTy == AMDGPUOperand::ImmTyIndexKey32bit) &&
7704 (ImmVal < 0 || ImmVal > 1))
7705 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7707 if (ImmTy == AMDGPUOperand::ImmTyIndexKey8bit && (ImmVal < 0 || ImmVal > 3))
7708 return Error(Loc, Twine(
"out of range ", StringRef(Pref)));
7710 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc, ImmTy));
7715 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey8bit);
7719 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey16bit);
7723 return tryParseIndexKey(
Operands, AMDGPUOperand::ImmTyIndexKey32bit);
7728 AMDGPUOperand::ImmTy
Type) {
7734 return tryParseMatrixFMT(
Operands,
"matrix_a_fmt",
7735 AMDGPUOperand::ImmTyMatrixAFMT);
7739 return tryParseMatrixFMT(
Operands,
"matrix_b_fmt",
7740 AMDGPUOperand::ImmTyMatrixBFMT);
7745 AMDGPUOperand::ImmTy
Type) {
7751 return tryParseMatrixScale(
Operands,
"matrix_a_scale",
7752 AMDGPUOperand::ImmTyMatrixAScale);
7756 return tryParseMatrixScale(
Operands,
"matrix_b_scale",
7757 AMDGPUOperand::ImmTyMatrixBScale);
7762 AMDGPUOperand::ImmTy
Type) {
7768 return tryParseMatrixScaleFmt(
Operands,
"matrix_a_scale_fmt",
7769 AMDGPUOperand::ImmTyMatrixAScaleFmt);
7773 return tryParseMatrixScaleFmt(
Operands,
"matrix_b_scale_fmt",
7774 AMDGPUOperand::ImmTyMatrixBScaleFmt);
7779ParseStatus AMDGPUAsmParser::parseDfmtNfmt(int64_t &
Format) {
7780 using namespace llvm::AMDGPU::MTBUFFormat;
7786 for (
int I = 0;
I < 2; ++
I) {
7787 if (Dfmt == DFMT_UNDEF && !tryParseFmt(
"dfmt", DFMT_MAX, Dfmt))
7790 if (Nfmt == NFMT_UNDEF && !tryParseFmt(
"nfmt", NFMT_MAX, Nfmt))
7795 if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
7801 if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
7804 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7805 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7811ParseStatus AMDGPUAsmParser::parseUfmt(int64_t &
Format) {
7812 using namespace llvm::AMDGPU::MTBUFFormat;
7816 if (!tryParseFmt(
"format", UFMT_MAX, Fmt))
7819 if (Fmt == UFMT_UNDEF)
7826bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt,
7827 StringRef FormatStr, SMLoc Loc) {
7828 using namespace llvm::AMDGPU::MTBUFFormat;
7832 if (
Format != DFMT_UNDEF) {
7838 if (
Format != NFMT_UNDEF) {
7843 Error(Loc,
"unsupported format");
7847ParseStatus AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
7850 using namespace llvm::AMDGPU::MTBUFFormat;
7854 if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
7859 SMLoc Loc = getLoc();
7860 if (!parseId(Str,
"expected a format string") ||
7861 !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc))
7863 if (Dfmt == DFMT_UNDEF)
7864 return Error(Loc,
"duplicate numeric format");
7865 if (Nfmt == NFMT_UNDEF)
7866 return Error(Loc,
"duplicate data format");
7869 Dfmt = (Dfmt ==
DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
7870 Nfmt = (Nfmt ==
NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;
7874 if (Ufmt == UFMT_UNDEF)
7875 return Error(FormatLoc,
"unsupported format");
7884ParseStatus AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
7887 using namespace llvm::AMDGPU::MTBUFFormat;
7890 if (Id == UFMT_UNDEF)
7894 return Error(Loc,
"unified format is not supported on this GPU");
7900ParseStatus AMDGPUAsmParser::parseNumericFormat(int64_t &
Format) {
7901 using namespace llvm::AMDGPU::MTBUFFormat;
7902 SMLoc Loc = getLoc();
7907 return Error(Loc,
"out of range format");
7912ParseStatus AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &
Format) {
7913 using namespace llvm::AMDGPU::MTBUFFormat;
7919 StringRef FormatStr;
7920 SMLoc Loc = getLoc();
7921 if (!parseId(FormatStr,
"expected a format string"))
7924 auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc,
Format);
7926 Res = parseSymbolicSplitFormat(FormatStr, Loc,
Format);
7936 return parseNumericFormat(
Format);
7940 using namespace llvm::AMDGPU::MTBUFFormat;
7944 SMLoc Loc = getLoc();
7954 AMDGPUOperand::CreateImm(
this,
Format, Loc, AMDGPUOperand::ImmTyFORMAT));
7973 Res = parseSymbolicOrNumericFormat(
Format);
7978 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
Size - 2]);
7979 assert(
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT);
7986 return Error(getLoc(),
"duplicate format");
7992 parseIntWithPrefix(
"offset",
Operands, AMDGPUOperand::ImmTyOffset);
7994 Res = parseIntWithPrefix(
"inst_offset",
Operands,
7995 AMDGPUOperand::ImmTyInstOffset);
8002 parseNamedBit(
"r128",
Operands, AMDGPUOperand::ImmTyR128A16);
8004 Res = parseNamedBit(
"a16",
Operands, AMDGPUOperand::ImmTyA16);
8010 parseIntWithPrefix(
"blgp",
Operands, AMDGPUOperand::ImmTyBLGP);
8013 parseOperandArrayWithPrefix(
"neg",
Operands, AMDGPUOperand::ImmTyBLGP);
8023 OptionalImmIndexMap OptionalIdx;
8025 unsigned OperandIdx[4];
8026 unsigned EnMask = 0;
8029 for (
unsigned i = 1, e =
Operands.size(); i != e; ++i) {
8030 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8035 OperandIdx[SrcIdx] = Inst.
size();
8036 Op.addRegOperands(Inst, 1);
8043 OperandIdx[SrcIdx] = Inst.
size();
8049 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
8050 Op.addImmOperands(Inst, 1);
8054 if (
Op.isToken() && (
Op.getToken() ==
"done" ||
Op.getToken() ==
"row_en"))
8058 OptionalIdx[
Op.getImmTy()] = i;
8064 if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
8071 for (
auto i = 0; i < SrcIdx; ++i) {
8073 EnMask |= Compr ? (0x3 << i * 2) : (0x1 << i);
8079 AMDGPUOperand::ImmTyExpCompr);
8089 int64_t CntVal,
bool Saturate,
8095 IntVal =
encode(ISA, IntVal, CntVal);
8096 if (CntVal !=
decode(ISA, IntVal)) {
8098 IntVal =
encode(ISA, IntVal, -1);
8106bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
8108 SMLoc CntLoc = getLoc();
8109 StringRef CntName = getTokenStr();
8116 SMLoc ValLoc = getLoc();
8123 if (CntName ==
"vmcnt" || CntName ==
"vmcnt_sat") {
8125 }
else if (CntName ==
"expcnt" || CntName ==
"expcnt_sat") {
8127 }
else if (CntName ==
"lgkmcnt" || CntName ==
"lgkmcnt_sat") {
8130 Error(CntLoc,
"invalid counter name " + CntName);
8135 Error(ValLoc,
"too large value for " + CntName);
8144 Error(getLoc(),
"expected a counter name");
8158 if (!parseCnt(Waitcnt))
8166 Operands.push_back(AMDGPUOperand::CreateImm(
this, Waitcnt, S));
8170bool AMDGPUAsmParser::parseDelay(int64_t &Delay) {
8171 SMLoc FieldLoc = getLoc();
8172 StringRef FieldName = getTokenStr();
8177 SMLoc ValueLoc = getLoc();
8184 if (FieldName ==
"instid0") {
8186 }
else if (FieldName ==
"instskip") {
8188 }
else if (FieldName ==
"instid1") {
8191 Error(FieldLoc,
"invalid field name " + FieldName);
8210 .Case(
"VALU_DEP_1", 1)
8211 .Case(
"VALU_DEP_2", 2)
8212 .Case(
"VALU_DEP_3", 3)
8213 .Case(
"VALU_DEP_4", 4)
8214 .Case(
"TRANS32_DEP_1", 5)
8215 .Case(
"TRANS32_DEP_2", 6)
8216 .Case(
"TRANS32_DEP_3", 7)
8217 .Case(
"FMA_ACCUM_CYCLE_1", 8)
8218 .Case(
"SALU_CYCLE_1", 9)
8219 .Case(
"SALU_CYCLE_2", 10)
8220 .Case(
"SALU_CYCLE_3", 11)
8228 Delay |=
Value << Shift;
8238 if (!parseDelay(Delay))
8246 Operands.push_back(AMDGPUOperand::CreateImm(
this, Delay, S));
8250bool AMDGPUOperand::isSWaitCnt()
const {
return isImm(); }
8252bool AMDGPUOperand::isSDelayALU()
const {
return isImm(); }
8258void AMDGPUAsmParser::depCtrError(SMLoc Loc,
int ErrorId,
8259 StringRef DepCtrName) {
8262 Error(Loc, Twine(
"invalid counter name ", DepCtrName));
8265 Error(Loc, Twine(DepCtrName,
" is not supported on this GPU"));
8268 Error(Loc, Twine(
"duplicate counter name ", DepCtrName));
8271 Error(Loc, Twine(
"invalid value for ", DepCtrName));
8278bool AMDGPUAsmParser::parseDepCtr(int64_t &DepCtr,
unsigned &UsedOprMask) {
8280 using namespace llvm::AMDGPU::DepCtr;
8282 SMLoc DepCtrLoc = getLoc();
8283 StringRef DepCtrName = getTokenStr();
8293 unsigned PrevOprMask = UsedOprMask;
8294 int CntVal =
encodeDepCtr(DepCtrName, ExprVal, UsedOprMask, getSTI());
8297 depCtrError(DepCtrLoc, CntVal, DepCtrName);
8306 Error(getLoc(),
"expected a counter name");
8311 int64_t CntValMask = PrevOprMask ^ UsedOprMask;
8312 DepCtr = (DepCtr & ~CntValMask) | CntVal;
8317 using namespace llvm::AMDGPU::DepCtr;
8320 SMLoc Loc = getLoc();
8323 unsigned UsedOprMask = 0;
8325 if (!parseDepCtr(DepCtr, UsedOprMask))
8333 Operands.push_back(AMDGPUOperand::CreateImm(
this, DepCtr, Loc));
8337bool AMDGPUOperand::isDepCtr()
const {
return isS16Imm(); }
8343ParseStatus AMDGPUAsmParser::parseHwregFunc(OperandInfoTy &HwReg,
8345 OperandInfoTy &Width) {
8346 using namespace llvm::AMDGPU::Hwreg;
8352 HwReg.Loc = getLoc();
8355 HwReg.IsSymbolic =
true;
8357 }
else if (!
parseExpr(HwReg.Val,
"a register name")) {
8365 if (!skipToken(
AsmToken::Comma,
"expected a comma or a closing parenthesis"))
8375 Width.Loc = getLoc();
8384 using namespace llvm::AMDGPU::Hwreg;
8387 SMLoc Loc = getLoc();
8389 StructuredOpField HwReg(
"id",
"hardware register", HwregId::Width,
8391 StructuredOpField
Offset(
"offset",
"bit offset", HwregOffset::Width,
8392 HwregOffset::Default);
8393 struct : StructuredOpField {
8394 using StructuredOpField::StructuredOpField;
8395 bool validate(AMDGPUAsmParser &Parser)
const override {
8397 return Error(Parser,
"only values from 1 to 32 are legal");
8400 } Width(
"size",
"bitfield width", HwregSize::Width, HwregSize::Default);
8401 ParseStatus Res = parseStructuredOpFields({&HwReg, &
Offset, &Width});
8404 Res = parseHwregFunc(HwReg,
Offset, Width);
8407 if (!validateStructuredOpFields({&HwReg, &
Offset, &Width}))
8409 ImmVal = HwregEncoding::encode(HwReg.Val,
Offset.Val, Width.Val);
8413 parseExpr(ImmVal,
"a hwreg macro, structured immediate"))
8420 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8422 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
8426bool AMDGPUOperand::isHwreg()
const {
return isImmTy(ImmTyHwreg); }
8432bool AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &
Msg, OperandInfoTy &
Op,
8433 OperandInfoTy &Stream) {
8434 using namespace llvm::AMDGPU::SendMsg;
8439 Msg.IsSymbolic =
true;
8446 Op.IsDefined =
true;
8452 }
else if (!
parseExpr(
Op.Val,
"an operation name")) {
8457 Stream.IsDefined =
true;
8458 Stream.Loc = getLoc();
8467bool AMDGPUAsmParser::validateSendMsg(
const OperandInfoTy &
Msg,
8468 const OperandInfoTy &
Op,
8469 const OperandInfoTy &Stream) {
8470 using namespace llvm::AMDGPU::SendMsg;
8479 Error(
Msg.Loc,
"specified message id is not supported on this GPU");
8484 Error(
Msg.Loc,
"invalid message id");
8490 Error(
Op.Loc,
"message does not support operations");
8492 Error(
Msg.Loc,
"missing message operation");
8498 Error(
Op.Loc,
"specified operation id is not supported on this GPU");
8500 Error(
Op.Loc,
"invalid operation id");
8505 Error(Stream.Loc,
"message operation does not support streams");
8509 Error(Stream.Loc,
"invalid message stream id");
8516 using namespace llvm::AMDGPU::SendMsg;
8519 SMLoc Loc = getLoc();
8523 OperandInfoTy
Op(OP_NONE_);
8524 OperandInfoTy Stream(STREAM_ID_NONE_);
8525 if (parseSendMsgBody(
Msg,
Op, Stream) && validateSendMsg(
Msg,
Op, Stream)) {
8530 }
else if (
parseExpr(ImmVal,
"a sendmsg macro")) {
8532 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8538 AMDGPUOperand::CreateImm(
this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
8542bool AMDGPUOperand::isSendMsg()
const {
return isImmTy(ImmTySendMsg); }
8545 using namespace llvm::AMDGPU::WaitEvent;
8547 SMLoc Loc = getLoc();
8550 StructuredOpField DontWaitExportReady(
"dont_wait_export_ready",
"bit value",
8552 StructuredOpField ExportReady(
"export_ready",
"bit value", 1, 0);
8554 StructuredOpField *TargetBitfield =
8555 isGFX11() ? &DontWaitExportReady : &ExportReady;
8557 ParseStatus Res = parseStructuredOpFields({TargetBitfield});
8561 if (!validateStructuredOpFields({TargetBitfield}))
8563 ImmVal = TargetBitfield->Val;
8570 return Error(Loc,
"invalid immediate: only 16-bit values are legal");
8572 Operands.push_back(AMDGPUOperand::CreateImm(
this, ImmVal, Loc,
8573 AMDGPUOperand::ImmTyWaitEvent));
8577bool AMDGPUOperand::isWaitEvent()
const {
return isImmTy(ImmTyWaitEvent); }
8590 int Slot = StringSwitch<int>(Str)
8597 return Error(S,
"invalid interpolation slot");
8600 AMDGPUOperand::CreateImm(
this, Slot, S, AMDGPUOperand::ImmTyInterpSlot));
8611 if (!Str.starts_with(
"attr"))
8612 return Error(S,
"invalid interpolation attribute");
8614 StringRef Chan = Str.take_back(2);
8615 int AttrChan = StringSwitch<int>(Chan)
8622 return Error(S,
"invalid or missing interpolation attribute channel");
8624 Str = Str.drop_back(2).drop_front(4);
8627 if (Str.getAsInteger(10, Attr))
8628 return Error(S,
"invalid or missing interpolation attribute number");
8631 return Error(S,
"out of bounds interpolation attribute number");
8636 AMDGPUOperand::CreateImm(
this, Attr, S, AMDGPUOperand::ImmTyInterpAttr));
8637 Operands.push_back(AMDGPUOperand::CreateImm(
8638 this, AttrChan, SChan, AMDGPUOperand::ImmTyInterpAttrChan));
8647 using namespace llvm::AMDGPU::Exp;
8657 return Error(S, (Id == ET_INVALID)
8658 ?
"invalid exp target"
8659 :
"exp target is not supported on this GPU");
8662 AMDGPUOperand::CreateImm(
this, Id, S, AMDGPUOperand::ImmTyExpTgt));
8670bool AMDGPUAsmParser::isId(
const AsmToken &Token,
const StringRef Id)
const {
8674bool AMDGPUAsmParser::isId(
const StringRef Id)
const {
8679 return getTokenKind() ==
Kind;
8682StringRef AMDGPUAsmParser::getId()
const {
8686bool AMDGPUAsmParser::trySkipId(
const StringRef Id) {
8694bool AMDGPUAsmParser::trySkipId(
const StringRef Pref,
const StringRef Id) {
8696 StringRef Tok = getTokenStr();
8705bool AMDGPUAsmParser::trySkipId(
const StringRef Id,
8707 if (isId(Id) && peekToken().is(Kind)) {
8716 if (isToken(Kind)) {
8724 const StringRef ErrMsg) {
8725 if (!trySkipToken(Kind)) {
8726 Error(getLoc(), ErrMsg);
8732bool AMDGPUAsmParser::parseExpr(int64_t &
Imm, StringRef Expected) {
8736 if (Parser.parseExpression(Expr))
8739 if (Expr->evaluateAsAbsolute(
Imm))
8742 if (Expected.empty()) {
8743 Error(S,
"expected absolute expression");
8746 Twine(
"expected ", Expected) + Twine(
" or an absolute expression"));
8755 if (Parser.parseExpression(Expr))
8759 if (Expr->evaluateAsAbsolute(IntVal)) {
8760 Operands.push_back(AMDGPUOperand::CreateImm(
this, IntVal, S));
8762 Operands.push_back(AMDGPUOperand::CreateExpr(
this, Expr, S));
8767bool AMDGPUAsmParser::parseString(StringRef &Val,
const StringRef ErrMsg) {
8769 Val =
getToken().getStringContents();
8773 Error(getLoc(), ErrMsg);
8777bool AMDGPUAsmParser::parseId(StringRef &Val,
const StringRef ErrMsg) {
8779 Val = getTokenStr();
8783 if (!ErrMsg.
empty())
8784 Error(getLoc(), ErrMsg);
8788AsmToken AMDGPUAsmParser::getToken()
const {
return Parser.getTok(); }
8790AsmToken AMDGPUAsmParser::peekToken(
bool ShouldSkipSpace) {
8793 : getLexer().peekTok(ShouldSkipSpace);
8797 auto TokCount = getLexer().peekTokens(Tokens);
8804 return getLexer().getKind();
8807SMLoc AMDGPUAsmParser::getLoc()
const {
return getToken().getLoc(); }
8809StringRef AMDGPUAsmParser::getTokenStr()
const {
8813void AMDGPUAsmParser::lex() { Parser.Lex(); }
8815const AMDGPUOperand &
8817 int MCOpIdx)
const {
8819 const AMDGPUOperand &TargetOp =
static_cast<AMDGPUOperand &
>(*Op);
8820 if (TargetOp.getMCOpIdx() == MCOpIdx)
8827 return ((AMDGPUOperand &)*
Operands[0]).getStartLoc();
8831SMLoc AMDGPUAsmParser::getLaterLoc(SMLoc a, SMLoc b) {
8836 int MCOpIdx)
const {
8837 return findMCOperand(
Operands, MCOpIdx).getStartLoc();
8840SMLoc AMDGPUAsmParser::getOperandLoc(
8841 std::function<
bool(
const AMDGPUOperand &)>
Test,
8843 for (
unsigned i =
Operands.size() - 1; i > 0; --i) {
8844 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
8846 return Op.getStartLoc();
8851SMLoc AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy
Type,
8853 auto Test = [=](
const AMDGPUOperand &
Op) {
return Op.isImmTy(
Type); };
8868 StringRef
Id = getTokenStr();
8869 SMLoc IdLoc = getLoc();
8875 find_if(Fields, [Id](StructuredOpField *
F) {
return F->Id ==
Id; });
8876 if (
I == Fields.
end())
8877 return Error(IdLoc,
"unknown field");
8878 if ((*I)->IsDefined)
8879 return Error(IdLoc,
"duplicate field");
8882 (*I)->Loc = getLoc();
8885 (*I)->IsDefined =
true;
8892bool AMDGPUAsmParser::validateStructuredOpFields(
8894 return all_of(Fields, [
this](
const StructuredOpField *
F) {
8895 return F->validate(*
this);
8905 const unsigned XorMask) {
8912bool AMDGPUAsmParser::parseSwizzleOperand(int64_t &
Op,
const unsigned MinVal,
8913 const unsigned MaxVal,
8914 const Twine &ErrMsg, SMLoc &Loc) {
8930bool AMDGPUAsmParser::parseSwizzleOperands(
const unsigned OpNum, int64_t *
Op,
8931 const unsigned MinVal,
8932 const unsigned MaxVal,
8933 const StringRef ErrMsg) {
8935 for (
unsigned i = 0; i < OpNum; ++i) {
8936 if (!parseSwizzleOperand(
Op[i], MinVal, MaxVal, ErrMsg, Loc))
8943bool AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &
Imm) {
8944 using namespace llvm::AMDGPU::Swizzle;
8947 if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
8948 "expected a 2-bit lane id")) {
8958bool AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &
Imm) {
8959 using namespace llvm::AMDGPU::Swizzle;
8965 if (!parseSwizzleOperand(GroupSize, 2, 32,
8966 "group size must be in the interval [2,32]", Loc)) {
8970 Error(Loc,
"group size must be a power of two");
8973 if (parseSwizzleOperand(LaneIdx, 0, GroupSize - 1,
8974 "lane id must be in the interval [0,group size - 1]",
8982bool AMDGPUAsmParser::parseSwizzleReverse(int64_t &
Imm) {
8983 using namespace llvm::AMDGPU::Swizzle;
8988 if (!parseSwizzleOperand(GroupSize, 2, 32,
8989 "group size must be in the interval [2,32]", Loc)) {
8993 Error(Loc,
"group size must be a power of two");
9001bool AMDGPUAsmParser::parseSwizzleSwap(int64_t &
Imm) {
9002 using namespace llvm::AMDGPU::Swizzle;
9007 if (!parseSwizzleOperand(GroupSize, 1, 16,
9008 "group size must be in the interval [1,16]", Loc)) {
9012 Error(Loc,
"group size must be a power of two");
9020bool AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &
Imm) {
9021 using namespace llvm::AMDGPU::Swizzle;
9028 SMLoc StrLoc = getLoc();
9029 if (!parseString(Ctl)) {
9032 if (Ctl.
size() != BITMASK_WIDTH) {
9033 Error(StrLoc,
"expected a 5-character mask");
9037 unsigned AndMask = 0;
9038 unsigned OrMask = 0;
9039 unsigned XorMask = 0;
9041 for (
size_t i = 0; i < Ctl.
size(); ++i) {
9045 Error(StrLoc,
"invalid mask");
9066bool AMDGPUAsmParser::parseSwizzleFFT(int64_t &
Imm) {
9067 using namespace llvm::AMDGPU::Swizzle;
9070 Error(getLoc(),
"FFT mode swizzle not supported on this GPU");
9076 if (!parseSwizzleOperand(Swizzle, 0, FFT_SWIZZLE_MAX,
9077 "FFT swizzle must be in the interval [0," +
9078 Twine(FFT_SWIZZLE_MAX) + Twine(
']'),
9086bool AMDGPUAsmParser::parseSwizzleRotate(int64_t &
Imm) {
9087 using namespace llvm::AMDGPU::Swizzle;
9090 Error(getLoc(),
"Rotate mode swizzle not supported on this GPU");
9097 if (!parseSwizzleOperand(
Direction, 0, 1,
9098 "direction must be 0 (left) or 1 (right)", Loc))
9102 if (!parseSwizzleOperand(
9103 RotateSize, 0, ROTATE_MAX_SIZE,
9104 "number of threads to rotate must be in the interval [0," +
9105 Twine(ROTATE_MAX_SIZE) + Twine(
']'),
9110 (RotateSize << ROTATE_SIZE_SHIFT);
9114bool AMDGPUAsmParser::parseSwizzleOffset(int64_t &
Imm) {
9116 SMLoc OffsetLoc = getLoc();
9122 Error(OffsetLoc,
"expected a 16-bit offset");
9128bool AMDGPUAsmParser::parseSwizzleMacro(int64_t &
Imm) {
9129 using namespace llvm::AMDGPU::Swizzle;
9133 SMLoc ModeLoc = getLoc();
9136 if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
9137 Ok = parseSwizzleQuadPerm(
Imm);
9138 }
else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
9139 Ok = parseSwizzleBitmaskPerm(
Imm);
9140 }
else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
9141 Ok = parseSwizzleBroadcast(
Imm);
9142 }
else if (trySkipId(IdSymbolic[ID_SWAP])) {
9143 Ok = parseSwizzleSwap(
Imm);
9144 }
else if (trySkipId(IdSymbolic[ID_REVERSE])) {
9145 Ok = parseSwizzleReverse(
Imm);
9146 }
else if (trySkipId(IdSymbolic[ID_FFT])) {
9147 Ok = parseSwizzleFFT(
Imm);
9148 }
else if (trySkipId(IdSymbolic[ID_ROTATE])) {
9149 Ok = parseSwizzleRotate(
Imm);
9151 Error(ModeLoc,
"expected a swizzle mode");
9154 return Ok && skipToken(
AsmToken::RParen,
"expected a closing parentheses");
9164 if (trySkipId(
"offset")) {
9168 if (trySkipId(
"swizzle")) {
9169 Ok = parseSwizzleMacro(
Imm);
9171 Ok = parseSwizzleOffset(
Imm);
9176 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTySwizzle));
9183bool AMDGPUOperand::isSwizzle()
const {
return isImmTy(ImmTySwizzle); }
9189int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
9191 using namespace llvm::AMDGPU::VGPRIndexMode;
9203 for (
unsigned ModeId = ID_MIN; ModeId <=
ID_MAX; ++ModeId) {
9204 if (trySkipId(IdSymbolic[ModeId])) {
9212 ?
"expected a VGPR index mode or a closing parenthesis"
9213 :
"expected a VGPR index mode");
9218 Error(S,
"duplicate VGPR index mode");
9226 "expected a comma or a closing parenthesis"))
9235 using namespace llvm::AMDGPU::VGPRIndexMode;
9241 Imm = parseGPRIdxMacro();
9245 if (getParser().parseAbsoluteExpression(
Imm))
9248 return Error(S,
"invalid immediate: only 4-bit values are legal");
9252 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
9256bool AMDGPUOperand::isGPRIdxMode()
const {
return isImmTy(ImmTyGprIdxMode); }
9267 if (isRegister() || isModifier())
9274 assert(Opr.isImm() || Opr.isExpr());
9275 SMLoc Loc = Opr.getStartLoc();
9279 if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
9280 Error(Loc,
"expected an absolute expression or a label");
9281 }
else if (Opr.isImm() && !Opr.isS16Imm()) {
9282 Error(Loc,
"expected a 16-bit signed jump offset");
9302 OptionalImmIndexMap OptionalIdx;
9303 unsigned FirstOperandIdx = 1;
9304 bool IsAtomicReturn =
false;
9310 for (
unsigned i = FirstOperandIdx, e =
Operands.size(); i != e; ++i) {
9311 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
9315 Op.addRegOperands(Inst, 1);
9319 if (IsAtomicReturn && i == FirstOperandIdx)
9320 Op.addRegOperands(Inst, 1);
9325 if (
Op.isImm() &&
Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
9326 Op.addImmOperands(Inst, 1);
9338 OptionalIdx[
Op.getImmTy()] = i;
9342 AMDGPUOperand::ImmTyOffset);
9358bool AMDGPUOperand::isSMRDOffset8()
const {
9362bool AMDGPUOperand::isSMEMOffset()
const {
9364 return isImmLiteral();
9367bool AMDGPUOperand::isSMRDLiteralOffset()
const {
9402bool AMDGPUAsmParser::convertDppBoundCtrl(int64_t &BoundCtrl) {
9403 if (BoundCtrl == 0 || BoundCtrl == 1) {
9411void AMDGPUAsmParser::onBeginOfFile() {
9412 if (!getParser().getStreamer().getTargetStreamer())
9415 if (!getTargetStreamer().getTargetID())
9416 getTargetStreamer().initializeTargetID(getSTI(),
9420void AMDGPUAsmParser::emitTargetDirective() {
9421 if (TargetDirectiveEmitted)
9423 TargetDirectiveEmitted =
true;
9425 if (!getParser().getStreamer().getTargetStreamer() ||
9430 getTargetStreamer().EmitDirectiveAMDGCNTarget();
9439bool AMDGPUAsmParser::parsePrimaryExpr(
const MCExpr *&Res, SMLoc &EndLoc) {
9443 StringRef TokenId = getTokenStr();
9444 AGVK VK = StringSwitch<AGVK>(TokenId)
9445 .Case(
"max", AGVK::AGVK_Max)
9446 .Case(
"min", AGVK::AGVK_Min)
9447 .Case(
"or", AGVK::AGVK_Or)
9448 .Case(
"extrasgprs", AGVK::AGVK_ExtraSGPRs)
9449 .Case(
"totalnumvgprs", AGVK::AGVK_TotalNumVGPRs)
9450 .Case(
"alignto", AGVK::AGVK_AlignTo)
9451 .Case(
"occupancy", AGVK::AGVK_Occupancy)
9452 .Case(
"instprefsize", AGVK::AGVK_InstPrefSize)
9453 .Default(AGVK::AGVK_None);
9462 if (Exprs.
empty()) {
9464 "empty " + Twine(TokenId) +
" expression");
9467 if (CommaCount + 1 != Exprs.
size()) {
9469 "mismatch of commas in " + Twine(TokenId) +
" expression");
9473 Expected && Exprs.
size() != Expected) {
9474 Error(
getToken().getLoc(), Twine(TokenId) +
" expression expects " +
9475 Twine(Expected) +
" operands");
9482 if (getParser().parseExpression(Expr, EndLoc))
9486 if (LastTokenWasComma)
9490 "unexpected token in " + Twine(TokenId) +
" expression");
9496 return getParser().parsePrimaryExpr(Res, EndLoc,
nullptr);
9500 StringRef
Name = getTokenStr();
9501 if (Name ==
"mul") {
9502 return parseIntWithPrefix(
"mul",
Operands, AMDGPUOperand::ImmTyOModSI,
9506 if (Name ==
"div") {
9507 return parseIntWithPrefix(
"div",
Operands, AMDGPUOperand::ImmTyOModSI,
9518 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9523 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9524 AMDGPU::OpName::src2};
9532 int DstIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst);
9537 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src0_modifiers);
9539 if (
DstOp.isReg() &&
9544 if ((OpSel & (1 << SrcNum)) != 0)
9550void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst,
9557 OptionalImmIndexMap &OptionalIdx) {
9558 cvtVOP3P(Inst,
Operands, OptionalIdx);
9567 &&
Desc.NumOperands > (OpNum + 1)
9569 &&
Desc.operands()[OpNum + 1].RegClass != -1
9571 &&
Desc.getOperandConstraint(OpNum + 1,
9575void AMDGPUAsmParser::cvtOpSelHelper(MCInst &Inst,
unsigned OpSel) {
9577 constexpr AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9578 AMDGPU::OpName::src2};
9579 constexpr AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9580 AMDGPU::OpName::src1_modifiers,
9581 AMDGPU::OpName::src2_modifiers};
9582 for (
int J = 0; J < 3; ++J) {
9583 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9589 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9592 if ((OpSel & (1 << J)) != 0)
9595 if (ModOps[J] == AMDGPU::OpName::src0_modifiers && (OpSel & (1 << 3)) != 0)
9602void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst,
9604 OptionalImmIndexMap OptionalIdx;
9609 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9610 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9614 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9616 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9617 }
else if (
Op.isInterpSlot() ||
Op.isInterpAttr() ||
9618 Op.isInterpAttrChan()) {
9620 }
else if (
Op.isImmModifier()) {
9621 OptionalIdx[
Op.getImmTy()] =
I;
9629 AMDGPUOperand::ImmTyHigh);
9633 AMDGPUOperand::ImmTyClamp);
9637 AMDGPUOperand::ImmTyOModSI);
9642 AMDGPUOperand::ImmTyOpSel);
9643 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9646 cvtOpSelHelper(Inst, OpSel);
9651 OptionalImmIndexMap OptionalIdx;
9656 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9657 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9661 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9663 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9664 }
else if (
Op.isImmModifier()) {
9665 OptionalIdx[
Op.getImmTy()] =
I;
9673 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9676 AMDGPUOperand::ImmTyOpSel);
9679 AMDGPUOperand::ImmTyWaitEXP);
9685 cvtOpSelHelper(Inst, OpSel);
9688void AMDGPUAsmParser::cvtScaledMFMA(MCInst &Inst,
9690 OptionalImmIndexMap OptionalIdx;
9693 int CbszOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::cbsz);
9697 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J)
9698 static_cast<AMDGPUOperand &
>(*
Operands[
I++]).addRegOperands(Inst, 1);
9701 AMDGPUOperand &
Op =
static_cast<AMDGPUOperand &
>(*
Operands[
I]);
9706 if (NumOperands == CbszOpIdx) {
9711 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9712 }
else if (
Op.isImmModifier()) {
9713 OptionalIdx[
Op.getImmTy()] =
I;
9715 Op.addRegOrImmOperands(Inst, 1);
9720 auto CbszIdx = OptionalIdx.find(AMDGPUOperand::ImmTyCBSZ);
9721 if (CbszIdx != OptionalIdx.end()) {
9722 int CbszVal = ((AMDGPUOperand &)*
Operands[CbszIdx->second]).
getImm();
9726 int BlgpOpIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::blgp);
9727 auto BlgpIdx = OptionalIdx.find(AMDGPUOperand::ImmTyBLGP);
9728 if (BlgpIdx != OptionalIdx.end()) {
9729 int BlgpVal = ((AMDGPUOperand &)*
Operands[BlgpIdx->second]).
getImm();
9740 auto OpselIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSel);
9741 if (OpselIdx != OptionalIdx.end()) {
9742 OpSel =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselIdx->second])
9746 unsigned OpSelHi = 0;
9747 auto OpselHiIdx = OptionalIdx.find(AMDGPUOperand::ImmTyOpSelHi);
9748 if (OpselHiIdx != OptionalIdx.end()) {
9749 OpSelHi =
static_cast<const AMDGPUOperand &
>(*
Operands[OpselHiIdx->second])
9752 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9753 AMDGPU::OpName::src1_modifiers};
9755 for (
unsigned J = 0; J < 2; ++J) {
9756 unsigned ModVal = 0;
9757 if (OpSel & (1 << J))
9759 if (OpSelHi & (1 << J))
9762 const int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9768 OptionalImmIndexMap &OptionalIdx) {
9773 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
9774 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
9778 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
9780 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
9781 }
else if (
Op.isImmModifier()) {
9782 OptionalIdx[
Op.getImmTy()] =
I;
9784 Op.addRegOrImmOperands(Inst, 1);
9790 AMDGPUOperand::ImmTyScaleSel);
9794 AMDGPUOperand::ImmTyClamp);
9800 AMDGPUOperand::ImmTyByteSel);
9805 AMDGPUOperand::ImmTyOModSI);
9812 auto *it = Inst.
begin();
9814 it, AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers));
9823 OptionalImmIndexMap OptionalIdx;
9824 cvtVOP3(Inst,
Operands, OptionalIdx);
9828 OptionalImmIndexMap &OptIdx) {
9833 if (
Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_F16_vi ||
9834 Opc == AMDGPU::V_CVT_SCALEF32_PK_FP4_BF16_vi ||
9835 Opc == AMDGPU::V_CVT_SR_BF8_F32_vi ||
9836 Opc == AMDGPU::V_CVT_SR_FP8_F32_vi ||
9837 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx11 ||
9838 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx11 ||
9839 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx12 ||
9840 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx12 ||
9841 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_gfx13 ||
9842 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_gfx13) {
9851 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
9852 if (VdstInIdx != -1 && VdstInIdx ==
static_cast<int>(Inst.
getNumOperands()))
9855 int BitOp3Idx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::bitop3);
9856 if (BitOp3Idx != -1) {
9863 int OpSelIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel);
9864 if (OpSelIdx != -1) {
9868 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::op_sel_hi);
9869 if (OpSelHiIdx != -1) {
9870 int DefaultVal =
IsPacked ? -1 : 0;
9876 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_fmt);
9877 if (MatrixAFMTIdx != -1) {
9879 AMDGPUOperand::ImmTyMatrixAFMT, 0);
9883 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_fmt);
9884 if (MatrixBFMTIdx != -1) {
9886 AMDGPUOperand::ImmTyMatrixBFMT, 0);
9889 int MatrixAScaleIdx =
9890 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale);
9891 if (MatrixAScaleIdx != -1) {
9893 AMDGPUOperand::ImmTyMatrixAScale, 0);
9896 int MatrixBScaleIdx =
9897 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale);
9898 if (MatrixBScaleIdx != -1) {
9900 AMDGPUOperand::ImmTyMatrixBScale, 0);
9903 int MatrixAScaleFmtIdx =
9904 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_a_scale_fmt);
9905 if (MatrixAScaleFmtIdx != -1) {
9907 AMDGPUOperand::ImmTyMatrixAScaleFmt, 0);
9910 int MatrixBScaleFmtIdx =
9911 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::matrix_b_scale_fmt);
9912 if (MatrixBScaleFmtIdx != -1) {
9914 AMDGPUOperand::ImmTyMatrixBScaleFmt, 0);
9919 AMDGPUOperand::ImmTyMatrixAReuse, 0);
9923 AMDGPUOperand::ImmTyMatrixBReuse, 0);
9925 int NegLoIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_lo);
9929 int NegHiIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::neg_hi);
9933 const AMDGPU::OpName
Ops[] = {AMDGPU::OpName::src0, AMDGPU::OpName::src1,
9934 AMDGPU::OpName::src2};
9935 const AMDGPU::OpName ModOps[] = {AMDGPU::OpName::src0_modifiers,
9936 AMDGPU::OpName::src1_modifiers,
9937 AMDGPU::OpName::src2_modifiers};
9940 unsigned OpSelHi = 0;
9947 if (OpSelHiIdx != -1)
9956 for (
int J = 0; J < 3; ++J) {
9957 int OpIdx = AMDGPU::getNamedOperandIdx(
Opc,
Ops[J]);
9961 int ModIdx = AMDGPU::getNamedOperandIdx(
Opc, ModOps[J]);
9971 uint32_t ModVal = 0;
9973 const MCOperand &SrcOp = Inst.
getOperand(OpIdx);
9974 if (SrcOp.
isReg() && getMRI()
9981 if ((OpSel & (1 << J)) != 0)
9985 if ((OpSelHi & (1 << J)) != 0)
9988 if ((NegLo & (1 << J)) != 0)
9991 if ((NegHi & (1 << J)) != 0)
9999 OptionalImmIndexMap OptIdx;
10005 unsigned i,
unsigned Opc,
10006 AMDGPU::OpName
OpName) {
10007 if (AMDGPU::getNamedOperandIdx(
Opc,
OpName) != -1)
10008 ((AMDGPUOperand &)*
Operands[i]).addRegOrImmWithFPInputModsOperands(Inst, 2);
10010 ((AMDGPUOperand &)*
Operands[i]).addRegOperands(Inst, 1);
10016 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10019 ((AMDGPUOperand &)*
Operands[1]).addRegOperands(Inst, 1);
10020 ((AMDGPUOperand &)*
Operands[4]).addRegOperands(Inst, 1);
10022 OptionalImmIndexMap OptIdx;
10023 for (
unsigned i = 5; i <
Operands.size(); ++i) {
10024 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[i]);
10025 OptIdx[
Op.getImmTy()] = i;
10030 AMDGPUOperand::ImmTyIndexKey8bit);
10034 AMDGPUOperand::ImmTyIndexKey16bit);
10038 AMDGPUOperand::ImmTyIndexKey32bit);
10055 SMLoc S = getLoc();
10058 Operands.push_back(AMDGPUOperand::CreateToken(
this,
"::", S));
10059 SMLoc OpYLoc = getLoc();
10062 Operands.push_back(AMDGPUOperand::CreateToken(
this, OpYName, OpYLoc));
10065 return Error(OpYLoc,
"expected a VOPDY instruction after ::");
10074 auto addOp = [&](uint16_t ParsedOprIdx) {
10075 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[ParsedOprIdx]);
10077 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10081 Op.addRegOperands(Inst, 1);
10084 if (
Op.isImm() ||
Op.isExpr()) {
10085 Op.addImmOperands(Inst, 1);
10097 addOp(InstInfo[CompIdx].getIndexOfDstInParsedOperands());
10101 const auto &CInfo = InstInfo[CompIdx];
10102 auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum();
10103 for (
unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx)
10104 addOp(CInfo.getIndexOfSrcInParsedOperands(CompSrcIdx));
10105 if (CInfo.hasSrc2Acc())
10106 addOp(CInfo.getIndexOfDstInParsedOperands());
10110 AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::bitop3);
10111 if (BitOp3Idx != -1) {
10112 OptionalImmIndexMap OptIdx;
10113 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands.back());
10115 OptIdx[
Op.getImmTy()] =
Operands.size() - 1;
10125bool AMDGPUOperand::isDPP8()
const {
return isImmTy(ImmTyDPP8); }
10127bool AMDGPUOperand::isDPPCtrl()
const {
10128 using namespace AMDGPU::DPP;
10130 bool result = isImm() && getImmTy() == ImmTyDppCtrl &&
isUInt<9>(
getImm());
10133 return (
Imm >= DppCtrl::QUAD_PERM_FIRST &&
10134 Imm <= DppCtrl::QUAD_PERM_LAST) ||
10135 (
Imm >= DppCtrl::ROW_SHL_FIRST &&
Imm <= DppCtrl::ROW_SHL_LAST) ||
10136 (
Imm >= DppCtrl::ROW_SHR_FIRST &&
Imm <= DppCtrl::ROW_SHR_LAST) ||
10137 (
Imm >= DppCtrl::ROW_ROR_FIRST &&
Imm <= DppCtrl::ROW_ROR_LAST) ||
10138 (
Imm == DppCtrl::WAVE_SHL1) || (
Imm == DppCtrl::WAVE_ROL1) ||
10139 (
Imm == DppCtrl::WAVE_SHR1) || (
Imm == DppCtrl::WAVE_ROR1) ||
10140 (
Imm == DppCtrl::ROW_MIRROR) || (
Imm == DppCtrl::ROW_HALF_MIRROR) ||
10141 (
Imm == DppCtrl::BCAST15) || (
Imm == DppCtrl::BCAST31) ||
10142 (
Imm >= DppCtrl::ROW_SHARE_FIRST &&
10143 Imm <= DppCtrl::ROW_SHARE_LAST) ||
10144 (
Imm >= DppCtrl::ROW_XMASK_FIRST &&
Imm <= DppCtrl::ROW_XMASK_LAST);
10153bool AMDGPUOperand::isBLGP()
const {
10157bool AMDGPUOperand::isS16Imm()
const {
10161bool AMDGPUOperand::isU16Imm()
const {
10169bool AMDGPUAsmParser::parseDimId(
unsigned &Encoding) {
10174 SMLoc Loc =
getToken().getEndLoc();
10175 Token = std::string(getTokenStr());
10177 if (getLoc() != Loc)
10182 if (!parseId(Suffix))
10186 StringRef DimId = Token;
10201 SMLoc S = getLoc();
10207 SMLoc Loc = getLoc();
10208 if (!parseDimId(Encoding))
10209 return Error(Loc,
"invalid dim value");
10212 AMDGPUOperand::CreateImm(
this, Encoding, S, AMDGPUOperand::ImmTyDim));
10221 SMLoc S = getLoc();
10230 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10233 for (
size_t i = 0; i < 8; ++i) {
10237 SMLoc Loc = getLoc();
10238 if (getParser().parseAbsoluteExpression(Sels[i]))
10240 if (0 > Sels[i] || 7 < Sels[i])
10241 return Error(Loc,
"expected a 3-bit value");
10244 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10248 for (
size_t i = 0; i < 8; ++i)
10249 DPP8 |= (Sels[i] << (i * 3));
10252 AMDGPUOperand::CreateImm(
this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
10256bool AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
10258 if (Ctrl ==
"row_newbcast")
10261 if (Ctrl ==
"row_share" || Ctrl ==
"row_xmask")
10264 if (Ctrl ==
"wave_shl" || Ctrl ==
"wave_shr" || Ctrl ==
"wave_rol" ||
10265 Ctrl ==
"wave_ror" || Ctrl ==
"row_bcast")
10268 return Ctrl ==
"row_mirror" ||
Ctrl ==
"row_half_mirror" ||
10269 Ctrl ==
"quad_perm" ||
Ctrl ==
"row_shl" ||
Ctrl ==
"row_shr" ||
10273int64_t AMDGPUAsmParser::parseDPPCtrlPerm() {
10276 if (!skipToken(
AsmToken::LBrac,
"expected an opening square bracket"))
10280 for (
int i = 0; i < 4; ++i) {
10285 SMLoc Loc = getLoc();
10286 if (getParser().parseAbsoluteExpression(Temp))
10288 if (Temp < 0 || Temp > 3) {
10289 Error(Loc,
"expected a 2-bit value");
10293 Val += (Temp << i * 2);
10296 if (!skipToken(
AsmToken::RBrac,
"expected a closing square bracket"))
10302int64_t AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
10303 using namespace AMDGPU::DPP;
10308 SMLoc Loc = getLoc();
10310 if (getParser().parseAbsoluteExpression(Val))
10313 struct DppCtrlCheck {
10319 DppCtrlCheck
Check =
10320 StringSwitch<DppCtrlCheck>(Ctrl)
10321 .Case(
"wave_shl", {DppCtrl::WAVE_SHL1, 1, 1})
10322 .Case(
"wave_rol", {DppCtrl::WAVE_ROL1, 1, 1})
10323 .Case(
"wave_shr", {DppCtrl::WAVE_SHR1, 1, 1})
10324 .Case(
"wave_ror", {DppCtrl::WAVE_ROR1, 1, 1})
10325 .Case(
"row_shl", {DppCtrl::ROW_SHL0, 1, 15})
10326 .Case(
"row_shr", {DppCtrl::ROW_SHR0, 1, 15})
10327 .Case(
"row_ror", {DppCtrl::ROW_ROR0, 1, 15})
10328 .Case(
"row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
10329 .Case(
"row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
10330 .Case(
"row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
10334 if (
Check.Ctrl == -1) {
10335 Valid = (
Ctrl ==
"row_bcast" && (Val == 15 || Val == 31));
10343 Error(Loc, Twine(
"invalid ", Ctrl) + Twine(
" value"));
10351 using namespace AMDGPU::DPP;
10354 !isSupportedDPPCtrl(getTokenStr(),
Operands))
10357 SMLoc S = getLoc();
10363 if (Ctrl ==
"row_mirror") {
10364 Val = DppCtrl::ROW_MIRROR;
10365 }
else if (Ctrl ==
"row_half_mirror") {
10366 Val = DppCtrl::ROW_HALF_MIRROR;
10369 if (Ctrl ==
"quad_perm") {
10370 Val = parseDPPCtrlPerm();
10372 Val = parseDPPCtrlSel(Ctrl);
10381 AMDGPUOperand::CreateImm(
this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
10387 OptionalImmIndexMap OptionalIdx;
10394 int OldIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::old);
10396 AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::src2_modifiers);
10397 bool IsMAC = OldIdx != -1 && Src2ModIdx != -1 &&
10401 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10402 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10406 int VdstInIdx = AMDGPU::getNamedOperandIdx(
Opc, AMDGPU::OpName::vdst_in);
10407 bool IsVOP3CvtSrDpp =
Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx12 ||
10408 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp8_gfx13 ||
10409 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx12 ||
10410 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp8_gfx13 ||
10411 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx12 ||
10412 Opc == AMDGPU::V_CVT_SR_BF8_F32_gfx12_e64_dpp_gfx13 ||
10413 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx12 ||
10414 Opc == AMDGPU::V_CVT_SR_FP8_F32_gfx12_e64_dpp_gfx13;
10420 if (OldIdx == NumOperands) {
10422 constexpr int DST_IDX = 0;
10424 }
else if (Src2ModIdx == NumOperands) {
10434 if (IsVOP3CvtSrDpp) {
10443 if (TiedTo != -1) {
10448 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10450 if (IsDPP8 &&
Op.isDppFI()) {
10453 Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
10454 }
else if (
Op.isReg()) {
10455 Op.addRegOperands(Inst, 1);
10456 }
else if (
Op.isImm() &&
10458 Op.addImmOperands(Inst, 1);
10459 }
else if (
Op.isImm()) {
10460 OptionalIdx[
Op.getImmTy()] =
I;
10468 AMDGPUOperand::ImmTyClamp);
10474 AMDGPUOperand::ImmTyByteSel);
10479 AMDGPUOperand::ImmTyOModSI);
10482 cvtVOP3P(Inst,
Operands, OptionalIdx);
10484 cvtVOP3OpSel(Inst,
Operands, OptionalIdx);
10487 AMDGPUOperand::ImmTyOpSel);
10492 AMDGPUOperand::ImmTyDPP8);
10493 using namespace llvm::AMDGPU::DPP;
10497 AMDGPUOperand::ImmTyDppCtrl, 0xe4);
10499 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10501 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10503 AMDGPUOperand::ImmTyDppBoundCtrl);
10507 AMDGPUOperand::ImmTyDppFI);
10513 OptionalImmIndexMap OptionalIdx;
10517 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10518 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10525 if (TiedTo != -1) {
10530 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10532 if (
Op.isReg() && validateVccOperand(
Op.getReg())) {
10540 Op.addImmOperands(Inst, 1);
10542 Op.addRegWithFPInputModsOperands(Inst, 2);
10543 }
else if (
Op.isDppFI()) {
10545 }
else if (
Op.isReg()) {
10546 Op.addRegOperands(Inst, 1);
10552 Op.addRegWithFPInputModsOperands(Inst, 2);
10553 }
else if (
Op.isReg()) {
10554 Op.addRegOperands(Inst, 1);
10555 }
else if (
Op.isDPPCtrl()) {
10556 Op.addImmOperands(Inst, 1);
10557 }
else if (
Op.isImm()) {
10559 OptionalIdx[
Op.getImmTy()] =
I;
10567 using namespace llvm::AMDGPU::DPP;
10571 AMDGPUOperand::ImmTyDppRowMask, 0xf);
10573 AMDGPUOperand::ImmTyDppBankMask, 0xf);
10575 AMDGPUOperand::ImmTyDppBoundCtrl);
10578 AMDGPUOperand::ImmTyDppFI);
10589 AMDGPUOperand::ImmTy
Type) {
10590 return parseStringOrIntWithPrefix(
10592 {
"BYTE_0",
"BYTE_1",
"BYTE_2",
"BYTE_3",
"WORD_0",
"WORD_1",
"DWORD"},
10597 return parseStringOrIntWithPrefix(
10598 Operands,
"dst_unused", {
"UNUSED_PAD",
"UNUSED_SEXT",
"UNUSED_PRESERVE"},
10599 AMDGPUOperand::ImmTySDWADstUnused);
10603 cvtSDWA(Inst,
Operands, SDWAInstType::VOP1);
10607 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2);
10610void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst,
10612 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
true,
true);
10615void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst,
10617 cvtSDWA(Inst,
Operands, SDWAInstType::VOP2,
false,
true);
10625 SDWAInstType BasicInstType,
bool SkipDstVcc,
10627 using namespace llvm::AMDGPU::SDWA;
10629 OptionalImmIndexMap OptionalIdx;
10630 bool SkipVcc = SkipDstVcc || SkipSrcVcc;
10631 bool SkippedVcc =
false;
10635 for (
unsigned J = 0; J <
Desc.getNumDefs(); ++J) {
10636 ((AMDGPUOperand &)*
Operands[
I++]).addRegOperands(Inst, 1);
10640 AMDGPUOperand &
Op = ((AMDGPUOperand &)*
Operands[
I]);
10641 if (SkipVcc && !SkippedVcc &&
Op.isReg() &&
10642 (
Op.getReg() == AMDGPU::VCC ||
Op.getReg() == AMDGPU::VCC_LO)) {
10648 if (BasicInstType == SDWAInstType::VOP2 &&
10654 if (BasicInstType == SDWAInstType::VOPC && Inst.
getNumOperands() == 0) {
10660 Op.addRegOrImmWithInputModsOperands(Inst, 2);
10661 }
else if (
Op.isImm()) {
10663 OptionalIdx[
Op.getImmTy()] =
I;
10667 SkippedVcc =
false;
10671 if (
Opc != AMDGPU::V_NOP_sdwa_gfx10 &&
Opc != AMDGPU::V_NOP_sdwa_gfx9 &&
10672 Opc != AMDGPU::V_NOP_sdwa_vi) {
10674 switch (BasicInstType) {
10675 case SDWAInstType::VOP1:
10678 AMDGPUOperand::ImmTyClamp, 0);
10682 AMDGPUOperand::ImmTyOModSI, 0);
10686 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10690 AMDGPUOperand::ImmTySDWADstUnused,
10691 DstUnused::UNUSED_PRESERVE);
10694 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10697 case SDWAInstType::VOP2:
10699 AMDGPUOperand::ImmTyClamp, 0);
10703 AMDGPUOperand::ImmTyOModSI, 0);
10706 AMDGPUOperand::ImmTySDWADstSel, SdwaSel::DWORD);
10708 AMDGPUOperand::ImmTySDWADstUnused,
10709 DstUnused::UNUSED_PRESERVE);
10711 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10713 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10716 case SDWAInstType::VOPC:
10719 AMDGPUOperand::ImmTyClamp, 0);
10721 AMDGPUOperand::ImmTySDWASrc0Sel, SdwaSel::DWORD);
10723 AMDGPUOperand::ImmTySDWASrc1Sel, SdwaSel::DWORD);
10730 if (Inst.
getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
10731 Inst.
getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
10732 auto *it = Inst.
begin();
10734 it, AMDGPU::getNamedOperandIdx(Inst.
getOpcode(), AMDGPU::OpName::src2));
10747#define GET_MATCHER_IMPLEMENTATION
10748#define GET_MNEMONIC_SPELL_CHECKER
10749#define GET_MNEMONIC_CHECKER
10750#include "AMDGPUGenAsmMatcher.inc"
10756 return parseTokenOp(
"addr64",
Operands);
10758 return parseNamedBit(
"done",
Operands, AMDGPUOperand::ImmTyDone,
true);
10760 return parseTokenOp(
"idxen",
Operands);
10762 return parseNamedBit(
"lds",
Operands, AMDGPUOperand::ImmTyLDS,
10765 return parseTokenOp(
"offen",
Operands);
10767 return parseTokenOp(
"off",
Operands);
10768 case MCK_row_95_en:
10769 return parseNamedBit(
"row_en",
Operands, AMDGPUOperand::ImmTyRowEn,
true);
10771 return parseNamedBit(
"gds",
Operands, AMDGPUOperand::ImmTyGDS);
10773 return parseNamedBit(
"tfe",
Operands, AMDGPUOperand::ImmTyTFE);
10775 return tryCustomParseOperand(
Operands, MCK);
10780unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &
Op,
10786 AMDGPUOperand &Operand = (AMDGPUOperand &)
Op;
10789 return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
10791 return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
10793 return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
10795 return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
10797 return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
10799 return Operand.isTFE() ? Match_Success : Match_InvalidOperand;
10801 return Operand.isDone() ? Match_Success : Match_InvalidOperand;
10802 case MCK_row_95_en:
10803 return Operand.isRowEn() ? Match_Success : Match_InvalidOperand;
10811 return Operand.isSSrc_b32() ? Match_Success : Match_InvalidOperand;
10813 return Operand.isSSrc_f32() ? Match_Success : Match_InvalidOperand;
10814 case MCK_SOPPBrTarget:
10815 return Operand.isSOPPBrTarget() ? Match_Success : Match_InvalidOperand;
10816 case MCK_VReg32OrOff:
10817 return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
10818 case MCK_InterpSlot:
10819 return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
10820 case MCK_InterpAttr:
10821 return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
10822 case MCK_InterpAttrChan:
10823 return Operand.isInterpAttrChan() ? Match_Success : Match_InvalidOperand;
10825 case MCK_SReg_64_XEXEC:
10835 return Operand.isNull() ? Match_Success : Match_InvalidOperand;
10837 return Match_InvalidOperand;
10846 SMLoc S = getLoc();
10855 return Error(S,
"expected a 16-bit value");
10858 AMDGPUOperand::CreateImm(
this,
Imm, S, AMDGPUOperand::ImmTyEndpgm));
10862bool AMDGPUOperand::isEndpgm()
const {
return isImmTy(ImmTyEndpgm); }
10868bool AMDGPUOperand::isSplitBarrier()
const {
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
SmallVector< int16_t, MAX_SRC_OPERANDS_NUM > OperandIndices
static bool checkWriteLane(const MCInst &Inst)
static bool getRegNum(StringRef Str, unsigned &Num)
static void addSrcModifiersAndSrc(MCInst &Inst, const OperandVector &Operands, unsigned i, unsigned Opc, AMDGPU::OpName OpName)
static constexpr RegInfo RegularRegisters[]
static const RegInfo * getRegularRegInfo(StringRef Str)
static ArrayRef< unsigned > getAllVariants()
static OperandIndices getSrcOperandIndices(unsigned Opcode, bool AddMandatoryLiterals=false)
static int IsAGPROperand(const MCInst &Inst, AMDGPU::OpName Name, const MCRegisterInfo *MRI)
static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
static const fltSemantics * getFltSemantics(unsigned Size)
static bool isRegularReg(RegisterKind Kind)
LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUAsmParser()
Force static initialization.
static bool ConvertOmodMul(int64_t &Mul)
#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)
static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi)
static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT)
static bool AMDGPUCheckMnemonic(StringRef Mnemonic, const FeatureBitset &AvailableFeatures, unsigned VariantID)
static void applyMnemonicAliases(StringRef &Mnemonic, const FeatureBitset &Features, unsigned VariantID)
constexpr unsigned MAX_SRC_OPERANDS_NUM
#define EXPR_RESOLVE_OR_ERROR(RESOLVED)
static bool ConvertOmodDiv(int64_t &Div)
static bool IsRevOpcode(const unsigned Opcode)
static bool encodeCnt(const AMDGPU::IsaVersion ISA, int64_t &IntVal, int64_t CntVal, bool Saturate, unsigned(*encode)(const IsaVersion &Version, unsigned, unsigned), unsigned(*decode)(const IsaVersion &Version, unsigned))
static MCRegister getSpecialRegForName(StringRef RegName)
static void addOptionalImmOperand(MCInst &Inst, const OperandVector &Operands, AMDGPUAsmParser::OptionalImmIndexMap &OptionalIdx, AMDGPUOperand::ImmTy ImmT, int64_t Default=0, std::optional< unsigned > InsertAt=std::nullopt)
static void cvtVOP3DstOpSelOnly(MCInst &Inst, const MCRegisterInfo &MRI)
static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum)
static const fltSemantics * getOpFltSemantics(uint8_t OperandType)
static bool isInvalidVOPDY(const OperandVector &Operands, uint64_t InvalidOprIdx)
static std::string AMDGPUMnemonicSpellCheck(StringRef S, const FeatureBitset &FBS, unsigned VariantID=0)
static LLVM_READNONE unsigned encodeBitmaskPerm(const unsigned AndMask, const unsigned OrMask, const unsigned XorMask)
static bool isSafeTruncation(int64_t Val, unsigned Size)
AMDHSA kernel descriptor MCExpr struct for use in MC layer.
Provides AMDGPU specific target descriptions.
Enums shared between the AMDGPU backend (LLVM) and the ELF linker (LLD) for the .amdgpu....
AMDHSA kernel descriptor definitions.
static bool parseExpr(MCAsmParser &MCParser, const MCExpr *&Value, raw_ostream &Err)
MC layer struct for AMDGPUMCKernelCodeT, provides MCExpr functionality where required.
@ AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_EXTERNAL_VISIBILITY
static llvm::Expected< InlineInfo > decode(GsymDataExtractor &Data, uint64_t &Offset, uint64_t BaseAddr)
Decode an InlineInfo in Data at the specified offset.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Loop::LoopBounds::Direction Direction
static bool hasFeature(StringRef Feature, const FeatureBitset &FeatureBits, ArrayRef< SubtargetFeatureKV > ProcFeatures)
Register const TargetRegisterInfo * TRI
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static bool isReg(const MCInst &MI, unsigned OpNo)
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t IntrinsicInst * II
static cl::opt< RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, "development", "for training")))
Interface definition for SIInstrInfo.
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
This file implements the SmallBitVector class.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, const llvm::StringTable &StandardNames, VectorLibrary VecLib)
Initialize the set of available library functions based on the specified target triple.
static const char * getRegisterName(MCRegister Reg)
static const AMDGPUMCExpr * createMax(ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static unsigned getNumExpectedArgs(VariantKind Kind)
static const AMDGPUMCExpr * createLit(LitModifier Lit, int64_t Value, MCContext &Ctx)
static const AMDGPUMCExpr * create(VariantKind Kind, ArrayRef< const MCExpr * > Args, MCContext &Ctx)
static const AMDGPUMCExpr * createExtraSGPRs(const MCExpr *VCCUsed, const MCExpr *FlatScrUsed, bool XNACKUsed, MCContext &Ctx)
Allow delayed MCExpr resolve of ExtraSGPRs (in case VCCUsed or FlatScrUsed are unresolvable but neede...
static const AMDGPUMCExpr * createAlignTo(const MCExpr *Value, const MCExpr *Align, MCContext &Ctx)
static std::optional< TargetID > parseTargetIDString(StringRef TargetIDDirective)
Parse and validate a TargetID from a full "<triple>-<processor>:<features>" directive string.
TargetIDSetting getXnackSetting() const
GPUKind getGPUKind() const
StringRef getTargetTripleString() const
std::string toString() const
TargetIDSetting getSramEccSetting() const
static const fltSemantics & IEEEsingle()
static const fltSemantics & BFloat()
static const fltSemantics & IEEEdouble()
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEhalf()
opStatus
IEEE-754R 7: Default exception handling.
LLVM_ABI opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
const T & front() const
Get the first element.
size_t size() const
Get the array size.
bool empty() const
Check if the array is empty.
StringRef getString() const
Get the string for the current token, this includes all characters (for example, the quotes on string...
bool is(TokenKind K) const
Container class for subtarget features.
constexpr bool test(unsigned I) const
constexpr FeatureBitset & flip(unsigned I)
void printExpr(raw_ostream &, const MCExpr &) const
virtual void Initialize(MCAsmParser &Parser)
Initialize the extension for parsing using the given Parser.
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc=SMLoc())
static const MCBinaryExpr * createDiv(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
static LLVM_ABI const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Context object for machine code objects.
LLVM_ABI MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Instances of this class represent a single low-level machine instruction.
unsigned getNumOperands() const
unsigned getOpcode() const
iterator insert(iterator I, const MCOperand &Op)
void addOperand(const MCOperand Op)
const MCOperand & getOperand(unsigned i) const
Describe properties that are true of each instruction in the target description file.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
const int16_t * getRegClassByHwModeTable(unsigned ModeId) const
int16_t getOpRegClassID(const MCOperandInfo &OpInfo, unsigned HwModeId) const
Return the ID of the register class to use for OpInfo, for the active HwMode HwModeId.
Instances of this class represent operands of the MCInst class.
static MCOperand createExpr(const MCExpr *Val)
static MCOperand createReg(MCRegister Reg)
static MCOperand createImm(int64_t Val)
void setReg(MCRegister Reg)
Set the register number.
MCRegister getReg() const
Returns the register number.
const MCExpr * getExpr() const
MCParsedAsmOperand - This abstract class represents a source-level assembly instruction operand.
MCRegisterClass - Base class of TargetRegisterClass.
MCRegister getRegister(unsigned i) const
getRegister - Return the specified register in the class.
unsigned getNumRegs() const
getNumRegs - Return the number of registers in this class.
bool contains(MCRegister Reg) const
contains - Return true if the specified register is included in this register class.
MCRegisterInfo base class - We assume that the target defines a static array of MCRegisterDesc object...
bool regsOverlap(MCRegister RegA, MCRegister RegB) const
Returns true if the two registers are equal or alias each other.
const MCRegisterClass & getRegClass(unsigned i) const
Returns the register class associated with the enumeration value.
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
Wrapper class representing physical registers. Should be passed by value.
constexpr bool isValid() const
Generic base class for all target subtargets.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
StringRef getName() const
getName - Get the symbol name.
bool isVariable() const
isVariable - Check if this is a variable symbol.
LLVM_ABI void setVariableValue(const MCExpr *Value)
void setRedefinable(bool Value)
Mark this symbol as redefinable.
const MCExpr * getVariableValue() const
Get the expression of the variable symbol.
MCTargetAsmParser - Generic interface to target specific assembly parsers.
uint64_t getScalarSizeInBits() const
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
Ternary parse status returned by various parse* methods.
constexpr bool isFailure() const
static constexpr StatusTy Failure
constexpr bool isSuccess() const
static constexpr StatusTy Success
static constexpr StatusTy NoMatch
constexpr bool isNoMatch() const
constexpr unsigned id() const
Represents a location in source code.
static SMLoc getFromPointer(const char *Ptr)
constexpr const char * getPointer() const
constexpr bool isValid() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Represent a constant reference to a string, i.e.
bool consume_back(StringRef Suffix)
Returns true if this StringRef has the given suffix and removes that suffix.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr bool empty() const
Check if the string is empty.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
bool consume_front(char Prefix)
Returns true if this StringRef has the given prefix and removes that prefix.
bool contains(StringRef key) const
Check if the set contains the given key.
std::pair< typename Base::iterator, bool > insert(StringRef key)
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM_ABI std::string str() const
Return the twine contents as a std::string.
std::pair< iterator, bool > insert(const ValueT &V)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
int encodeDepCtr(const StringRef Name, int64_t Val, unsigned &UsedOprMask, const MCSubtargetInfo &STI)
int getDefaultDepCtrEncoding(const MCSubtargetInfo &STI)
bool isSupportedTgtId(unsigned Id, const MCSubtargetInfo &STI)
unsigned getTgtId(const StringRef Name)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char NumSGPRs[]
Key for Kernel::CodeProps::Metadata::mNumSGPRs.
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
constexpr char AssemblerDirectiveBegin[]
HSA metadata beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
HSA metadata ending assembler directive.
constexpr char AssemblerDirectiveBegin[]
Old HSA metadata beginning assembler directive for V2.
int64_t getHwregId(StringRef Name, const MCSubtargetInfo &STI)
@ FIXED_NUM_SGPRS_FOR_INIT_BUG
unsigned getVGPREncodingGranule(const MCSubtargetInfo &STI, std::optional< bool > EnableWavefrontSize32)
unsigned getSGPREncodingGranule(const MCSubtargetInfo &STI)
bool targetIDSettingsConflict(TargetIDSetting Lhs, TargetIDSetting Rhs)
Returns true if Lhs and Rhs are incompatible (both specific but different).
unsigned getLocalMemorySize(const MCSubtargetInfo &STI)
constexpr char AssemblerDirective[]
PAL metadata (old linear format) assembler directive.
constexpr char AssemblerDirectiveBegin[]
PAL metadata (new MsgPack format) beginning assembler directive.
constexpr char AssemblerDirectiveEnd[]
PAL metadata (new MsgPack format) ending assembler directive.
int64_t getMsgOpId(int64_t MsgId, StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a sendmsg operation to the operation portion of the immediate encoding.
int64_t getMsgId(StringRef Name, const MCSubtargetInfo &STI)
Map from a symbolic name for a msg_id to the message portion of the immediate encoding.
uint64_t encodeMsg(uint64_t MsgId, uint64_t OpId, uint64_t StreamId)
bool msgSupportsStream(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI)
bool isValidMsgId(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgStream(int64_t MsgId, int64_t OpId, int64_t StreamId, const MCSubtargetInfo &STI, bool Strict)
bool msgRequiresOp(int64_t MsgId, const MCSubtargetInfo &STI)
bool isValidMsgOp(int64_t MsgId, int64_t OpId, const MCSubtargetInfo &STI, bool Strict)
ArrayRef< GFXVersion > getGFXVersions()
constexpr unsigned COMPONENTS[]
constexpr const char *const ModMatrixFmt[]
constexpr const char *const ModMatrixScaleFmt[]
constexpr const char *const ModMatrixScale[]
bool isInlinableLiteralBF16(int16_t Literal, bool HasInv2Pi)
bool isInlineValue(MCRegister Reg)
bool isPKFMACF16InlineConstant(uint32_t Literal, bool IsGFX11Plus)
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
bool isInlinableLiteralFP16(int16_t Literal, bool HasInv2Pi)
bool isSGPR(MCRegister Reg, const MCRegisterInfo *TRI)
Is Reg - scalar register.
MCRegister getMCReg(MCRegister Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
FuncInfoFlags
Per-function flags packed into INFO_FLAGS entries.
uint8_t wmmaScaleF8F6F4FormatToNumRegs(unsigned Fmt)
const int OPR_ID_UNSUPPORTED
bool isInlinableLiteralV2I16(uint32_t Literal)
bool isHi16Reg(MCRegister Reg, const MCRegisterInfo &MRI)
unsigned getTemporalHintType(const MCInstrDesc TID)
LLVM_READONLY bool isLitExpr(const MCExpr *Expr)
bool isInlinableLiteralV2BF16(uint32_t Literal)
LLVM_ABI bool isCPUValidForSubArch(Triple::SubArchType SubArch, GPUKind AK)
Return true if the GPU AK is usable with the triple subarch SubArch.
unsigned getMaxNumUserSGPRs(const MCSubtargetInfo &STI)
unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST)
For pre-GFX12 FLAT instructions the offset must be positive; MSB is ignored and forced to zero.
bool hasA16(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedSignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset, bool IsBuffer)
bool isGFX12Plus(const MCSubtargetInfo &STI)
unsigned getNSAMaxSize(const MCSubtargetInfo &STI, bool HasSampler)
bool hasPackedD16(const MCSubtargetInfo &STI)
bool isGFX940(const MCSubtargetInfo &STI)
bool isInlinableLiteralV2F16(uint32_t Literal)
bool isHsaAbi(const MCSubtargetInfo &STI)
bool isGFX11(const MCSubtargetInfo &STI)
const int OPR_VAL_INVALID
bool getSMEMIsBuffer(unsigned Opc)
bool isPackedSingleSGPRFP32Inst(unsigned Opc)
The opcode is a packed fp32 instruction which only reads low 32 bits of a scalar operand and propagat...
LLVM_ABI unsigned getAddressableNumSGPRs(GPUKind AK)
uint8_t mfmaScaleF8F6F4FormatToNumRegs(unsigned EncodingVal)
LLVM_ABI IsaVersion getIsaVersion(StringRef GPU)
unsigned getRegBitWidth(unsigned RCID)
Get the size in bits of a register from the register class RC.
bool isValid32BitLiteral(uint64_t Val, bool IsFP64)
LLVM_ABI unsigned getTotalNumVGPRs(GPUKind AK, bool IsWave32)
CanBeVOPD getCanBeVOPD(unsigned Opc, unsigned EncodingFamily, bool VOPD3)
LLVM_READNONE bool isLegalDPALU_DPPControl(const MCSubtargetInfo &ST, unsigned DC)
bool isSI(const MCSubtargetInfo &STI)
bool hasPrivateApertureRegs(const MCSubtargetInfo &STI)
unsigned decodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt)
unsigned getWaitcntBitMask(const IsaVersion &Version)
bool isGFX9(const MCSubtargetInfo &STI)
unsigned getVOPDEncodingFamily(const MCSubtargetInfo &ST)
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
bool isKImmOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this a KImm operand?
GPUKind
GPU kinds supported by the AMDGPU target.
bool isGFX90A(const MCSubtargetInfo &STI)
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByEncoding(uint8_t DimEnc)
bool isInlinableLiteral32(int32_t Literal, bool HasInv2Pi)
bool isGFX12(const MCSubtargetInfo &STI)
unsigned encodeExpcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Expcnt)
bool hasMAIInsts(const MCSubtargetInfo &STI)
constexpr bool isSISrcOperand(const MCOperandInfo &OpInfo)
Is this an AMDGPU specific source operand?
LLVM_READONLY const MIMGDimInfo * getMIMGDimInfoByAsmSuffix(StringRef AsmSuffix)
bool hasMIMG_R128(const MCSubtargetInfo &STI)
LLVM_ABI GPUKind parseArchAMDGCN(StringRef CPU)
bool hasG16(const MCSubtargetInfo &STI)
unsigned getAddrSizeMIMGOp(const MIMGBaseOpcodeInfo *BaseOpcode, const MIMGDimInfo *Dim, bool IsA16, bool IsG16Supported)
bool hasArchitectedFlatScratch(const MCSubtargetInfo &STI)
LLVM_READONLY int64_t getLitValue(const MCExpr *Expr)
bool isGFX11Plus(const MCSubtargetInfo &STI)
bool isSISrcFPOperand(const MCInstrDesc &Desc, unsigned OpNo)
Is this floating-point operand?
bool isGFX10Plus(const MCSubtargetInfo &STI)
AMDGPU::TargetID TargetID
int64_t encode32BitLiteral(int64_t Imm, OperandType Type, bool IsLit)
bool isValidWMMAScaleFmtCombination(unsigned AFmt, unsigned AScale, unsigned BFmt, unsigned BScale)
@ OPERAND_KIMM32
Operand with 32-bit immediate that uses the constant bus.
@ OPERAND_REG_INLINE_C_FP64
@ OPERAND_REG_IMM_NOINLINE_FP16
@ OPERAND_REG_INLINE_C_BF16
@ OPERAND_REG_INLINE_C_V2BF16
@ OPERAND_REG_IMM_V2INT64
@ OPERAND_REG_IMM_V2INT16
@ OPERAND_REG_IMM_INT32
Operands with register, 32-bit, or 64-bit immediate.
@ OPERAND_REG_IMM_V2FP16_SPLAT
@ OPERAND_REG_INLINE_C_INT64
@ OPERAND_REG_INLINE_C_INT16
Operands with register or inline constant.
@ OPERAND_REG_IMM_NOINLINE_V2FP16
@ OPERAND_REG_INLINE_C_V2FP16
@ OPERAND_REG_INLINE_AC_INT32
Operands with an AccVGPR register or inline constant.
@ OPERAND_REG_INLINE_AC_FP32
@ OPERAND_REG_IMM_V2INT32
@ OPERAND_REG_INLINE_C_FP32
@ OPERAND_REG_INLINE_C_INT32
@ OPERAND_REG_INLINE_C_V2INT16
@ OPERAND_REG_INLINE_AC_FP64
@ OPERAND_REG_INLINE_C_FP16
@ OPERAND_INLINE_SPLIT_BARRIER_INT32
constexpr bool isBF16SrcOperand(const MCOperandInfo &OpInfo)
Is this a scalar (i.e. not packed) bf16 source operand?
bool isDPALU_DPP(const MCInstrDesc &OpDesc, const MCInstrInfo &MII, const MCSubtargetInfo &ST)
LLVM_ABI StringRef getArchNameAMDGCN(GPUKind AK)
bool hasGDS(const MCSubtargetInfo &STI)
bool isLegalSMRDEncodedUnsignedOffset(const MCSubtargetInfo &ST, int64_t EncodedOffset)
bool isGFX9Plus(const MCSubtargetInfo &STI)
bool hasDPPSrc1SGPR(const MCSubtargetInfo &STI)
const int OPR_ID_DUPLICATE
bool isVOPD(unsigned Opc)
VOPD::InstInfo getVOPDInstInfo(const MCInstrDesc &OpX, const MCInstrDesc &OpY)
unsigned encodeVmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Vmcnt)
unsigned decodeExpcnt(const IsaVersion &Version, unsigned Waitcnt)
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
bool isVI(const MCSubtargetInfo &STI)
bool supportsScaleOffset(const MCInstrInfo &MII, unsigned Opcode)
MCRegister mc2PseudoReg(MCRegister Reg)
Convert hardware register Reg to a pseudo register.
unsigned hasKernargPreload(const MCSubtargetInfo &STI)
bool supportsWGP(const MCSubtargetInfo &STI)
LLVM_READNONE unsigned getOperandSize(const MCOperandInfo &OpInfo)
bool isCI(const MCSubtargetInfo &STI)
unsigned encodeLgkmcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned Lgkmcnt)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
bool isGFX1250Plus(const MCSubtargetInfo &STI)
bool hasPopsExitingWaveID(const MCSubtargetInfo &STI)
unsigned decodeVmcnt(const IsaVersion &Version, unsigned Waitcnt)
bool isInlinableLiteralI16(int32_t Literal, bool HasInv2Pi)
bool hasVOPD(const MCSubtargetInfo &STI)
bool isInlinableLiteral64(int64_t Literal, bool HasInv2Pi)
Is this literal inlinable.
bool isPermlane16(unsigned Opc)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ UNDEF
UNDEF - An undefined node.
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
void validate(const Triple &TT, const FeatureBitset &FeatureBits)
constexpr bool isAtomicRet(const T &...O)
constexpr bool isVOPC(const T &...O)
constexpr bool isVOP3(const T &...O)
constexpr bool isVOP1(const T &...O)
constexpr bool usesTENSOR_CNT(const T &...O)
constexpr bool isMAI(const T &...O)
constexpr bool isVOP2(const T &...O)
constexpr bool isSWMMAC(const T &...O)
constexpr bool isSOP2(const T &...O)
constexpr bool isFLAT(const T &...O)
constexpr bool isVOP3P(const T &...O)
constexpr bool isBuffer(const T &...O)
constexpr bool hasIntClamp(const T &...O)
constexpr bool isAtomicNoRet(const T &...O)
constexpr bool isSMRD(const T &...O)
constexpr bool isVOP3Like(const T &...O)
constexpr bool isMIMG(const T &...O)
constexpr bool isVMEM(const T &...O)
constexpr bool isImage(const T &...O)
constexpr bool isWMMA(const T &...O)
constexpr bool isVOPD3(const T &...O)
constexpr bool isGWS(const T &...O)
constexpr bool isMUBUF(const T &...O)
constexpr bool isSDWA(const T &...O)
constexpr bool isSOPC(const T &...O)
constexpr bool isDOT(const T &...O)
constexpr bool isVSAMPLE(const T &...O)
constexpr bool isDS(const T &...O)
constexpr bool isAtomic(const T &...O)
constexpr bool isGather4(const T &...O)
constexpr bool isPacked(const T &...O)
constexpr bool isDPP(const T &...O)
constexpr bool isSegmentSpecificFLAT(const T &...O)
@ Valid
The data is already valid.
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
EnumSet< Modifier > Modifiers
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
bool errorToBool(Error Err)
Helper for converting an Error to a bool.
StringMapEntry< Value * > ValueName
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
unsigned encode(MaybeAlign A)
Returns a representation of the alignment that encodes undefined as 0.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
static bool isMem(const MachineInstr &MI, unsigned Op)
LLVM_ABI std::pair< StringRef, StringRef > getToken(StringRef Source, StringRef Delimiters=" \t\n\v\f\r")
getToken - This function extracts one token from source, ignoring any leading characters that appear ...
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
testing::Matcher< const detail::ErrorHolder & > Failed()
LLVM_ABI void PrintError(const Twine &Msg)
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
Target & getTheR600Target()
The target for R600 GPUs.
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
SmallVectorImpl< std::unique_ptr< MCParsedAsmOperand > > OperandVector
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Target & getTheGCNTarget()
The target for GCN GPUs.
@ Sub
Subtraction of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
unsigned M0(unsigned Val)
ArrayRef(const T &OneElt) -> ArrayRef< T >
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Target & getTheGCNLegacyTarget()
The target for GCN GPUs, registered under the legacy "amdgcn" architecture name for use with -march.
@ Enabled
Convert any .debug_str_offsets tables to DWARF64 if needed.
void initDefault(const MCSubtargetInfo &STI, MCContext &Ctx, bool InitMCExpr=true)
void validate(const MCSubtargetInfo *STI, MCContext &Ctx)
uint32_t PrivateSegmentSize
SmallVector< std::pair< MCSymbol *, std::string >, 4 > IndirectCalls
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 8 > Calls
SmallVector< FuncInfo, 8 > Funcs
SmallVector< std::pair< MCSymbol *, std::string >, 4 > TypeIds
SmallVector< std::pair< MCSymbol *, MCSymbol * >, 4 > Uses
Instruction set architecture version.
const MCExpr * compute_pgm_rsrc2
const MCExpr * kernarg_size
const MCExpr * kernarg_preload
const MCExpr * compute_pgm_rsrc3
const MCExpr * private_segment_fixed_size
const MCExpr * compute_pgm_rsrc1
static void bits_set(const MCExpr *&Dst, const MCExpr *Value, uint32_t Shift, uint32_t Mask, MCContext &Ctx)
const MCExpr * group_segment_fixed_size
static MCKernelDescriptor getDefaultAmdhsaKernelDescriptor(const MCSubtargetInfo *STI, MCContext &Ctx)
const MCExpr * kernel_code_properties
RegisterMCAsmParser - Helper template for registering a target specific assembly parser,...
uint32_t group_segment_fixed_size
uint32_t private_segment_fixed_size
1.14.0